Merge pull request 'update modelarts algorithm' (#40) from tzwang/pcm-participant:master into master

This commit is contained in:
tzwang 2025-07-28 17:27:39 +08:00
commit fe28feeee9
8 changed files with 548 additions and 6 deletions

View File

@ -1,5 +1,6 @@
package common
import "C"
import (
"bytes"
"crypto/tls"

View File

@ -5,4 +5,7 @@ const (
// algorithm
Algorithm = "v2/{project_id}/algorithms"
AlgorithmDetail = "v2/{project_id}/algorithms/{algorithm_id}"
// job
TrainingJobList = "v2/{project_id}/training-job-searches"
)

View File

@ -9,3 +9,161 @@ type AlgorithmListParam struct {
Searches string `json:"searches,omitempty"`
WorkspaceId string `json:"workspace_id,omitempty"`
}
type AlgorithmList struct {
Total int `json:"total"`
Count int `json:"count"`
Limit int `json:"limit"`
Offset int `json:"offset"`
SortBy string `json:"sort_by"`
Order string `json:"order"`
GroupBy string `json:"group_by"`
Items []struct {
Metadata struct {
Id string `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
CreateTime int64 `json:"create_time"`
WorkspaceId string `json:"workspace_id"`
AiProject string `json:"ai_project"`
UserName string `json:"user_name"`
DomainId string `json:"domain_id"`
Source string `json:"source"`
ApiVersion string `json:"api_version"`
IsValid bool `json:"is_valid"`
State string `json:"state"`
Size int `json:"size"`
Tags interface{} `json:"tags"`
AttrList interface{} `json:"attr_list"`
VersionNum int `json:"version_num"`
UpdateTime int `json:"update_time"`
} `json:"metadata"`
ShareInfo struct {
} `json:"share_info"`
JobConfig struct {
CodeDir string `json:"code_dir"`
BootFile string `json:"boot_file"`
ParametersCustomization bool `json:"parameters_customization"`
Engine struct {
EngineId string `json:"engine_id"`
EngineName string `json:"engine_name"`
EngineVersion string `json:"engine_version"`
V1Compatible bool `json:"v1_compatible"`
ImageUrl string `json:"image_url"`
NonSwrImage bool `json:"non_swr_image"`
RunUser string `json:"run_user"`
ImageSource bool `json:"image_source"`
ImageRepoId string `json:"image_repo_id"`
InstallSysPackages bool `json:"install_sys_packages,omitempty"`
} `json:"engine"`
} `json:"job_config"`
ResourceRequirements []interface{} `json:"resource_requirements"`
AdvancedConfig struct {
} `json:"advanced_config"`
} `json:"items"`
}
type Algorithm struct {
Metadata struct {
Id string `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
CreateTime int64 `json:"create_time"`
WorkspaceId string `json:"workspace_id"`
AiProject string `json:"ai_project"`
UserName string `json:"user_name"`
DomainId string `json:"domain_id"`
Source string `json:"source"`
ApiVersion string `json:"api_version"`
IsValid bool `json:"is_valid"`
State string `json:"state"`
Size int `json:"size"`
Tags interface{} `json:"tags"`
AttrList interface{} `json:"attr_list"`
VersionNum int `json:"version_num"`
UpdateTime int `json:"update_time"`
} `json:"metadata"`
ShareInfo struct {
} `json:"share_info"`
JobConfig struct {
CodeDir string `json:"code_dir"`
BootFile string `json:"boot_file"`
ParametersCustomization bool `json:"parameters_customization"`
Engine struct {
EngineId string `json:"engine_id"`
EngineName string `json:"engine_name"`
EngineVersion string `json:"engine_version"`
V1Compatible bool `json:"v1_compatible"`
ImageUrl string `json:"image_url"`
NonSwrImage bool `json:"non_swr_image"`
RunUser string `json:"run_user"`
ImageSource bool `json:"image_source"`
ImageRepoId string `json:"image_repo_id"`
} `json:"engine"`
} `json:"job_config"`
ResourceRequirements interface{} `json:"resource_requirements"`
AdvancedConfig struct {
} `json:"advanced_config"`
}
type AlgorithmCreateParam struct {
AdvancedConfig struct {
} `json:"advanced_config,omitempty"`
JobConfig struct {
CodeDir string `json:"code_dir,omitempty"`
BootFile string `json:"boot_file,omitempty"`
Engine struct {
EngineName string `json:"engine_name,omitempty"`
EngineVersion string `json:"engine_version,omitempty"`
ImageUrl string `json:"image_url,omitempty"`
InstallSysPackages bool `json:"install_sys_packages,omitempty"`
} `json:"engine"`
Command string `json:"command,omitempty"`
Inputs []struct {
Name string `json:"name,omitempty"`
Description string `json:"description,omitempty"`
AccessMethod string `json:"access_method,omitempty"`
I18NDescription []struct {
} `json:"i18n_description,omitempty"`
RemoteConstraints []struct {
DataType string `json:"data_type,omitempty"`
Attributes *struct {
DataFormat []string `json:"data_format,omitempty"`
DataSegmentation []string `json:"data_segmentation,omitempty"`
DatasetType []string `json:"dataset_type,omitempty"`
} `json:"attributes,omitempty"`
} `json:"remote_constraints,omitempty"`
} `json:"inputs,omitempty"`
Outputs []struct {
Name string `json:"name,omitempty"`
Description string `json:"description,omitempty"`
AccessMethod string `json:"access_method,omitempty"`
I18NDescription []struct {
} `json:"i18n_description,omitempty"`
} `json:"outputs,omitempty"`
Parameters []struct {
Name string `json:"name,omitempty"`
Description string `json:"description,omitempty"`
Value string `json:"value,omitempty"`
Constraint struct {
Editable bool `json:"editable,omitempty"`
Required bool `json:"required,omitempty"`
Sensitive bool `json:"sensitive,omitempty"`
Type string `json:"type,omitempty"`
ValidRange []interface{} `json:"valid_range,omitempty"`
ValidType string `json:"valid_type,omitempty"`
} `json:"constraint,omitempty"`
} `json:"parameters,omitempty"`
ParametersCustomization bool `json:"parameters_customization,omitempty"`
} `json:"job_config"`
Metadata struct {
Name string `json:"name"`
Description string `json:"description,omitempty"`
Tags interface{} `json:"tags,omitempty"`
WorkspaceId string `json:"workspace_id,omitempty"`
} `json:"metadata"`
ResourceRequirements []struct {
Key string `json:"key,omitempty"`
Operator string `json:"operator,omitempty"`
Values []string `json:"values,omitempty"`
} `json:"resource_requirements,omitempty"`
}

View File

@ -0,0 +1,224 @@
package model
type JobListSearchParam struct {
Limit int `json:"limit,omitempty"`
Offset int `json:"offset,omitempty"`
SortBy string `json:"sort_by,omitempty"`
Order string `json:"order,omitempty"`
GroupBy string `json:"group_by,omitempty"`
Filters []struct {
Key string `json:"key,omitempty"`
Operator string `json:"operator,omitempty"`
Value []string `json:"value,omitempty"`
} `json:"filters,omitempty"`
}
type JobList struct {
Total int `json:"total"`
Count int `json:"count"`
Limit int `json:"limit"`
Offset int `json:"offset"`
SortBy string `json:"sort_by"`
Order string `json:"order"`
GroupBy string `json:"group_by"`
WorkspaceId string `json:"workspace_id"`
AiProject string `json:"ai_project"`
Items []struct {
Kind string `json:"kind"`
Metadata struct {
Id string `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
CreateTime int64 `json:"create_time"`
WorkspaceId string `json:"workspace_id"`
AiProject string `json:"ai_project"`
UserName string `json:"user_name"`
Annotations struct {
JobTemplate string `json:"job_template"`
KeyTask string `json:"key_task"`
} `json:"annotations"`
TrainingExperimentReference struct {
} `json:"training_experiment_reference"`
Tags []interface{} `json:"tags"`
Labels struct {
TrainingJob string `json:"training-job"`
} `json:"labels,omitempty"`
} `json:"metadata"`
Status struct {
Phase string `json:"phase"`
SecondaryPhase string `json:"secondary_phase"`
PendingTime int `json:"pending_time"`
Duration int `json:"duration"`
IsHanged bool `json:"is_hanged"`
RetryCount int `json:"retry_count"`
StartTime int64 `json:"start_time"`
NodeCountMetrics [][]int64 `json:"node_count_metrics"`
Tasks []string `json:"tasks"`
MetricsStatistics struct {
CpuUsage struct {
Average float64 `json:"average"`
Max float64 `json:"max"`
Min float64 `json:"min"`
} `json:"cpu_usage"`
MemUsage struct {
Average float64 `json:"average"`
Max float64 `json:"max"`
Min float64 `json:"min"`
} `json:"mem_usage"`
Npu struct {
MemUsage struct {
Average float64 `json:"average"`
Max float64 `json:"max"`
Min float64 `json:"min"`
} `json:"mem_usage"`
Util struct {
Average float64 `json:"average"`
Max float64 `json:"max"`
Min int `json:"min"`
} `json:"util"`
} `json:"npu"`
} `json:"metrics_statistics"`
RunningRecords []struct {
StartAt int `json:"start_at"`
EndAt int `json:"end_at"`
StartType string `json:"start_type"`
EndReason string `json:"end_reason"`
} `json:"running_records"`
} `json:"status"`
Algorithm struct {
Id string `json:"id"`
Name string `json:"name"`
CodeDir string `json:"code_dir"`
BootFile string `json:"boot_file"`
ParametersCustomization bool `json:"parameters_customization"`
Inputs []struct {
Name string `json:"name"`
LocalDir string `json:"local_dir"`
AccessMethod string `json:"access_method"`
Remote struct {
Obs struct {
ObsUrl string `json:"obs_url"`
} `json:"obs"`
} `json:"remote"`
} `json:"inputs,omitempty"`
Outputs []struct {
Name string `json:"name"`
LocalDir string `json:"local_dir"`
AccessMethod string `json:"access_method"`
Remote struct {
Obs struct {
ObsUrl string `json:"obs_url"`
} `json:"obs"`
} `json:"remote"`
Mode string `json:"mode"`
Period int `json:"period"`
PrefetchToLocal bool `json:"prefetch_to_local"`
} `json:"outputs,omitempty"`
Engine struct {
EngineId string `json:"engine_id"`
EngineName string `json:"engine_name"`
EngineVersion string `json:"engine_version"`
V1Compatible bool `json:"v1_compatible"`
ImageUrl string `json:"image_url"`
NonSwrImage bool `json:"non_swr_image"`
RunUser string `json:"run_user"`
ImageSource bool `json:"image_source"`
ImageRepoId string `json:"image_repo_id"`
InstallSysPackages bool `json:"install_sys_packages,omitempty"`
} `json:"engine"`
Environments struct {
PCMDATASET string `json:"PCM_DATASET"`
Output string `json:"output,omitempty"`
} `json:"environments,omitempty"`
Policies struct {
} `json:"policies,omitempty"`
Parameters []struct {
Name string `json:"name"`
Description string `json:"description"`
I18NDescription interface{} `json:"i18n_description"`
Value string `json:"value"`
Constraint struct {
Type string `json:"type"`
Editable bool `json:"editable"`
Required bool `json:"required"`
Sensitive bool `json:"sensitive"`
ValidType string `json:"valid_type"`
ValidRange interface{} `json:"valid_range"`
} `json:"constraint"`
} `json:"parameters,omitempty"`
} `json:"algorithm"`
Spec struct {
Resource struct {
Policy string `json:"policy,omitempty"`
FlavorId string `json:"flavor_id"`
FlavorName string `json:"flavor_name,omitempty"`
PoolResourceFlavor string `json:"pool_resource_flavor"`
NodeCount int `json:"node_count"`
FlavorDetail struct {
FlavorType string `json:"flavor_type"`
Billing struct {
Code string `json:"code"`
UnitNum int `json:"unit_num"`
} `json:"billing"`
Attributes struct {
IsMultiDevicesSupported string `json:"is_multi_devices_supported"`
IsMultiNodesSupported string `json:"is_multi_nodes_supported"`
} `json:"attributes"`
FlavorInfo struct {
Cpu struct {
Arch string `json:"arch"`
CoreNum int `json:"core_num"`
} `json:"cpu"`
Npu struct {
UnitNum int `json:"unit_num"`
ProductName string `json:"product_name"`
Memory string `json:"memory"`
} `json:"npu"`
Memory struct {
Size int `json:"size"`
Unit string `json:"unit"`
} `json:"memory"`
Disk struct {
Size int `json:"size"`
Unit string `json:"unit"`
} `json:"disk"`
} `json:"flavor_info"`
} `json:"flavor_detail,omitempty"`
PoolId string `json:"pool_id,omitempty"`
PoolInfo struct {
CpuArch string `json:"cpu_arch"`
CoreNum int `json:"core_num"`
MemSize int `json:"mem_size"`
CacheSize int `json:"cache_size"`
Accelerator string `json:"accelerator"`
AcceleratorNum int `json:"accelerator_num"`
AcceleratorType string `json:"accelerator_type"`
AcceleratorSize int `json:"accelerator_size"`
UsePrivileged bool `json:"use_privileged"`
UseHostNetwork bool `json:"use_host_network"`
UseIbNetwork bool `json:"use_ib_network"`
ProjectId string `json:"project_id"`
PoolId string `json:"pool_id"`
ClusterId string `json:"cluster_id"`
MaosPool bool `json:"maos_pool"`
QuotaId string `json:"quota_id"`
MaosMigrated bool `json:"maos_migrated"`
DetectAllInInt bool `json:"detect_all_in_int"`
PoolType string `json:"pool_type"`
EnableCabinet bool `json:"enable_cabinet"`
EnableMemarts bool `json:"enable_memarts"`
EnableEms bool `json:"enable_ems"`
} `json:"pool_info,omitempty"`
} `json:"resource"`
LogExportPath struct {
ObsUrl string `json:"obs_url"`
} `json:"log_export_path"`
IsHostedLog bool `json:"is_hosted_log"`
SchedulePolicy struct {
Priority int `json:"priority"`
RequiredAffinity struct {
} `json:"required_affinity"`
} `json:"schedule_policy,omitempty"`
} `json:"spec"`
} `json:"items"`
}

View File

@ -1,13 +1,85 @@
package service
import (
json "github.com/json-iterator/go"
"gitlink.org.cn/JointCloud/pcm-participant-modelarts/common"
"gitlink.org.cn/JointCloud/pcm-participant-modelarts/model"
"net/http"
"net/url"
"strconv"
"strings"
)
func AlgorithmList(ip, projectId string, param *model.AlgorithmListParam) (interface{}, error) {
func AlgorithmList(ip, projectId string, param *model.AlgorithmListParam) (*model.AlgorithmList, error) {
var reqUrl = ip + common.Forward_Slash + common.Algorithm
qs := url.Values{}
if param != nil {
if param.Limit > 0 {
qs.Add("limit", strconv.Itoa(param.Limit))
}
if param.Offset > 0 {
qs.Add("offset", strconv.Itoa(param.Offset))
}
if param.SortBy != "" {
qs.Add("sort_by", param.SortBy)
}
if param.Order != "" {
qs.Add("order", param.Order)
}
if param.GroupBy != "" {
qs.Add("group_by", param.GroupBy)
}
if param.Searches != "" {
qs.Add("searches", param.Searches)
}
if param.WorkspaceId != "" {
qs.Add("workspace_id", param.WorkspaceId)
}
}
reqUrl = strings.Replace(reqUrl, "{project_id}", projectId, -1)
if len(qs) > 0 {
reqUrl += "?" + qs.Encode()
}
body, err := common.SendRequestHttp(http.MethodGet, reqUrl, nil, "")
if err != nil {
return nil, err
}
resp := &model.AlgorithmList{}
err = json.Unmarshal(body, resp)
if err != nil {
return nil, err
}
return resp, nil
}
func AlgorithmDetail(ip, projectId, algorithmId string) (*model.Algorithm, error) {
var reqUrl = ip + common.Forward_Slash + common.AlgorithmDetail
reqUrl = strings.Replace(reqUrl, "{project_id}", projectId, -1)
reqUrl = strings.Replace(reqUrl, "{algorithm_id}", algorithmId, -1)
body, err := common.SendRequestHttp(http.MethodGet, reqUrl, nil, "")
if err != nil {
return nil, err
}
resp := &model.Algorithm{}
err = json.Unmarshal(body, resp)
if err != nil {
return nil, err
}
return resp, nil
}
func AlgorithmCreate(ip, projectId string, param *model.AlgorithmCreateParam) (*model.Algorithm, error) {
var reqUrl = ip + common.Forward_Slash + common.Algorithm
reqUrl = strings.Replace(reqUrl, "{project_id}", projectId, -1)
@ -16,7 +88,14 @@ func AlgorithmList(ip, projectId string, param *model.AlgorithmListParam) (inter
if err != nil {
return nil, err
}
resp, err := common.SendRequestHttp(http.MethodGet, reqUrl, reader, "")
body, err := common.SendRequestHttp(http.MethodPost, reqUrl, reader, "")
if err != nil {
return nil, err
}
resp := &model.Algorithm{}
err = json.Unmarshal(body, resp)
if err != nil {
return nil, err
}

View File

@ -9,10 +9,12 @@ import (
)
func TestAlgorithm(t *testing.T) {
url := ""
projectId := ""
url := "https://modelarts.cn-south-222.ai.pcl.cn"
projectId := "d18190e28e3f45a281ef0b0696ec9d52"
convey.Convey("AlgorithmList", t, func() {
param := &model.AlgorithmListParam{}
param := &model.AlgorithmListParam{
Offset: 5,
}
list, err := AlgorithmList(url, projectId, param)
if err != nil {
fmt.Println(err.Error())
@ -23,8 +25,25 @@ func TestAlgorithm(t *testing.T) {
})
convey.Convey("AlgorithmList", t, func() {
convey.Convey("AlgorithmDetail", t, func() {
id := "d126a422-4ead-4c08-9ace-bad9451308c1"
resp, err := AlgorithmDetail(url, projectId, id)
if err != nil {
fmt.Println(err.Error())
}
fmt.Println(resp)
convey.So(err, convey.ShouldBeNil)
})
convey.Convey("AlgorithmCreate", t, func() {
param := &model.AlgorithmCreateParam{}
resp, err := AlgorithmCreate(url, projectId, param)
if err != nil {
fmt.Println(err.Error())
}
fmt.Println(resp)
convey.So(err, convey.ShouldBeNil)
})
}

View File

@ -0,0 +1,33 @@
package service
import (
json "github.com/json-iterator/go"
"gitlink.org.cn/JointCloud/pcm-participant-modelarts/common"
"gitlink.org.cn/JointCloud/pcm-participant-modelarts/model"
"net/http"
"strings"
)
func TrainingJobList(ip, projectId string, param *model.JobListSearchParam) (*model.JobList, error) {
var reqUrl = ip + common.Forward_Slash + common.TrainingJobList
reqUrl = strings.Replace(reqUrl, "{project_id}", projectId, -1)
reader, err := common.ToIOReader(param)
if err != nil {
return nil, err
}
body, err := common.SendRequestHttp(http.MethodPost, reqUrl, reader, "")
if err != nil {
return nil, err
}
resp := &model.JobList{}
err = json.Unmarshal(body, resp)
if err != nil {
return nil, err
}
return resp, nil
}

View File

@ -0,0 +1,25 @@
package service
import (
"fmt"
"github.com/smartystreets/goconvey/convey"
"gitlink.org.cn/JointCloud/pcm-participant-modelarts/model"
"testing"
)
func TestJob(t *testing.T) {
url := "https://modelarts.cn-south-222.ai.pcl.cn"
projectId := "d18190e28e3f45a281ef0b0696ec9d52"
convey.Convey("AlgorithmList", t, func() {
param := &model.JobListSearchParam{}
list, err := TrainingJobList(url, projectId, param)
if err != nil {
fmt.Println(err.Error())
}
convey.So(err, convey.ShouldBeNil)
convey.So(list, convey.ShouldNotBeEmpty)
})
}