JointCloud
/
pcm-modelarts
26
0
Fork 3
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: ADD received message larger than max

This commit is contained in:
qiwang 2025-03-11 12:16:51 +08:00
parent 56ecf12732
commit d676f57aac
9 changed files with 5408 additions and 5132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
client/imagesservice/imagesservice.go
View File

@ -75,6 +75,7 @@ type (
DataVolumesRes = modelarts.DataVolumesRes
Dataset = modelarts.Dataset
DatasetTra = modelarts.DatasetTra
DatasetTrain = modelarts.DatasetTrain
DeleteAlgorithmsReq = modelarts.DeleteAlgorithmsReq
DeleteAlgorithmsResp = modelarts.DeleteAlgorithmsResp
DeleteDataSetReq = modelarts.DeleteDataSetReq

1
client/modelartsservice/modelartsservice.go
View File

@ -75,6 +75,7 @@ type (
DataVolumesRes = modelarts.DataVolumesRes
Dataset = modelarts.Dataset
DatasetTra = modelarts.DatasetTra
DatasetTrain = modelarts.DatasetTrain
DeleteAlgorithmsReq = modelarts.DeleteAlgorithmsReq
DeleteAlgorithmsResp = modelarts.DeleteAlgorithmsResp
DeleteDataSetReq = modelarts.DeleteDataSetReq

68
internal/logic/httpmodelarts/dataset.go
View File

@ -33,3 +33,71 @@ type CreateRespError struct {
ErrorCode string `json:"errorCode"`
ErrorMsg string `json:"errorMsg"`
}
type CreateDatasetVersionsReq struct {
VersionName string `json:"version_name"`
}
type CreateDatasetVersionsResp struct {
VersionId string `json:"version_id"`
}
type GetDatasetResp struct {
VersionId string `json:"version_id"`
}
// DataSource 定义 data_sources 数组中的元素结构
type DataSource struct {
DataType int `json:"data_type"`
DataPath string `json:"data_path"`
}
// LabelProperty 定义 labels 中每个标签的属性结构
type LabelProperty struct {
Color string `json:"@modelarts:color"`
}
// Label 定义 labels 数组中的元素结构
type Label struct {
Name string `json:"name"`
Type int `json:"type"`
Property LabelProperty `json:"property"`
}
// Dataset 定义整个数据集的结构体
type GetDatasetsResp struct {
DatasetID string `json:"dataset_id"`
DatasetName string `json:"dataset_name"`
DatasetType int `json:"dataset_type"`
DataFormat string `json:"data_format"`
NextVersionNum int `json:"next_version_num"`
Status int `json:"status"`
DataSources []DataSource `json:"data_sources"`
CreateTime int64 `json:"create_time"`
UpdateTime int64 `json:"update_time"`
Description string `json:"description"`
CurrentVersionID string `json:"current_version_id"`
CurrentVersionName string `json:"current_version_name"`
TotalSampleCount int `json:"total_sample_count"`
AnnotatedSampleCount int `json:"annotated_sample_count"`
UnconfirmedSampleCount int `json:"unconfirmed_sample_count"`
WorkPath string `json:"work_path"`
InnerWorkPath string `json:"inner_work_path"`
InnerAnnotationPath string `json:"inner_annotation_path"`
InnerDataPath string `json:"inner_data_path"`
InnerLogPath string `json:"inner_log_path"`
InnerTempPath string `json:"inner_temp_path"`
InnerTaskPath string `json:"inner_task_path"`
WorkPathType int `json:"work_path_type"`
WorkspaceID string `json:"workspace_id"`
EnterpriseProjectID string `json:"enterprise_project_id"`
WorkforceTaskCount int `json:"workforce_task_count"`
FeatureSupports []string `json:"feature_supports"`
Managed bool `json:"managed"`
ImportData bool `json:"import_data"`
LabelTaskCount int `json:"label_task_count"`
DatasetFormat int `json:"dataset_format"`
DatasetVersionCount int `json:"dataset_version_count"`
ContentLabeling bool `json:"content_labeling"`
Labels []Label `json:"labels"`
}

55
internal/logic/httpmodelarts/dataset_ipml.go
View File

@ -8,6 +8,7 @@ import (
"gitlink.org.cn/JointCloud/pcm-modelarts/internal/model"
"gitlink.org.cn/JointCloud/pcm-modelarts/internal/util"
"net/http"
"time"
)
func CreateDataset(ctx *gin.Context) {
@ -54,13 +55,65 @@ func CreateDataset(ctx *gin.Context) {
return
}
json.Unmarshal(convertedCamelCaseJson, &createDatasetResp)
//Status := GetDatasets(ctx, createDatasetResp.DatasetId)
fmt.Println("开始检查数据集状态...")
for {
fmt.Println("查詢数据集状态...")
status := GetDatasets(ctx, createDatasetResp.DatasetId)
if status == 1 {
fmt.Println("数据集已正常，可以继续下一步操作。")
break // 一旦状态变为published，就退出循环
}
fmt.Println("数据集正在创建中，等待 5 秒后再次检查...")
time.Sleep(5 * time.Second)
}
fmt.Println("数据集已正常，可以继续下一步操作。")
VersionId := CreateDatasetVersions(ctx, createDatasetResp.DatasetId, param.DatasetName)
if createDatasetResp.DatasetId == "" || len(createDatasetResp.DatasetId) == 0 {
json.Unmarshal(convertedCamelCaseJson, &CreateRespError)
model.Response(ctx, 500, "error", CreateRespError)
} else {
model.Response(ctx, http.StatusOK, "success", map[string]string{
"id": createDatasetResp.DatasetId,
"name": param.DatasetName,
"name": VersionId,
})
}
}
func CreateDatasetVersions(ctx *gin.Context, DatasetId string, Name string) (VersionId string) {
var param CreateDatasetVersionsReq
var createDatasetVersionsResp CreateDatasetVersionsResp
platform, _ := util.GetModelArtsConfWithPlatform("modelarts-CloudBrain2")
param.VersionName = Name
reqByte, _ := json.Marshal(param)
body, err := util.SendRequestHttp("POST", platform.Endpoint+"v2/"+platform.ProjectId+"/datasets/"+DatasetId+"/versions",
bytes.NewBuffer(reqByte), "modelarts-CloudBrain2")
if err != nil {
model.Response(ctx, 500, "failed to invoke", err)
return
}
error := json.Unmarshal(body, &createDatasetVersionsResp)
if error != nil {
fmt.Println("Error marshalling JSON:", err)
return
}
return createDatasetVersionsResp.VersionId
}
func GetDatasets(ctx *gin.Context, DatasetId string) (status int) {
var getDatasetsResp GetDatasetsResp
platform, _ := util.GetModelArtsConfWithPlatform("modelarts-CloudBrain2")
body, err := util.SendRequestHttp("GET", platform.Endpoint+"v2/"+platform.ProjectId+"/datasets/"+DatasetId,
nil, "modelarts-CloudBrain2")
if err != nil {
model.Response(ctx, 500, "failed to invoke", err)
return
}
error := json.Unmarshal(body, &getDatasetsResp)
if error != nil {
fmt.Println("Error marshalling JSON:", err)
return
}
return getDatasetsResp.Status
}

14
internal/logic/httpmodelarts/trainjoblogspreview.go Normal file
View File

@ -0,0 +1,14 @@
package httpmodelarts
type GetTrainingJobLogsPreviewReq struct {
ProjectId string `json:"project_id,omitempty"`
TrainingJobId string `json:"training_job_id,omitempty"`
TaskId string `json:"task_id,omitempty"`
Platform string `json:"platform ,omitempty"`
}
type GetTrainingJobLogsPreviewResp struct {
Content string `json:"content ,omitempty"`
CurrentSize int32 `json:" current_size ,omitempty"`
FullSize int32 `json:" full_size ,omitempty"`
}

7
internal/logic/httpmodelarts/trainjoblogspreview_ipml.go Normal file
View File

@ -0,0 +1,7 @@
package httpmodelarts
import "github.com/gin-gonic/gin"
func GetTrainingJobLogsPreview(ctx *gin.Context) {
}

10372
modelarts/pcm-modelarts.pb.go
View File

File diff suppressed because it is too large Load Diff

14
pb/pcm-modelarts.proto
View File

@ -583,8 +583,10 @@ message Algorithms {
message InputTraining{
string name = 1; // @gotags: copier:"Name"
string access_method = 2; // @gotags: copier:"AccessMethod"
RemoteTra remote = 3; // @gotags: copier:"RemoteIn"
string description = 2;
string local_dir = 3; // @gotags: copier:"AccessMethod"
RemoteTra remote = 4; // @gotags: copier:"RemoteIn"
string access_method =5;
}
message OutputTraining{
string name = 1; // @gotags: copier:"Name"
@ -594,7 +596,15 @@ message OutputTraining{
}
message RemoteTra{
ObsTra obs = 1; // @gotags: copier:"ObsTra"
DatasetTrain dataset =2;
}
message DatasetTrain {
string id =1;
string version_id =3;
string obs_url =2;
}
message RemoteOut{
ObsTra obs = 1; // @gotags: copier:"Obs"
}

8
routers/router.go
View File

@ -22,9 +22,17 @@ func InitRouter(conf *config.Configuration) (*gin.Engine, error) {
datasets := apiv1.Group("datasets")
datasets.POST("/create", httpmodelarts.CreateDataset)
//datasets
/*versions := apiv1.Group("versions")
versions.POST("/create", httpmodelarts.PocessorTasksVersions)*/
//algorithms
algorithms := apiv1.Group("algorithms", httpmodelarts.CreateAlgorithms)
algorithms.POST("/create")
//algorithms
trainingJobLogsPreview := apiv1.Group("getTrainingJobLogsPreview", httpmodelarts.GetTrainingJobLogsPreview)
trainingJobLogsPreview.GET("/getTrainingJobLogsPreview")
}
return r, nil