JointCloud
/
pcm-participant
24
0
Fork 2
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' of https://gitlink.org.cn/JointCloud/pcm-participant into master-zw

This commit is contained in:
zhangwei 2025-07-23 08:42:52 +08:00
parent ceae117b21 20e45d18d4
commit 860d651f7d
13 changed files with 54 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
ai/service/algorithm_test.go
View File

@ -9,6 +9,7 @@ import (
aicom "gitlink.org.cn/JointCloud/pcm-participant-ai/common"
aiconf "gitlink.org.cn/JointCloud/pcm-participant-ai/config"
"gitlink.org.cn/JointCloud/pcm-participant-ai/platform"
"gitlink.org.cn/JointCloud/pcm-participant-client/config"
"gitlink.org.cn/JointCloud/pcm-participant-octopus"
openI "gitlink.org.cn/JointCloud/pcm-participant-openi"
"gitlink.org.cn/JointCloud/pcm-participant-openi/common"
@ -16,9 +17,18 @@ import (
"time"
)
var (
Jcs = &config.Jcs{
Name: aiconf.Cfg["jcs"].Name,
ApiKey: aiconf.Cfg["jcs"].APIKey,
Url: aiconf.Cfg["jcs"].URL,
}
)
func TestGet(t *testing.T) {
convey.Convey(aiconf.Octopus, t, func() {
o, err := octopus.New(aiconf.Cfg[aiconf.Octopus].URL, aiconf.Cfg[aiconf.Octopus].Username, aiconf.Cfg[aiconf.Octopus].Password, aiconf.Octopus, platform.Id(123))
o, err := octopus.New(Jcs, aiconf.Cfg[aiconf.Octopus].URL, aiconf.Cfg[aiconf.Octopus].Username, aiconf.Cfg[aiconf.Octopus].Password, aiconf.Octopus, platform.Id(123))
if err != nil {
fmt.Println(err)
return
@ -41,7 +51,7 @@ func TestGet(t *testing.T) {
})
convey.Convey(aiconf.OpenI, t, func() {
o, _ := openI.New(aiconf.Cfg[aiconf.OpenI].Username, aiconf.Cfg[aiconf.OpenI].Password, aiconf.Cfg[aiconf.OpenI].APIKey, aiconf.OpenI, platform.Id(123), aiconf.Cfg[aiconf.OpenI].DataRepo)
o, _ := openI.New(Jcs, aiconf.Cfg[aiconf.OpenI].Username, aiconf.Cfg[aiconf.OpenI].Password, aiconf.Cfg[aiconf.OpenI].APIKey, aiconf.OpenI, platform.Id(123), aiconf.Cfg[aiconf.OpenI].DataRepo)
common.InitClient()
alg := NewAlgorithm(o.Alg)
@ -92,7 +102,7 @@ func TestCreate(t *testing.T) {
//o, _ := openI.New("nudt-ysz", "", "8cff1d2db9171462c02901d086d13221389fd082", platform.Id(123), "data")
//common.InitClient()
oct, _ := octopus.New(aiconf.Cfg[aiconf.Octopus].URL, aiconf.Cfg[aiconf.Octopus].Username, aiconf.Cfg[aiconf.Octopus].Password, aiconf.Octopus, platform.Id(123))
oct, _ := octopus.New(Jcs, aiconf.Cfg[aiconf.Octopus].URL, aiconf.Cfg[aiconf.Octopus].Username, aiconf.Cfg[aiconf.Octopus].Password, aiconf.Octopus, platform.Id(123))
alg := NewAlgorithm(oct.Alg)

6
ai/service/dataset_test.go
View File

@ -18,7 +18,7 @@ import (
func TestDataset(t *testing.T) {
convey.Convey(aiconf.OpenI, t, func() {
o, _ := openI.New(aiconf.Cfg[aiconf.OpenI].Username, aiconf.Cfg[aiconf.OpenI].Password, aiconf.Cfg[aiconf.OpenI].APIKey, aiconf.OpenI, platform.Id(123), aiconf.Cfg[aiconf.OpenI].DataRepo)
o, _ := openI.New(Jcs, aiconf.Cfg[aiconf.OpenI].Username, aiconf.Cfg[aiconf.OpenI].Password, aiconf.Cfg[aiconf.OpenI].APIKey, aiconf.OpenI, platform.Id(123), aiconf.Cfg[aiconf.OpenI].DataRepo)
common.InitClient()
ds := NewDataset(o.Ds)
@ -62,7 +62,7 @@ func TestDataset(t *testing.T) {
})
convey.Convey(aiconf.Octopus, t, func() {
oct, _ := octopus.New(aiconf.Cfg[aiconf.Octopus].URL, aiconf.Cfg[aiconf.Octopus].Username, aiconf.Cfg[aiconf.Octopus].Password, aiconf.Octopus, platform.Id(123))
oct, _ := octopus.New(Jcs, aiconf.Cfg[aiconf.Octopus].URL, aiconf.Cfg[aiconf.Octopus].Username, aiconf.Cfg[aiconf.Octopus].Password, aiconf.Octopus, platform.Id(123))
ds := NewDataset(oct.Ds)
@ -90,7 +90,7 @@ func TestCreateDataset(t *testing.T) {
//o, _ := openI.New(aiconf.Cfg[aiconf.OpenI].Username, aiconf.Cfg[aiconf.OpenI].Password, aiconf.Cfg[aiconf.OpenI].APIKey, aiconf.OpenI, platform.Id(123), aiconf.Cfg[aiconf.OpenI].DataRepo)
//common.InitClient()
oct, _ := octopus.New(aiconf.Cfg[aiconf.Octopus].URL, aiconf.Cfg[aiconf.Octopus].Username, aiconf.Cfg[aiconf.Octopus].Password, aiconf.Octopus, platform.Id(123))
oct, _ := octopus.New(Jcs, aiconf.Cfg[aiconf.Octopus].URL, aiconf.Cfg[aiconf.Octopus].Username, aiconf.Cfg[aiconf.Octopus].Password, aiconf.Octopus, platform.Id(123))
ds := NewDataset(oct.Ds)

4
ai/service/image_test.go
View File

@ -16,7 +16,7 @@ import (
func TestImage(t *testing.T) {
convey.Convey(aiconf.OpenI, t, func() {
o, _ := openI.New(aiconf.Cfg[aiconf.OpenI].Username, aiconf.Cfg[aiconf.OpenI].Password, aiconf.Cfg[aiconf.OpenI].APIKey, aiconf.OpenI, platform.Id(123), aiconf.Cfg[aiconf.OpenI].DataRepo)
o, _ := openI.New(Jcs, aiconf.Cfg[aiconf.OpenI].Username, aiconf.Cfg[aiconf.OpenI].Password, aiconf.Cfg[aiconf.OpenI].APIKey, aiconf.OpenI, platform.Id(123), aiconf.Cfg[aiconf.OpenI].DataRepo)
common.InitClient()
i := NewImage(o.Img)
@ -83,7 +83,7 @@ func TestImage(t *testing.T) {
func TestOctopusImage(t *testing.T) {
convey.Convey(aiconf.Octopus, t, func() {
o, _ := octopus.New(aiconf.Cfg[aiconf.Octopus].URL, aiconf.Cfg[aiconf.Octopus].Username, aiconf.Cfg[aiconf.Octopus].Password, aiconf.Octopus, platform.Id(123))
o, _ := octopus.New(Jcs, aiconf.Cfg[aiconf.Octopus].URL, aiconf.Cfg[aiconf.Octopus].Username, aiconf.Cfg[aiconf.Octopus].Password, aiconf.Octopus, platform.Id(123))
i := NewImage(o.Img)

2
ai/service/model_test.go
View File

@ -20,7 +20,7 @@ import (
func TestCreateModel(t *testing.T) {
convey.Convey("Create Model", t, func() {
o, _ := openI.New(aiconf.Cfg[aiconf.OpenI].Username, aiconf.Cfg[aiconf.OpenI].Password, aiconf.Cfg[aiconf.OpenI].APIKey, aiconf.OpenI, platform.Id(123), aiconf.Cfg[aiconf.OpenI].DataRepo)
o, _ := openI.New(Jcs, aiconf.Cfg[aiconf.OpenI].Username, aiconf.Cfg[aiconf.OpenI].Password, aiconf.Cfg[aiconf.OpenI].APIKey, aiconf.OpenI, platform.Id(123), aiconf.Cfg[aiconf.OpenI].DataRepo)
common.InitClient()
mdl := NewModel(o.Mdl)

6
ai/service/resource_test.go
View File

@ -16,7 +16,7 @@ import (
func TestResource(t *testing.T) {
convey.Convey(aiconf.OpenI, t, func() {
o, _ := openI.New(aiconf.Cfg[aiconf.OpenI].Username, aiconf.Cfg[aiconf.OpenI].Password, aiconf.Cfg[aiconf.OpenI].APIKey, aiconf.OpenI, platform.Id(123), aiconf.Cfg[aiconf.OpenI].DataRepo)
o, _ := openI.New(Jcs, aiconf.Cfg[aiconf.OpenI].Username, aiconf.Cfg[aiconf.OpenI].Password, aiconf.Cfg[aiconf.OpenI].APIKey, aiconf.OpenI, platform.Id(123), aiconf.Cfg[aiconf.OpenI].DataRepo)
common.InitClient()
res := NewResource(o.Res)
@ -97,7 +97,7 @@ func TestResource(t *testing.T) {
func TestGetResourceSpecs(t *testing.T) {
convey.Convey(aiconf.OpenI, t, func() {
o, _ := openI.New(aiconf.Cfg[aiconf.OpenI].Username, aiconf.Cfg[aiconf.OpenI].Password, aiconf.Cfg[aiconf.OpenI].APIKey, aiconf.OpenI, platform.Id(123), aiconf.Cfg[aiconf.OpenI].DataRepo)
o, _ := openI.New(Jcs, aiconf.Cfg[aiconf.OpenI].Username, aiconf.Cfg[aiconf.OpenI].Password, aiconf.Cfg[aiconf.OpenI].APIKey, aiconf.OpenI, platform.Id(123), aiconf.Cfg[aiconf.OpenI].DataRepo)
common.InitClient()
res := NewResource(o.Res)
@ -116,7 +116,7 @@ func TestGetResourceSpecs(t *testing.T) {
})
convey.Convey(aiconf.Octopus, t, func() {
oct, _ := octopus.New(aiconf.Cfg[aiconf.Octopus].URL, aiconf.Cfg[aiconf.Octopus].Username, aiconf.Cfg[aiconf.Octopus].Password, aiconf.Octopus, platform.Id(123))
oct, _ := octopus.New(Jcs, aiconf.Cfg[aiconf.Octopus].URL, aiconf.Cfg[aiconf.Octopus].Username, aiconf.Cfg[aiconf.Octopus].Password, aiconf.Octopus, platform.Id(123))
res := NewResource(oct.Res)

6
ai/service/service_test.go
View File

@ -16,7 +16,7 @@ import (
func TestAlgorithm_Train(t *testing.T) {
convey.Convey("Test Service", t, func() {
o, _ := openI.New(aiconf.Cfg[aiconf.OpenI].Username, aiconf.Cfg[aiconf.OpenI].Password, aiconf.Cfg[aiconf.OpenI].APIKey, aiconf.OpenI, platform.Id(123), aiconf.Cfg[aiconf.OpenI].DataRepo)
o, _ := openI.New(Jcs, aiconf.Cfg[aiconf.OpenI].Username, aiconf.Cfg[aiconf.OpenI].Password, aiconf.Cfg[aiconf.OpenI].APIKey, aiconf.OpenI, platform.Id(123), aiconf.Cfg[aiconf.OpenI].DataRepo)
//oct, _ := octopus.New(aiconf.Cfg[aiconf.Octopus].URL, aiconf.Cfg[aiconf.Octopus].Username, aiconf.Cfg[aiconf.Octopus].Password, platform.Id(456))
common.InitClient()
@ -71,10 +71,10 @@ func TestAlgorithm_Train(t *testing.T) {
func TestService(t *testing.T) {
convey.Convey("Test Service", t, func() {
o, _ := openI.New(aiconf.Cfg[aiconf.OpenI].Username, aiconf.Cfg[aiconf.OpenI].Password, aiconf.Cfg[aiconf.OpenI].APIKey, aiconf.OpenI, platform.Id(123), aiconf.Cfg[aiconf.OpenI].DataRepo)
o, _ := openI.New(Jcs, aiconf.Cfg[aiconf.OpenI].Username, aiconf.Cfg[aiconf.OpenI].Password, aiconf.Cfg[aiconf.OpenI].APIKey, aiconf.OpenI, platform.Id(123), aiconf.Cfg[aiconf.OpenI].DataRepo)
common.InitClient()
o1, err := octopus.New(aiconf.Cfg[aiconf.Octopus].URL, aiconf.Cfg[aiconf.Octopus].Username, aiconf.Cfg[aiconf.Octopus].Password, aiconf.Octopus, platform.Id(456))
o1, err := octopus.New(nil, aiconf.Cfg[aiconf.Octopus].URL, aiconf.Cfg[aiconf.Octopus].Username, aiconf.Cfg[aiconf.Octopus].Password, aiconf.Octopus, platform.Id(456))
if err != nil {
fmt.Println(err)
}

6
ai/service/task_test.go
View File

@ -18,7 +18,7 @@ import (
func TestOpenITask(t *testing.T) {
convey.Convey(aiconf.OpenI, t, func() {
o, _ := openI.New(aiconf.Cfg[aiconf.OpenI].Username, aiconf.Cfg[aiconf.OpenI].Password, aiconf.Cfg[aiconf.OpenI].APIKey, aiconf.OpenI, platform.Id(123), aiconf.Cfg[aiconf.OpenI].DataRepo)
o, _ := openI.New(Jcs, aiconf.Cfg[aiconf.OpenI].Username, aiconf.Cfg[aiconf.OpenI].Password, aiconf.Cfg[aiconf.OpenI].APIKey, aiconf.OpenI, platform.Id(123), aiconf.Cfg[aiconf.OpenI].DataRepo)
common.InitClient()
tps := &task.TrainParams{
@ -64,7 +64,7 @@ func TestOpenITask(t *testing.T) {
func TestResultSync(t *testing.T) {
convey.Convey(aiconf.OpenI, t, func() {
o, _ := openI.New(aiconf.Cfg[aiconf.OpenI].Username, aiconf.Cfg[aiconf.OpenI].Password, aiconf.Cfg[aiconf.OpenI].APIKey, aiconf.OpenI, platform.Id(123), aiconf.Cfg[aiconf.OpenI].DataRepo)
o, _ := openI.New(Jcs, aiconf.Cfg[aiconf.OpenI].Username, aiconf.Cfg[aiconf.OpenI].Password, aiconf.Cfg[aiconf.OpenI].APIKey, aiconf.OpenI, platform.Id(123), aiconf.Cfg[aiconf.OpenI].DataRepo)
common.InitClient()
src := &aicom.Source{
@ -99,7 +99,7 @@ func TestResultSync(t *testing.T) {
func TestOctopusTask(t *testing.T) {
convey.Convey(aiconf.Octopus, t, func() {
o, _ := octopus.New(aiconf.Cfg[aiconf.Octopus].URL, aiconf.Cfg[aiconf.Octopus].Username, aiconf.Cfg[aiconf.Octopus].Password, aiconf.Octopus, platform.Id(123))
o, _ := octopus.New(Jcs, aiconf.Cfg[aiconf.Octopus].URL, aiconf.Cfg[aiconf.Octopus].Username, aiconf.Cfg[aiconf.Octopus].Password, aiconf.Octopus, platform.Id(123))
tps := &task.TrainParams{
TaskName: "testTask123",

1
client/config/config.go
View File

@ -72,4 +72,5 @@ func LoadConfig(path string) *Server {
type Server struct {
System System `mapstructure:"system" json:"system" yaml:"system"`
PcmCore PcmCore `mapstructure:"pcm-core" json:"pcm-core" yaml:"pcm-core"`
Jcs Jcs `mapstructure:"jcs" json:"jcs" yaml:"jcs"`
}

7
client/config/jcs.go Normal file
View File

@ -0,0 +1,7 @@
package config
type Jcs struct {
Name string `mapstructure:"name" json:"name" yaml:"name"`
ApiKey string `mapstructure:"api_key" json:"api_key" yaml:"api_key"`
Url string `mapstructure:"url" json:"url" yaml:"url"`
}

1
client/config/pcm_core.go
View File

@ -5,4 +5,5 @@ type PcmCore struct {
ParticipantHost string `mapstructure:"participant-host" json:"participant-host" yaml:"participant-host"` // 本P端服务主机地址
HPCClusterList string `mapstructure:"hpc-cluster-list" json:"hpc-cluster-list" yaml:"hpc-cluster-list"` // 集群列表接口
AIClusterList string `mapstructure:"ai-cluster-list" json:"ai-cluster-list" yaml:"ai-cluster-list"` // 集群列表接口
Token string `mapstructure:"token" json:"token" yaml:"token"`
}

16
client/initialize/aiCluster.go
View File

@ -28,14 +28,15 @@ func GetAllAIClusterInfos() map[string]types.ClusterInfo {
func InitAICluster(cfg *config.Server) (*service.Service, error) {
client := utils.InitClient(cfg.PcmCore.CoordinatorHost, "")
return initAISvcs(client, cfg.PcmCore)
return initAISvcs(client, cfg.PcmCore, cfg.Jcs)
}
// 初始化智算集群连接池
func initAISvcs(client *utils.RestyClient, core config.PcmCore) (*service.Service, error) {
func initAISvcs(client *utils.RestyClient, core config.PcmCore, jcs config.Jcs) (*service.Service, error) {
resp := types.ResultResp{}
token := "Bearer " + core.Token
_, err := client.Request(core.CoordinatorHost+core.AIClusterList, "GET", func(req *resty.Request) {
req.SetResult(&resp)
req.SetHeader("Authorization", token).SetResult(&resp)
})
if err != nil {
return nil, fmt.Errorf("获取集群列表失败: %w", err)
@ -131,7 +132,7 @@ func initAISvcs(client *utils.RestyClient, core config.PcmCore) (*service.Servic
switch cluster.Label {
case "openI":
oi, err := openI.New(cluster.Username, cluster.Password, cluster.Token, cluster.Name, platform.Id(cluster.Id), "data")
oi, err := openI.New(&jcs, cluster.Username, cluster.Password, cluster.Token, cluster.Name, platform.Id(cluster.Id), "data")
if err != nil {
Error("初始化失败", zap.Error(err))
continue
@ -140,7 +141,7 @@ func initAISvcs(client *utils.RestyClient, core config.PcmCore) (*service.Servic
common.InitClient()
//更新C端集群状态
case "octopus":
oct, err := octopus.New(cluster.Address, cluster.Username, cluster.Password, cluster.Name, platform.Id(cluster.Id))
oct, err := octopus.New(&jcs, cluster.Address, cluster.Username, cluster.Password, cluster.Name, platform.Id(cluster.Id))
if err != nil {
Error("初始化失败", zap.Error(err))
continue
@ -152,6 +153,11 @@ func initAISvcs(client *utils.RestyClient, core config.PcmCore) (*service.Servic
if len(platforms) == 0 {
return nil, fmt.Errorf("注册集群列表为空")
}
for _, p := range platforms {
Info("注册集群列表:", zap.Any("id", p.Id()), zap.Any("type", p.Type()), zap.Any("name", p.Name()))
}
svc, err = service.NewService(platforms...)
if err != nil {
return nil, err

10
participant/octopus/octopus.go
View File

@ -7,7 +7,6 @@ import (
"errors"
"fmt"
json "github.com/json-iterator/go"
"gitlink.org.cn/JointCloud/jcs/config"
jcs "gitlink.org.cn/JointCloud/jcs/storagekit"
"gitlink.org.cn/JointCloud/pcm-participant-ai/algorithm"
"gitlink.org.cn/JointCloud/pcm-participant-ai/dataset"
@ -16,6 +15,7 @@ import (
"gitlink.org.cn/JointCloud/pcm-participant-ai/platform"
"gitlink.org.cn/JointCloud/pcm-participant-ai/resource"
"gitlink.org.cn/JointCloud/pcm-participant-ai/task"
"gitlink.org.cn/JointCloud/pcm-participant-client/config"
"gitlink.org.cn/JointCloud/pcm-participant-octopus/common"
"gitlink.org.cn/JointCloud/pcm-participant-octopus/model"
"gitlink.org.cn/JointCloud/pcm-participant-octopus/service"
@ -149,16 +149,12 @@ type ResourceSpec struct {
cardType string
}
func New(ip, user, pwd, clusterName string, id platform.Id) (*Octopus, error) {
func New(conf *config.Jcs, ip, user, pwd, clusterName string, id platform.Id) (*Octopus, error) {
token, err := common.NewToken(ip, user, pwd)
if err != nil {
return nil, err
}
cfg, err := config.LoadConfig("../jcs/config/config.yaml")
if err != nil {
return nil, err
}
jcsClient := jcs.NewJcs(cfg.Platforms.APIKey, cfg.Platforms.URL, -1)
jcsClient := jcs.NewJcs(conf.ApiKey, conf.Url, -1)
opt := &Option{
ip: ip,
user: user,

10
participant/openI/openI.go
View File

@ -11,7 +11,6 @@ import (
"github.com/go-git/go-git/v5/plumbing/object"
git_http "github.com/go-git/go-git/v5/plumbing/transport/http"
json "github.com/json-iterator/go"
"gitlink.org.cn/JointCloud/jcs/config"
jcs "gitlink.org.cn/JointCloud/jcs/storagekit"
"gitlink.org.cn/JointCloud/jcs/storagekit/types"
"gitlink.org.cn/JointCloud/jcs/util"
@ -24,6 +23,7 @@ import (
"gitlink.org.cn/JointCloud/pcm-participant-ai/platform"
"gitlink.org.cn/JointCloud/pcm-participant-ai/resource"
"gitlink.org.cn/JointCloud/pcm-participant-ai/task"
"gitlink.org.cn/JointCloud/pcm-participant-client/config"
"gitlink.org.cn/JointCloud/pcm-participant-openi/apis"
"gitlink.org.cn/JointCloud/pcm-participant-openi/model"
"gitlink.org.cn/JointCloud/pcm-participant-openi/service"
@ -204,12 +204,8 @@ type BaseResp struct {
Data interface{} `json:"data"`
}
func New(username, pwd, token, clusterName string, id platform.Id, datasetRepo string) (*OpenI, error) {
cfg, err := config.LoadConfig("../jcs/config/config.yaml")
if err != nil {
return nil, err
}
jcsClient := jcs.NewJcs(cfg.Platforms.APIKey, cfg.Platforms.URL, -1)
func New(conf *config.Jcs, username, pwd, token, clusterName string, id platform.Id, datasetRepo string) (*OpenI, error) {
jcsClient := jcs.NewJcs(conf.ApiKey, conf.Url, -1)
opt := &Option{
username: username,