forked from JointCloud/pcm-coordinator
122 lines
2.9 KiB
Go
122 lines
2.9 KiB
Go
package updater
|
|
|
|
import (
|
|
"gitlink.org.cn/JointCloud/pcm-coordinator/internal/storeLink"
|
|
"gitlink.org.cn/JointCloud/pcm-coordinator/internal/svc"
|
|
"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/constants"
|
|
"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/models"
|
|
"net/http"
|
|
"strconv"
|
|
"time"
|
|
)
|
|
|
|
func UpdateDeployInstanceStatusBatch(svc *svc.ServiceContext, insList []*models.AiInferDeployInstance) {
|
|
list := make([]*models.AiInferDeployInstance, len(insList))
|
|
copy(list, insList)
|
|
for i := len(list) - 1; i >= 0; i-- {
|
|
if list[i].Status == constants.Running || list[i].Status == constants.Stopped {
|
|
list = append(list[:i], list[i+1:]...)
|
|
}
|
|
}
|
|
|
|
if len(list) == 0 {
|
|
return
|
|
}
|
|
|
|
for _, instance := range list {
|
|
go UpdateDeployInstanceStatus(svc, instance, false)
|
|
}
|
|
}
|
|
|
|
func UpdateDeployTaskStatus(svc *svc.ServiceContext) {
|
|
list, err := svc.Scheduler.AiStorages.GetAllDeployTasks()
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
ins := list[0]
|
|
for i := range list {
|
|
uTime, _ := time.Parse(time.RFC3339, ins.UpdateTime)
|
|
latest, _ := time.Parse(time.RFC3339, list[i].UpdateTime)
|
|
if latest.After(uTime) {
|
|
ins = list[i]
|
|
}
|
|
}
|
|
inslist, err := svc.Scheduler.AiStorages.GetInstanceListByDeployTaskId(ins.Id)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
for _, instance := range inslist {
|
|
go UpdateDeployInstanceStatus(svc, instance, false)
|
|
}
|
|
}
|
|
|
|
func UpdateDeployInstanceStatus(svc *svc.ServiceContext, instance *models.AiInferDeployInstance, updatetime bool) {
|
|
amap, found := svc.Scheduler.AiService.InferenceAdapterMap[strconv.FormatInt(instance.AdapterId, 10)]
|
|
if !found {
|
|
return
|
|
}
|
|
cmap, found := amap[strconv.FormatInt(instance.ClusterId, 10)]
|
|
if !found {
|
|
return
|
|
}
|
|
h := http.Request{}
|
|
ins, err := cmap.GetInferDeployInstance(h.Context(), instance.InstanceId)
|
|
if err != nil {
|
|
return
|
|
}
|
|
switch instance.ClusterType {
|
|
case storeLink.TYPE_OCTOPUS:
|
|
switch ins.Status {
|
|
case "running":
|
|
if instance.Status == constants.Running {
|
|
return
|
|
}
|
|
instance.Status = constants.Running
|
|
case "stopped":
|
|
if instance.Status == constants.Stopped {
|
|
return
|
|
}
|
|
instance.Status = constants.Stopped
|
|
default:
|
|
instance.Status = ins.Status
|
|
}
|
|
case storeLink.TYPE_MODELARTS:
|
|
switch ins.Status {
|
|
case "running":
|
|
if instance.Status == constants.Running {
|
|
return
|
|
}
|
|
instance.Status = constants.Running
|
|
case "stopped":
|
|
if instance.Status == constants.Stopped {
|
|
return
|
|
}
|
|
instance.Status = constants.Stopped
|
|
default:
|
|
instance.Status = ins.Status
|
|
}
|
|
case storeLink.TYPE_SHUGUANGAI:
|
|
switch ins.Status {
|
|
case "Running":
|
|
if instance.Status == constants.Running {
|
|
return
|
|
}
|
|
instance.Status = constants.Running
|
|
case "Terminated":
|
|
if instance.Status == constants.Stopped {
|
|
return
|
|
}
|
|
instance.Status = constants.Stopped
|
|
default:
|
|
instance.Status = ins.Status
|
|
}
|
|
}
|
|
|
|
err = svc.Scheduler.AiStorages.UpdateInferDeployInstance(instance, updatetime)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|