forked from JointCloud/pcm-coordinator
82 lines
2.4 KiB
Go
82 lines
2.4 KiB
Go
/*
|
|
|
|
Copyright (c) [2023] [pcm]
|
|
[pcm-coordinator] is licensed under Mulan PSL v2.
|
|
You can use this software according to the terms and conditions of the Mulan PSL v2.
|
|
You may obtain a copy of Mulan PSL v2 at:
|
|
http://license.coscl.org.cn/MulanPSL2
|
|
THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
|
|
EITHER EXPaRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
|
|
MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
|
|
See the Mulan PSL v2 for more details.
|
|
|
|
*/
|
|
|
|
package cron
|
|
|
|
import (
|
|
"github.com/zeromicro/go-zero/core/logx"
|
|
"gitlink.org.cn/JointCloud/pcm-coordinator/internal/logic/schedule"
|
|
"gitlink.org.cn/JointCloud/pcm-coordinator/internal/scheduler/service/utils/status"
|
|
"gitlink.org.cn/JointCloud/pcm-coordinator/internal/svc"
|
|
)
|
|
|
|
func AddCronGroup(svc *svc.ServiceContext) {
|
|
|
|
svc.Cron.AddFunc("*/5 * * * * ?", func() {
|
|
UpdateAiAdapterMaps(svc)
|
|
})
|
|
|
|
//svc.Cron.AddFunc("30 * * * * ?", func() {
|
|
// adapterList, err := svc.Scheduler.AiStorages.GetAdaptersByType("1")
|
|
// if err != nil {
|
|
// logx.Errorf(err.Error())
|
|
// return
|
|
// }
|
|
// stat.UpdateClusterResources(svc, adapterList)
|
|
//})
|
|
|
|
svc.Cron.AddFunc("@hourly", func() {
|
|
status.UpdateAutoStoppedInstance(svc)
|
|
})
|
|
|
|
svc.Cron.AddFunc("0 5/5 * * * *", func() {
|
|
queryResource := schedule.NewQueryResourcesLogic(svc.HttpClient.R().Context(), svc)
|
|
trainResrc, err := queryResource.QueryResourcesByClusterId(nil, "Train")
|
|
if err != nil {
|
|
logx.Error(err)
|
|
}
|
|
svc.Scheduler.AiService.LocalCache[schedule.QUERY_TRAIN_RESOURCES] = trainResrc
|
|
inferResrc, err := queryResource.QueryResourcesByClusterId(nil, "Inference")
|
|
if err != nil {
|
|
logx.Error(err)
|
|
}
|
|
svc.Scheduler.AiService.LocalCache[schedule.QUERY_INFERENCE_RESOURCES] = inferResrc
|
|
})
|
|
|
|
//更新hpc任务状态
|
|
svc.Cron.AddFunc("*/5 * * * * ?", func() {
|
|
status.UpdateHpcTaskStatus(svc)
|
|
})
|
|
|
|
//更新推理任务状态
|
|
svc.Cron.AddFunc("*/5 * * * * ?", func() {
|
|
tasks, err := svc.Scheduler.AiStorages.GetInferDeployInstanceListLastMonth()
|
|
if err != nil {
|
|
logx.Error(err)
|
|
}
|
|
svc.Scheduler.AiService.Si.UpdateDeployInstanceStatusBatch(tasks, true)
|
|
})
|
|
|
|
//更新训练任务状态
|
|
svc.Cron.AddFunc("*/10 * * * * ?", func() {
|
|
tasks, err := svc.Scheduler.AiStorages.AllTaskLastMonth()
|
|
if err != nil {
|
|
logx.Error(err)
|
|
}
|
|
go svc.Scheduler.AiService.St.UpdateTaskStatus(tasks)
|
|
go svc.Scheduler.AiService.St.UpdateAiTaskStatus(tasks)
|
|
})
|
|
|
|
}
|