269 lines
8.0 KiB
Go
269 lines
8.0 KiB
Go
package services
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"path"
|
|
"strings"
|
|
|
|
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
|
|
"gitlink.org.cn/cloudream/common/pkgs/logger"
|
|
"gitlink.org.cn/cloudream/common/pkgs/trie"
|
|
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
|
|
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
|
|
|
|
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
|
|
"gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader/strategy"
|
|
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
|
|
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/distlock/reqbuilder"
|
|
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
|
|
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser"
|
|
hubmq "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/mq/hub"
|
|
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
|
|
)
|
|
|
|
type UserSpaceService struct {
|
|
*Service
|
|
}
|
|
|
|
func (svc *Service) UserSpaceSvc() *UserSpaceService {
|
|
return &UserSpaceService{Service: svc}
|
|
}
|
|
|
|
func (svc *UserSpaceService) Get(userspaceID clitypes.UserSpaceID) (clitypes.UserSpace, error) {
|
|
return svc.DB.UserSpace().GetByID(svc.DB.DefCtx(), userspaceID)
|
|
}
|
|
|
|
func (svc *UserSpaceService) GetByName(name string) (clitypes.UserSpace, error) {
|
|
return svc.DB.UserSpace().GetByName(svc.DB.DefCtx(), name)
|
|
}
|
|
|
|
func (svc *UserSpaceService) LoadPackage(packageID clitypes.PackageID, userspaceID clitypes.UserSpaceID, rootPath string) error {
|
|
coorCli, err := stgglb.CoordinatorMQPool.Acquire()
|
|
if err != nil {
|
|
return fmt.Errorf("new coordinator client: %w", err)
|
|
}
|
|
defer stgglb.CoordinatorMQPool.Release(coorCli)
|
|
|
|
destStg := svc.UserSpaceMeta.Get(userspaceID)
|
|
if destStg == nil {
|
|
return fmt.Errorf("userspace not found: %d", userspaceID)
|
|
}
|
|
if destStg.MasterHub == nil {
|
|
return fmt.Errorf("userspace %v has no master hub", userspaceID)
|
|
}
|
|
|
|
details, err := db.DoTx11(svc.DB, svc.DB.Object().GetPackageObjectDetails, packageID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
var pinned []clitypes.ObjectID
|
|
plans := exec.NewPlanBuilder()
|
|
for _, obj := range details {
|
|
strg, err := svc.StrategySelector.Select(strategy.Request{
|
|
Detail: obj,
|
|
DestHub: destStg.MasterHub.HubID,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("select download strategy: %w", err)
|
|
}
|
|
|
|
ft := ioswitch2.NewFromTo()
|
|
switch strg := strg.(type) {
|
|
case *strategy.DirectStrategy:
|
|
ft.AddFrom(ioswitch2.NewFromShardstore(strg.Detail.Object.FileHash, *strg.UserSpace.MasterHub, strg.UserSpace, ioswitch2.RawStream()))
|
|
|
|
case *strategy.ECReconstructStrategy:
|
|
for i, b := range strg.Blocks {
|
|
ft.AddFrom(ioswitch2.NewFromShardstore(b.FileHash, *strg.UserSpaces[i].MasterHub, strg.UserSpaces[i], ioswitch2.ECStream(b.Index)))
|
|
ft.ECParam = &strg.Redundancy
|
|
}
|
|
default:
|
|
return fmt.Errorf("unsupported download strategy: %T", strg)
|
|
}
|
|
|
|
ft.AddTo(ioswitch2.NewToPublicStore(*destStg.MasterHub, *destStg, path.Join(rootPath, obj.Object.Path)))
|
|
// 顺便保存到同存储服务的分片存储中
|
|
if destStg.UserSpace.ShardStore != nil {
|
|
ft.AddTo(ioswitch2.NewToShardStore(*destStg.MasterHub, *destStg, ioswitch2.RawStream(), ""))
|
|
pinned = append(pinned, obj.Object.ObjectID)
|
|
}
|
|
|
|
err = parser.Parse(ft, plans)
|
|
if err != nil {
|
|
return fmt.Errorf("parse plan: %w", err)
|
|
}
|
|
}
|
|
|
|
mutex, err := reqbuilder.NewBuilder().
|
|
Shard().Buzy(userspaceID).
|
|
MutexLock(svc.PubLock)
|
|
if err != nil {
|
|
return fmt.Errorf("acquire locks failed, err: %w", err)
|
|
}
|
|
defer mutex.Unlock()
|
|
|
|
// 记录访问统计
|
|
for _, obj := range details {
|
|
svc.AccessStat.AddAccessCounter(obj.Object.ObjectID, packageID, userspaceID, 1)
|
|
}
|
|
|
|
drv := plans.Execute(exec.NewExecContext())
|
|
_, err = drv.Wait(context.Background())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (svc *UserSpaceService) SpaceToSpace(srcSpaceID clitypes.UserSpaceID, srcPath string, dstSpaceID clitypes.UserSpaceID, dstPath string) (clitypes.SpaceToSpaceResult, error) {
|
|
srcSpace := svc.UserSpaceMeta.Get(srcSpaceID)
|
|
if srcSpace == nil {
|
|
return clitypes.SpaceToSpaceResult{}, fmt.Errorf("source userspace not found: %d", srcSpaceID)
|
|
}
|
|
if srcSpace.MasterHub == nil {
|
|
return clitypes.SpaceToSpaceResult{}, fmt.Errorf("source userspace %v has no master hub", srcSpaceID)
|
|
}
|
|
srcSpaceCli, err := stgglb.HubMQPool.Acquire(srcSpace.MasterHub.HubID)
|
|
if err != nil {
|
|
return clitypes.SpaceToSpaceResult{}, fmt.Errorf("new source userspace client: %w", err)
|
|
}
|
|
defer stgglb.HubMQPool.Release(srcSpaceCli)
|
|
|
|
dstSpace := svc.UserSpaceMeta.Get(dstSpaceID)
|
|
if dstSpace == nil {
|
|
return clitypes.SpaceToSpaceResult{}, fmt.Errorf("destination userspace not found: %d", dstSpaceID)
|
|
}
|
|
if dstSpace.MasterHub == nil {
|
|
return clitypes.SpaceToSpaceResult{}, fmt.Errorf("destination userspace %v has no master hub", dstSpaceID)
|
|
}
|
|
dstSpaceCli, err := stgglb.HubMQPool.Acquire(dstSpace.MasterHub.HubID)
|
|
if err != nil {
|
|
return clitypes.SpaceToSpaceResult{}, fmt.Errorf("new destination userspace client: %w", err)
|
|
}
|
|
defer stgglb.HubMQPool.Release(dstSpaceCli)
|
|
|
|
srcPath = strings.Trim(srcPath, cdssdk.ObjectPathSeparator)
|
|
dstPath = strings.Trim(dstPath, cdssdk.ObjectPathSeparator)
|
|
|
|
if srcPath == "" {
|
|
return clitypes.SpaceToSpaceResult{}, fmt.Errorf("source path is empty")
|
|
}
|
|
|
|
if dstPath == "" {
|
|
return clitypes.SpaceToSpaceResult{}, fmt.Errorf("destination path is empty")
|
|
}
|
|
|
|
listAllResp, err := srcSpaceCli.PublicStoreListAll(&hubmq.PublicStoreListAll{
|
|
UserSpace: *srcSpace,
|
|
Path: srcPath,
|
|
})
|
|
if err != nil {
|
|
return clitypes.SpaceToSpaceResult{}, fmt.Errorf("list all from source userspace: %w", err)
|
|
}
|
|
|
|
srcPathComps := clitypes.SplitObjectPath(srcPath)
|
|
srcDirCompLen := len(srcPathComps) - 1
|
|
|
|
entryTree := trie.NewTrie[*types.PublicStoreEntry]()
|
|
for _, e := range listAllResp.Entries {
|
|
pa, ok := strings.CutSuffix(e.Path, clitypes.ObjectPathSeparator)
|
|
comps := clitypes.SplitObjectPath(pa)
|
|
e.Path = pa
|
|
|
|
e2 := e
|
|
entryTree.CreateWords(comps[srcDirCompLen:]).Value = &e2
|
|
e2.IsDir = e2.IsDir || ok
|
|
}
|
|
|
|
entryTree.Iterate(func(path []string, node *trie.Node[*types.PublicStoreEntry], isWordNode bool) trie.VisitCtrl {
|
|
if node.Value == nil {
|
|
return trie.VisitContinue
|
|
}
|
|
|
|
if node.Value.IsDir && len(node.WordNexts) > 0 {
|
|
node.Value = nil
|
|
return trie.VisitContinue
|
|
}
|
|
|
|
if !node.Value.IsDir && len(node.WordNexts) == 0 {
|
|
node.WordNexts = nil
|
|
}
|
|
|
|
return trie.VisitContinue
|
|
})
|
|
|
|
var filePathes []string
|
|
var dirPathes []string
|
|
entryTree.Iterate(func(path []string, node *trie.Node[*types.PublicStoreEntry], isWordNode bool) trie.VisitCtrl {
|
|
if node.Value == nil {
|
|
return trie.VisitContinue
|
|
}
|
|
|
|
if node.Value.IsDir {
|
|
dirPathes = append(dirPathes, node.Value.Path)
|
|
} else {
|
|
filePathes = append(filePathes, node.Value.Path)
|
|
}
|
|
|
|
return trie.VisitContinue
|
|
})
|
|
|
|
var success []string
|
|
var failed []string
|
|
|
|
for _, f := range filePathes {
|
|
newPath := strings.Replace(f, srcPath, dstPath, 1)
|
|
|
|
ft := ioswitch2.NewFromTo()
|
|
ft.AddFrom(ioswitch2.NewFromPublicStore(*srcSpace.MasterHub, *srcSpace, f))
|
|
ft.AddTo(ioswitch2.NewToPublicStore(*dstSpace.MasterHub, *dstSpace, newPath))
|
|
|
|
plans := exec.NewPlanBuilder()
|
|
err = parser.Parse(ft, plans)
|
|
if err != nil {
|
|
failed = append(failed, f)
|
|
logger.Warnf("s2s: parse plan of file %v: %v", f, err)
|
|
continue
|
|
}
|
|
|
|
_, err = plans.Execute(exec.NewExecContext()).Wait(context.Background())
|
|
if err != nil {
|
|
failed = append(failed, f)
|
|
logger.Warnf("s2s: execute plan of file %v: %v", f, err)
|
|
continue
|
|
}
|
|
|
|
success = append(success, f)
|
|
}
|
|
|
|
newDirPathes := make([]string, 0, len(dirPathes))
|
|
for i := range dirPathes {
|
|
newDirPathes = append(newDirPathes, strings.Replace(dirPathes[i], srcPath, dstPath, 1))
|
|
}
|
|
|
|
mkdirResp, err := dstSpaceCli.PublicStoreMkdirs(&hubmq.PublicStoreMkdirs{
|
|
UserSpace: *dstSpace,
|
|
Pathes: newDirPathes,
|
|
})
|
|
if err != nil {
|
|
failed = append(failed, dirPathes...)
|
|
logger.Warnf("s2s: mkdirs to destination userspace: %v", err)
|
|
} else {
|
|
for i := range dirPathes {
|
|
if mkdirResp.Successes[i] {
|
|
success = append(success, dirPathes[i])
|
|
} else {
|
|
failed = append(failed, dirPathes[i])
|
|
}
|
|
}
|
|
}
|
|
|
|
return clitypes.SpaceToSpaceResult{
|
|
Success: success,
|
|
Failed: failed,
|
|
}, nil
|
|
}
|