JointCloud
/
JCS-pub
24
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

coor的types移动到common

This commit is contained in:
Sydonian 2025-07-16 09:55:31 +08:00
parent e9200bc78c
commit dd23f68926
152 changed files with 1168 additions and 1192 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
client/internal/accessstat/access_stat.go
View File

@ -7,7 +7,7 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/async"
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type AccessStatEventChan = async.UnboundChannel[AccessStatEvent]
@ -30,9 +30,9 @@ type AccessStat struct {
}
type entryKey struct {
objID jcsypes.ObjectID
pkgID jcsypes.PackageID
spaceID jcsypes.UserSpaceID
objID jcstypes.ObjectID
pkgID jcstypes.PackageID
spaceID jcstypes.UserSpaceID
}
func NewAccessStat(cfg Config, db *db.DB) *AccessStat {
@ -44,7 +44,7 @@ func NewAccessStat(cfg Config, db *db.DB) *AccessStat {
}
}
func (p *AccessStat) AddAccessCounter(objID jcsypes.ObjectID, pkgID jcsypes.PackageID, spaceID jcsypes.UserSpaceID, value float64) {
func (p *AccessStat) AddAccessCounter(objID jcstypes.ObjectID, pkgID jcstypes.PackageID, spaceID jcstypes.UserSpaceID, value float64) {
p.lock.Lock()
defer p.lock.Unlock()

6
client/internal/accesstoken/accesstoken.go
View File

@ -15,7 +15,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/accesstoken"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type KeeperEvent interface {
@ -30,7 +30,7 @@ type ExitEvent struct {
type Keeper struct {
cfg Config
enabled bool
token cortypes.UserAccessToken
token jcstypes.UserAccessToken
priKey ed25519.PrivateKey
lock sync.RWMutex
done chan any
@ -144,7 +144,7 @@ func (k *Keeper) Stop() {
}
}
func (k *Keeper) GetToken() cortypes.UserAccessToken {
func (k *Keeper) GetToken() jcstypes.UserAccessToken {
k.lock.RLock()
defer k.lock.RUnlock()

20
client/internal/cmdline/migrate.go
View File

@ -6,7 +6,7 @@ import (
"github.com/spf13/cobra"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/config"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/driver/mysql"
"gorm.io/gorm"
)
@ -38,15 +38,15 @@ func migrate(configPath string) {
}
db = db.Set("gorm:table_options", "CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci")
migrateOne(db, jcsypes.Bucket{})
migrateOne(db, jcsypes.ObjectAccessStat{})
migrateOne(db, jcsypes.ObjectBlock{})
migrateOne(db, jcsypes.Object{})
migrateOne(db, jcsypes.PackageAccessStat{})
migrateOne(db, jcsypes.Package{})
migrateOne(db, jcsypes.PinnedObject{})
migrateOne(db, jcsypes.UserSpace{})
migrateOne(db, jcsypes.SpaceSyncTask{})
migrateOne(db, jcstypes.Bucket{})
migrateOne(db, jcstypes.ObjectAccessStat{})
migrateOne(db, jcstypes.ObjectBlock{})
migrateOne(db, jcstypes.Object{})
migrateOne(db, jcstypes.PackageAccessStat{})
migrateOne(db, jcstypes.Package{})
migrateOne(db, jcstypes.PinnedObject{})
migrateOne(db, jcstypes.UserSpace{})
migrateOne(db, jcstypes.SpaceSyncTask{})
fmt.Println("migrate success")
}

4
client/internal/cmdline/test.go
View File

@ -27,7 +27,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/sysevent"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
)
@ -61,7 +61,7 @@ func doTest(svc *services.Service) {
ft = ioswitch2.NewFromTo()
ft.AddFrom(ioswitch2.NewFromShardstore("Full1AE5436AF72D8EF93923486E0E167315CEF0C91898064DADFAC22216FFBC5E3D", *space1, ioswitch2.RawStream()))
ft.AddTo(ioswitch2.NewToBaseStore(*space2, jcsypes.PathFromComps("test3.txt")))
ft.AddTo(ioswitch2.NewToBaseStore(*space2, jcstypes.PathFromComps("test3.txt")))
plans := exec.NewPlanBuilder()
parser.Parse(ft, plans)
fmt.Println(plans)

46
client/internal/downloader/downloader.go
View File

@ -11,7 +11,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/speedstats"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/connectivity"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
const (
@ -21,18 +21,18 @@ const (
type DownloadIterator = iterator.Iterator[*Downloading]
type DownloadReqeust struct {
ObjectID jcsypes.ObjectID
ObjectID jcstypes.ObjectID
Offset int64
Length int64
}
type downloadReqeust2 struct {
Detail *jcsypes.ObjectDetail
Detail *jcstypes.ObjectDetail
Raw DownloadReqeust
}
type Downloading struct {
Object *jcsypes.Object
Object *jcstypes.Object
File io.ReadCloser // 文件流，如果文件不存在，那么为nil
Request DownloadReqeust
}
@ -65,7 +65,7 @@ func NewDownloader(cfg Config, conn *connectivity.Collector, stgPool *pool.Pool,
}
func (d *Downloader) DownloadObjects(reqs []DownloadReqeust) DownloadIterator {
objIDs := make([]jcsypes.ObjectID, len(reqs))
objIDs := make([]jcstypes.ObjectID, len(reqs))
for i, req := range reqs {
objIDs[i] = req.ObjectID
}
@ -79,7 +79,7 @@ func (d *Downloader) DownloadObjects(reqs []DownloadReqeust) DownloadIterator {
return iterator.FuseError[*Downloading](fmt.Errorf("request to db: %w", err))
}
detailsMap := make(map[jcsypes.ObjectID]*jcsypes.ObjectDetail)
detailsMap := make(map[jcstypes.ObjectID]*jcstypes.ObjectDetail)
for _, detail := range objDetails {
d := detail
detailsMap[detail.Object.ObjectID] = &d
@ -96,7 +96,7 @@ func (d *Downloader) DownloadObjects(reqs []DownloadReqeust) DownloadIterator {
return NewDownloadObjectIterator(d, req2s)
}
func (d *Downloader) DownloadObjectByDetail(detail jcsypes.ObjectDetail, off int64, length int64) (*Downloading, error) {
func (d *Downloader) DownloadObjectByDetail(detail jcstypes.ObjectDetail, off int64, length int64) (*Downloading, error) {
req2s := []downloadReqeust2{{
Detail: &detail,
Raw: DownloadReqeust{
@ -110,56 +110,56 @@ func (d *Downloader) DownloadObjectByDetail(detail jcsypes.ObjectDetail, off int
return iter.MoveNext()
}
func (d *Downloader) DownloadPackage(pkgID jcsypes.PackageID, prefix string) (jcsypes.Package, DownloadIterator, error) {
pkg, details, err := db.DoTx02(d.db, func(tx db.SQLContext) (jcsypes.Package, []jcsypes.ObjectDetail, error) {
func (d *Downloader) DownloadPackage(pkgID jcstypes.PackageID, prefix string) (jcstypes.Package, DownloadIterator, error) {
pkg, details, err := db.DoTx02(d.db, func(tx db.SQLContext) (jcstypes.Package, []jcstypes.ObjectDetail, error) {
pkg, err := d.db.Package().GetByID(tx, pkgID)
if err != nil {
return jcsypes.Package{}, nil, err
return jcstypes.Package{}, nil, err
}
var details []jcsypes.ObjectDetail
var details []jcstypes.ObjectDetail
if prefix != "" {
objs, err := d.db.Object().GetWithPathPrefix(tx, pkgID, prefix)
if err != nil {
return jcsypes.Package{}, nil, err
return jcstypes.Package{}, nil, err
}
objIDs := make([]jcsypes.ObjectID, len(objs))
objIDs := make([]jcstypes.ObjectID, len(objs))
for i, obj := range objs {
objIDs[i] = obj.ObjectID
}
allBlocks, err := d.db.ObjectBlock().BatchGetByObjectID(tx, objIDs)
if err != nil {
return jcsypes.Package{}, nil, err
return jcstypes.Package{}, nil, err
}
allPinnedObjs, err := d.db.PinnedObject().BatchGetByObjectID(tx, objIDs)
if err != nil {
return jcsypes.Package{}, nil, err
return jcstypes.Package{}, nil, err
}
details = make([]jcsypes.ObjectDetail, 0, len(objs))
details = make([]jcstypes.ObjectDetail, 0, len(objs))
for _, obj := range objs {
detail := jcsypes.ObjectDetail{
detail := jcstypes.ObjectDetail{
Object: obj,
}
details = append(details, detail)
}
jcsypes.DetailsFillObjectBlocks(details, allBlocks)
jcsypes.DetailsFillPinnedAt(details, allPinnedObjs)
jcstypes.DetailsFillObjectBlocks(details, allBlocks)
jcstypes.DetailsFillPinnedAt(details, allPinnedObjs)
} else {
details, err = d.db.Object().GetPackageObjectDetails(tx, pkgID)
if err != nil {
return jcsypes.Package{}, nil, err
return jcstypes.Package{}, nil, err
}
}
return pkg, details, nil
})
if err != nil {
return jcsypes.Package{}, nil, err
return jcstypes.Package{}, nil, err
}
req2s := make([]downloadReqeust2, len(details))
@ -180,11 +180,11 @@ func (d *Downloader) DownloadPackage(pkgID jcsypes.PackageID, prefix string) (jc
type ObjectECStrip struct {
Data []byte
ObjectFileHash jcsypes.FileHash // 添加这条缓存时，Object的FileHash
ObjectFileHash jcstypes.FileHash // 添加这条缓存时，Object的FileHash
}
type ECStripKey struct {
ObjectID jcsypes.ObjectID
ObjectID jcstypes.ObjectID
StripIndex int64
}

8
client/internal/downloader/lrc_strip_iterator.go
View File

@ -12,14 +12,14 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc/parser"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type LRCStripIterator struct {
downloader *Downloader
object jcsypes.Object
object jcstypes.Object
blocks []downloadBlock
red jcsypes.LRCRedundancy
red jcstypes.LRCRedundancy
curStripIndex int64
cache *StripCache
dataChan chan dataChanEntry
@ -28,7 +28,7 @@ type LRCStripIterator struct {
inited bool
}
func NewLRCStripIterator(downloder *Downloader, object jcsypes.Object, blocks []downloadBlock, red jcsypes.LRCRedundancy, beginStripIndex int64, cache *StripCache, maxPrefetch int) *LRCStripIterator {
func NewLRCStripIterator(downloder *Downloader, object jcstypes.Object, blocks []downloadBlock, red jcstypes.LRCRedundancy, beginStripIndex int64, cache *StripCache, maxPrefetch int) *LRCStripIterator {
if maxPrefetch <= 0 {
maxPrefetch = 1
}

6
client/internal/downloader/strategy/selector.go
View File

@ -12,13 +12,13 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/metacache"
"gitlink.org.cn/cloudream/jcs-pub/common/consts"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type Request struct {
Detail types.ObjectDetail
Range math2.Range
DestLocation cortypes.Location
DestLocation jcstypes.Location
}
type Strategy interface {
@ -113,7 +113,7 @@ type downloadBlock struct {
type request2 struct {
Detail types.ObjectDetail
Range math2.Range
DestLocation cortypes.Location
DestLocation jcstypes.Location
}
func (s *Selector) selectForNoneOrRep(req request2) (Strategy, error) {

12
client/internal/http/v1/object.go
View File

@ -19,7 +19,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
"gitlink.org.cn/cloudream/jcs-pub/common/ecode"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type ObjectService struct {
@ -113,9 +113,9 @@ func (s *ObjectService) Upload(ctx *gin.Context) {
return
}
copyToPath := make([]jcsypes.JPath, 0, len(info.CopyToPath))
copyToPath := make([]jcstypes.JPath, 0, len(info.CopyToPath))
for _, p := range info.CopyToPath {
copyToPath = append(copyToPath, jcsypes.PathFromJcsPathString(p))
copyToPath = append(copyToPath, jcstypes.PathFromJcsPathString(p))
}
up, err := s.svc.Uploader.BeginUpdate(info.PackageID, info.Affinity, info.CopyTo, copyToPath)
@ -147,7 +147,7 @@ func (s *ObjectService) Upload(ctx *gin.Context) {
}
path = filepath.ToSlash(path)
err = up.Upload(jcsypes.PathFromJcsPathString(path), file)
err = up.Upload(jcstypes.PathFromJcsPathString(path), file)
if err != nil {
log.Warnf("uploading file: %s", err.Error())
ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("uploading file %v: %v", file.FileName(), err)))
@ -163,7 +163,7 @@ func (s *ObjectService) Upload(ctx *gin.Context) {
return
}
uploadeds := make([]jcsypes.Object, len(pathes))
uploadeds := make([]jcstypes.Object, len(pathes))
for i := range pathes {
uploadeds[i] = ret.Objects[pathes[i]]
}
@ -398,7 +398,7 @@ func (s *ObjectService) DeleteByPath(ctx *gin.Context) {
return
}
err = s.svc.ObjectSvc().Delete([]jcsypes.ObjectID{resp.Objects[0].ObjectID})
err = s.svc.ObjectSvc().Delete([]jcstypes.ObjectID{resp.Objects[0].ObjectID})
if err != nil {
log.Warnf("deleting objects: %s", err.Error())
ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, "delete objects failed"))

14
client/internal/http/v1/package.go
View File

@ -21,7 +21,7 @@ import (
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
"gitlink.org.cn/cloudream/jcs-pub/common/ecode"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/iterator"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm"
)
@ -151,9 +151,9 @@ func (s *PackageService) CreateLoad(ctx *gin.Context) {
return
}
copyToPath := make([]jcsypes.JPath, 0, len(info.CopyToPath))
copyToPath := make([]jcstypes.JPath, 0, len(info.CopyToPath))
for _, p := range info.CopyToPath {
copyToPath = append(copyToPath, jcsypes.PathFromJcsPathString(p))
copyToPath = append(copyToPath, jcstypes.PathFromJcsPathString(p))
}
up, err := s.svc.Uploader.BeginCreateUpload(info.BucketID, info.Name, info.CopyTo, copyToPath)
@ -184,7 +184,7 @@ func (s *PackageService) CreateLoad(ctx *gin.Context) {
}
path = filepath.ToSlash(path)
err = up.Upload(jcsypes.PathFromJcsPathString(path), file)
err = up.Upload(jcstypes.PathFromJcsPathString(path), file)
if err != nil {
log.Warnf("uploading file: %s", err.Error())
ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("uploading file %v: %v", file.FileName(), err)))
@ -200,7 +200,7 @@ func (s *PackageService) CreateLoad(ctx *gin.Context) {
return
}
objs := make([]jcsypes.Object, len(pathes))
objs := make([]jcstypes.Object, len(pathes))
for i := range pathes {
objs[i] = ret.Objects[pathes[i]]
}
@ -233,7 +233,7 @@ func (s *PackageService) Download(ctx *gin.Context) {
}
}
func (s *PackageService) downloadZip(ctx *gin.Context, req cliapi.PackageDownload, pkg jcsypes.Package, iter downloader.DownloadIterator) {
func (s *PackageService) downloadZip(ctx *gin.Context, req cliapi.PackageDownload, pkg jcstypes.Package, iter downloader.DownloadIterator) {
log := logger.WithField("HTTP", "Package.Download")
ctx.Header("Content-Disposition", "attachment; filename="+url.PathEscape(pkg.Name)+".zip")
@ -276,7 +276,7 @@ func (s *PackageService) downloadZip(ctx *gin.Context, req cliapi.PackageDownloa
}
}
func (s *PackageService) downloadTar(ctx *gin.Context, req cliapi.PackageDownload, pkg jcsypes.Package, iter downloader.DownloadIterator) {
func (s *PackageService) downloadTar(ctx *gin.Context, req cliapi.PackageDownload, pkg jcstypes.Package, iter downloader.DownloadIterator) {
log := logger.WithField("HTTP", "Package.Download")
ctx.Header("Content-Disposition", "attachment; filename="+url.PathEscape(pkg.Name)+".tar")

8
client/internal/http/v1/presigned.go
View File

@ -15,7 +15,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
"gitlink.org.cn/cloudream/jcs-pub/common/ecode"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type PresignedService struct {
@ -156,9 +156,9 @@ func (s *PresignedService) ObjectUpload(ctx *gin.Context) {
return
}
copyToPath := make([]jcsypes.JPath, 0, len(req.CopyToPath))
copyToPath := make([]jcstypes.JPath, 0, len(req.CopyToPath))
for _, p := range req.CopyToPath {
copyToPath = append(copyToPath, jcsypes.PathFromJcsPathString(p))
copyToPath = append(copyToPath, jcstypes.PathFromJcsPathString(p))
}
up, err := s.svc.Uploader.BeginUpdate(req.PackageID, req.Affinity, req.CopyTo, copyToPath)
@ -171,7 +171,7 @@ func (s *PresignedService) ObjectUpload(ctx *gin.Context) {
path := filepath.ToSlash(req.Path)
err = up.Upload(jcsypes.PathFromJcsPathString(path), ctx.Request.Body)
err = up.Upload(jcstypes.PathFromJcsPathString(path), ctx.Request.Body)
if err != nil {
log.Warnf("uploading file: %s", err.Error())
ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("uploading file %v: %v", req.Path, err)))

14
client/internal/http/v1/space_syncer.go
View File

@ -8,7 +8,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
"gitlink.org.cn/cloudream/jcs-pub/common/ecode"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type SpaceSyncerService struct {
@ -41,21 +41,21 @@ func (s *SpaceSyncerService) CreateTask(ctx *gin.Context) {
return
}
dests := make([]jcsypes.SpaceSyncDest, 0, len(req.DestUserSpaceIDs))
dests := make([]jcstypes.SpaceSyncDest, 0, len(req.DestUserSpaceIDs))
for _, id := range req.DestUserSpaceIDs {
dests = append(dests, jcsypes.SpaceSyncDest{
DestUserSpaceID: jcsypes.UserSpaceID(id),
DestPath: jcsypes.PathFromJcsPathString(req.DestPathes[0]),
dests = append(dests, jcstypes.SpaceSyncDest{
DestUserSpaceID: jcstypes.UserSpaceID(id),
DestPath: jcstypes.PathFromJcsPathString(req.DestPathes[0]),
})
}
info, err := s.svc.SpaceSyncer.CreateTask(jcsypes.SpaceSyncTask{
info, err := s.svc.SpaceSyncer.CreateTask(jcstypes.SpaceSyncTask{
Trigger: req.Trigger,
Mode: req.Mode,
Filters: req.Filters,
Options: req.Options,
SrcUserSpaceID: req.SrcUserSpaceID,
SrcPath: jcsypes.PathFromJcsPathString(req.SrcPath),
SrcPath: jcstypes.PathFromJcsPathString(req.SrcPath),
Dests: dests,
})
if err != nil {

4
client/internal/http/v1/user_space.go
View File

@ -9,7 +9,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
"gitlink.org.cn/cloudream/jcs-pub/common/ecode"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type UserSpaceService struct {
@ -52,7 +52,7 @@ func (s *UserSpaceService) CreatePackage(ctx *gin.Context) {
return
}
pkg, err := s.svc.Uploader.UserSpaceUpload(req.UserSpaceID, jcsypes.PathFromJcsPathString(req.Path), req.BucketID, req.Name, req.SpaceAffinity)
pkg, err := s.svc.Uploader.UserSpaceUpload(req.UserSpaceID, jcstypes.PathFromJcsPathString(req.Path), req.BucketID, req.Name, req.SpaceAffinity)
if err != nil {
log.Warnf("userspace create package: %s", err.Error())
ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("userspace create package: %v", err)))

18
client/internal/metacache/connectivity.go
View File

@ -8,12 +8,12 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/logger"
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func (m *MetaCacheHost) AddConnectivity() *Connectivity {
cache := &Connectivity{
entries: make(map[cortypes.HubID]*ConnectivityEntry),
entries: make(map[jcstypes.HubID]*ConnectivityEntry),
}
m.caches = append(m.caches, cache)
@ -22,10 +22,10 @@ func (m *MetaCacheHost) AddConnectivity() *Connectivity {
type Connectivity struct {
lock sync.RWMutex
entries map[cortypes.HubID]*ConnectivityEntry
entries map[jcstypes.HubID]*ConnectivityEntry
}
func (c *Connectivity) Get(from cortypes.HubID, to cortypes.HubID) *time.Duration {
func (c *Connectivity) Get(from jcstypes.HubID, to jcstypes.HubID) *time.Duration {
for i := 0; i < 2; i++ {
c.lock.RLock()
entry, ok := c.entries[from]
@ -60,12 +60,12 @@ func (c *Connectivity) ClearOutdated() {
}
}
func (c *Connectivity) load(hubID cortypes.HubID) {
func (c *Connectivity) load(hubID jcstypes.HubID) {
coorCli := stgglb.CoordinatorRPCPool.Get()
defer coorCli.Release()
get, cerr := coorCli.GetHubConnectivities(context.Background(), corrpc.ReqGetHubConnectivities([]cortypes.HubID{hubID}))
get, cerr := coorCli.GetHubConnectivities(context.Background(), corrpc.ReqGetHubConnectivities([]jcstypes.HubID{hubID}))
if cerr != nil {
logger.Warnf("get hub connectivities: %v", cerr)
return
@ -76,7 +76,7 @@ func (c *Connectivity) load(hubID cortypes.HubID) {
ce := &ConnectivityEntry{
From: hubID,
To: make(map[cortypes.HubID]cortypes.HubConnectivity),
To: make(map[jcstypes.HubID]jcstypes.HubConnectivity),
UpdateTime: time.Now(),
}
@ -88,7 +88,7 @@ func (c *Connectivity) load(hubID cortypes.HubID) {
}
type ConnectivityEntry struct {
From cortypes.HubID
To map[cortypes.HubID]cortypes.HubConnectivity
From jcstypes.HubID
To map[jcstypes.HubID]jcstypes.HubConnectivity
UpdateTime time.Time
}

16
client/internal/metacache/hubmeta.go
View File

@ -7,12 +7,12 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/logger"
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func (m *MetaCacheHost) AddHubMeta() *HubMeta {
meta := &HubMeta{}
meta.cache = NewSimpleMetaCache(SimpleMetaCacheConfig[cortypes.HubID, cortypes.Hub]{
meta.cache = NewSimpleMetaCache(SimpleMetaCacheConfig[jcstypes.HubID, jcstypes.Hub]{
Getter: meta.load,
Expire: time.Minute * 5,
})
@ -22,10 +22,10 @@ func (m *MetaCacheHost) AddHubMeta() *HubMeta {
}
type HubMeta struct {
cache *SimpleMetaCache[cortypes.HubID, cortypes.Hub]
cache *SimpleMetaCache[jcstypes.HubID, jcstypes.Hub]
}
func (h *HubMeta) Get(hubID cortypes.HubID) *cortypes.Hub {
func (h *HubMeta) Get(hubID jcstypes.HubID) *jcstypes.Hub {
v, ok := h.cache.Get(hubID)
if ok {
return &v
@ -33,9 +33,9 @@ func (h *HubMeta) Get(hubID cortypes.HubID) *cortypes.Hub {
return nil
}
func (h *HubMeta) GetMany(hubIDs []cortypes.HubID) []*cortypes.Hub {
func (h *HubMeta) GetMany(hubIDs []jcstypes.HubID) []*jcstypes.Hub {
vs, oks := h.cache.GetMany(hubIDs)
ret := make([]*cortypes.Hub, len(vs))
ret := make([]*jcstypes.Hub, len(vs))
for i := range vs {
if oks[i] {
ret[i] = &vs[i]
@ -48,8 +48,8 @@ func (h *HubMeta) ClearOutdated() {
h.cache.ClearOutdated()
}
func (h *HubMeta) load(keys []cortypes.HubID) ([]cortypes.Hub, []bool) {
vs := make([]cortypes.Hub, len(keys))
func (h *HubMeta) load(keys []jcstypes.HubID) ([]jcstypes.Hub, []bool) {
vs := make([]jcstypes.Hub, len(keys))
oks := make([]bool, len(keys))
coorCli := stgglb.CoordinatorRPCPool.Get()

4
client/internal/metacache/user_space_meta.go
View File

@ -8,7 +8,7 @@ import (
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func (m *MetaCacheHost) AddStorageMeta() *UserSpaceMeta {
@ -78,7 +78,7 @@ func (s *UserSpaceMeta) load(keys []types.UserSpaceID) ([]types.UserSpaceDetail,
coorCli := stgglb.CoordinatorRPCPool.Get()
defer coorCli.Release()
stgs := make([]cortypes.StorageType, len(spaces))
stgs := make([]jcstypes.StorageType, len(spaces))
for i := range spaces {
stgs[i] = spaces[i].Storage
}

8
client/internal/mount/mount_linux.go
View File

@ -12,7 +12,7 @@ import (
fuse2 "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type Mount struct {
@ -107,14 +107,14 @@ func (m *Mount) StartReclaimSpace() {
m.vfs.ReclaimSpace()
}
func (m *Mount) NotifyObjectInvalid(obj jcsypes.Object) {
func (m *Mount) NotifyObjectInvalid(obj jcstypes.Object) {
}
func (m *Mount) NotifyPackageInvalid(pkg jcsypes.Package) {
func (m *Mount) NotifyPackageInvalid(pkg jcstypes.Package) {
}
func (m *Mount) NotifyBucketInvalid(bkt jcsypes.Bucket) {
func (m *Mount) NotifyBucketInvalid(bkt jcstypes.Bucket) {
}

8
client/internal/mount/mount_win.go
View File

@ -8,7 +8,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/config"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type Mount struct {
@ -43,14 +43,14 @@ func (m *Mount) StartReclaimSpace() {
}
func (m *Mount) NotifyObjectInvalid(obj jcsypes.Object) {
func (m *Mount) NotifyObjectInvalid(obj jcstypes.Object) {
}
func (m *Mount) NotifyPackageInvalid(pkg jcsypes.Package) {
func (m *Mount) NotifyPackageInvalid(pkg jcstypes.Package) {
}
func (m *Mount) NotifyBucketInvalid(bkt jcsypes.Bucket) {
func (m *Mount) NotifyBucketInvalid(bkt jcstypes.Bucket) {
}

14
client/internal/mount/vfs/cache/cache.go vendored
View File

@ -21,7 +21,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/config"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type CacheEntry interface {
@ -196,7 +196,7 @@ func (c *Cache) CreateFile(pathComps []string) *CacheFile {
// 尝试加载缓存文件，如果文件不存在，则使用obj的信息创建一个新缓存文件，而如果obj为nil，那么会返回nil。
//
// 记得使用Release减少引用计数
func (c *Cache) LoadFile(pathComps []string, obj *jcsypes.Object) *CacheFile {
func (c *Cache) LoadFile(pathComps []string, obj *jcstypes.Object) *CacheFile {
c.lock.Lock()
defer c.lock.Unlock()
@ -490,7 +490,7 @@ func (c *Cache) Move(pathComps []string, newPathComps []string) error {
type syncPackage struct {
bktName string
pkgName string
pkg jcsypes.Package
pkg jcstypes.Package
upObjs []*uploadingObject
}
@ -917,7 +917,7 @@ func (c *Cache) doUpdatingOnly(pkgs []*syncPackage) {
pathes := make([]string, 0, len(p.upObjs))
modTimes := make([]time.Time, 0, len(p.upObjs))
for _, obj := range p.upObjs {
pathes = append(pathes, jcsypes.JoinObjectPath(obj.pathComps[2:]...))
pathes = append(pathes, jcstypes.JoinObjectPath(obj.pathComps[2:]...))
modTimes = append(modTimes, obj.modTime)
}
@ -1008,7 +1008,7 @@ func (c *Cache) doUploading(pkgs []*syncPackage) {
counter := io2.Counter(&rd)
err = upder.Upload(jcsypes.PathFromComps(o.pathComps[2:]...), counter, uploader.UploadOption{
err = upder.Upload(jcstypes.PathFromComps(o.pathComps[2:]...), counter, uploader.UploadOption{
CreateTime: o.modTime,
})
if err != nil {
@ -1036,8 +1036,8 @@ func (c *Cache) doUploading(pkgs []*syncPackage) {
continue
}
oldPath := jcsypes.JoinObjectPath(o.pathComps[2:]...)
newPath := jcsypes.JoinObjectPath(o.cache.pathComps[2:]...)
oldPath := jcstypes.JoinObjectPath(o.pathComps[2:]...)
newPath := jcstypes.JoinObjectPath(o.cache.pathComps[2:]...)
if o.isDeleted {
upder.CancelObject(oldPath)

6
client/internal/mount/vfs/cache/file.go vendored
View File

@ -14,7 +14,7 @@ import (
"gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/common/utils/serder"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type CacheLevel int
@ -102,7 +102,7 @@ type CacheFile struct {
cache *Cache
pathComps []string
info FileInfo
remoteObj *jcsypes.Object
remoteObj *jcstypes.Object
rwLock *sync.RWMutex
readers []*CacheFileHandle
writers []*CacheFileHandle
@ -262,7 +262,7 @@ func loadCacheFile(cache *Cache, pathComps []string) (*CacheFile, error) {
return ch, nil
}
func newCacheFileFromObject(cache *Cache, pathComps []string, obj *jcsypes.Object) (*CacheFile, error) {
func newCacheFileFromObject(cache *Cache, pathComps []string, obj *jcstypes.Object) (*CacheFile, error) {
metaPath := cache.GetCacheMetaPath(pathComps...)
dataPath := cache.GetCacheDataPath(pathComps...)

34
client/internal/mount/vfs/fuse.go
View File

@ -9,7 +9,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm"
)
@ -35,7 +35,7 @@ func child(vfs *Vfs, ctx context.Context, parent FuseNode, name string) (fuse.Fs
return nil
}
objPath := jcsypes.JoinObjectPath(childPathComps[2:]...)
objPath := jcstypes.JoinObjectPath(childPathComps[2:]...)
obj, err := d.Object().GetByPath(tx, pkg.PackageID, objPath)
if err == nil {
ret = newFileFromObject(vfs, childPathComps, obj)
@ -45,7 +45,7 @@ func child(vfs *Vfs, ctx context.Context, parent FuseNode, name string) (fuse.Fs
return err
}
err = d.Object().HasObjectWithPrefix(tx, pkg.PackageID, objPath+jcsypes.ObjectPathSeparator)
err = d.Object().HasObjectWithPrefix(tx, pkg.PackageID, objPath+jcstypes.ObjectPathSeparator)
if err == nil {
dir := vfs.cache.LoadDir(childPathComps, &cache.CreateDirOption{
ModTime: time.Now(),
@ -98,10 +98,10 @@ func listChildren(vfs *Vfs, ctx context.Context, parent FuseNode) ([]fuse.FsEntr
return err
}
objPath := jcsypes.JoinObjectPath(myPathComps[2:]...)
objPath := jcstypes.JoinObjectPath(myPathComps[2:]...)
objPrefix := objPath
if objPath != "" {
objPrefix += jcsypes.ObjectPathSeparator
objPrefix += jcstypes.ObjectPathSeparator
}
objs, coms, err := d.Object().GetByPrefixGrouped(tx, pkg.PackageID, objPrefix)
@ -110,8 +110,8 @@ func listChildren(vfs *Vfs, ctx context.Context, parent FuseNode) ([]fuse.FsEntr
}
for _, dir := range coms {
dir = strings.TrimSuffix(dir, jcsypes.ObjectPathSeparator)
pathComps := lo2.AppendNew(myPathComps, jcsypes.BaseName(dir))
dir = strings.TrimSuffix(dir, jcstypes.ObjectPathSeparator)
pathComps := lo2.AppendNew(myPathComps, jcstypes.BaseName(dir))
cd := vfs.cache.LoadDir(pathComps, &cache.CreateDirOption{
ModTime: time.Now(),
@ -124,7 +124,7 @@ func listChildren(vfs *Vfs, ctx context.Context, parent FuseNode) ([]fuse.FsEntr
}
for _, obj := range objs {
pathComps := lo2.AppendNew(myPathComps, jcsypes.BaseName(obj.Path))
pathComps := lo2.AppendNew(myPathComps, jcstypes.BaseName(obj.Path))
file := newFileFromObject(vfs, pathComps, obj)
dbEntries[file.Name()] = file
}
@ -179,14 +179,14 @@ func newFile(vfs *Vfs, ctx context.Context, name string, parent FuseNode, flags
func removeChild(vfs *Vfs, ctx context.Context, name string, parent FuseNode) error {
pathComps := lo2.AppendNew(parent.PathComps(), name)
joinedPath := jcsypes.JoinObjectPath(pathComps[2:]...)
joinedPath := jcstypes.JoinObjectPath(pathComps[2:]...)
d := vfs.db
// TODO 生成系统事件
return vfs.db.DoTx(func(tx db.SQLContext) error {
pkg, err := d.Package().GetByFullName(tx, pathComps[0], pathComps[1])
if err == nil {
err := d.Object().HasObjectWithPrefix(tx, pkg.PackageID, joinedPath+jcsypes.ObjectPathSeparator)
err := d.Object().HasObjectWithPrefix(tx, pkg.PackageID, joinedPath+jcstypes.ObjectPathSeparator)
if err == nil {
return fuse.ErrNotEmpty
}
@ -211,7 +211,7 @@ func removeChild(vfs *Vfs, ctx context.Context, name string, parent FuseNode) er
func moveChild(vfs *Vfs, ctx context.Context, oldName string, oldParent FuseNode, newName string, newParent FuseNode) error {
newParentPath := newParent.PathComps()
newChildPath := lo2.AppendNew(newParentPath, newName)
newChildPathJoined := jcsypes.JoinObjectPath(newChildPath[2:]...)
newChildPathJoined := jcstypes.JoinObjectPath(newChildPath[2:]...)
// 不允许移动任何内容到Package层级以上
if len(newParentPath) < 2 {
@ -219,7 +219,7 @@ func moveChild(vfs *Vfs, ctx context.Context, oldName string, oldParent FuseNode
}
oldChildPath := lo2.AppendNew(oldParent.PathComps(), oldName)
oldChildPathJoined := jcsypes.JoinObjectPath(oldChildPath[2:]...)
oldChildPathJoined := jcstypes.JoinObjectPath(oldChildPath[2:]...)
// 先更新远程，再更新本地，因为远程使用事务更新，可以回滚，而本地不行
return vfs.db.DoTx(func(tx db.SQLContext) error {
@ -259,7 +259,7 @@ func moveRemote(vfs *Vfs, tx db.SQLContext, oldChildPath []string, newParentPath
return fuse.ErrExists
}
err = d.Object().HasObjectWithPrefix(tx, newPkg.PackageID, newChildPathJoined+jcsypes.ObjectPathSeparator)
err = d.Object().HasObjectWithPrefix(tx, newPkg.PackageID, newChildPathJoined+jcstypes.ObjectPathSeparator)
if err == nil {
return fuse.ErrExists
}
@ -283,17 +283,17 @@ func moveRemote(vfs *Vfs, tx db.SQLContext, oldChildPath []string, newParentPath
oldObj.PackageID = newPkg.PackageID
oldObj.Path = newChildPathJoined
return d.Object().BatchUpdate(tx, []jcsypes.Object{oldObj})
return d.Object().BatchUpdate(tx, []jcstypes.Object{oldObj})
}
if err != gorm.ErrRecordNotFound {
return err
}
err = d.Object().HasObjectWithPrefix(tx, oldPkg.PackageID, oldChildPathJoined+jcsypes.ObjectPathSeparator)
err = d.Object().HasObjectWithPrefix(tx, oldPkg.PackageID, oldChildPathJoined+jcstypes.ObjectPathSeparator)
if err == nil {
return d.Object().MoveByPrefix(tx,
oldPkg.PackageID, oldChildPathJoined+jcsypes.ObjectPathSeparator,
newPkg.PackageID, newChildPathJoined+jcsypes.ObjectPathSeparator,
oldPkg.PackageID, oldChildPathJoined+jcstypes.ObjectPathSeparator,
newPkg.PackageID, newChildPathJoined+jcstypes.ObjectPathSeparator,
)
}
if err == gorm.ErrRecordNotFound {

4
client/internal/mount/vfs/fuse_bucket.go
View File

@ -9,7 +9,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm"
)
@ -117,7 +117,7 @@ func (r *FuseBucket) listChildren() ([]fuse.FsEntry, error) {
return nil, err
}
pkgMap := make(map[string]*jcsypes.Package)
pkgMap := make(map[string]*jcstypes.Package)
for _, pkg := range pkgs {
p := pkg
pkgMap[pkg.Name] = &p

4
client/internal/mount/vfs/fuse_dir.go
View File

@ -8,7 +8,7 @@ import (
db2 "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm"
)
@ -108,7 +108,7 @@ func (r *FuseDir) loadCacheDir() *cache.CacheDir {
return err
}
err = r.vfs.db.Object().HasObjectWithPrefix(tx, pkg.PackageID, jcsypes.JoinObjectPath(r.pathComps[2:]...))
err = r.vfs.db.Object().HasObjectWithPrefix(tx, pkg.PackageID, jcstypes.JoinObjectPath(r.pathComps[2:]...))
if err == nil {
createOpt = &cache.CreateDirOption{
ModTime: time.Now(),

6
client/internal/mount/vfs/fuse_file.go
View File

@ -6,7 +6,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm"
)
@ -28,7 +28,7 @@ func newFileFromCache(info cache.CacheEntryInfo, vfs *Vfs) *FuseFileNode {
}
}
func newFileFromObject(vfs *Vfs, pathComps []string, obj jcsypes.Object) *FuseFileNode {
func newFileFromObject(vfs *Vfs, pathComps []string, obj jcstypes.Object) *FuseFileNode {
return &FuseFileNode{
vfs: vfs,
pathComps: pathComps,
@ -117,7 +117,7 @@ func (n *FuseFileNode) loadCacheFile() *cache.CacheFile {
return n.vfs.cache.LoadFile(n.pathComps, nil)
}
cdsObj, err := n.vfs.db.Object().GetByFullPath(n.vfs.db.DefCtx(), n.pathComps[0], n.pathComps[1], jcsypes.JoinObjectPath(n.pathComps[2:]...))
cdsObj, err := n.vfs.db.Object().GetByFullPath(n.vfs.db.DefCtx(), n.pathComps[0], n.pathComps[1], jcstypes.JoinObjectPath(n.pathComps[2:]...))
if err == nil {
file := n.vfs.cache.LoadFile(n.pathComps, &cdsObj)
if file == nil {

4
client/internal/mount/vfs/fuse_root.go
View File

@ -8,7 +8,7 @@ import (
db2 "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm"
)
@ -104,7 +104,7 @@ func (r *FuseRoot) listChildren() ([]fuse.FsEntry, error) {
return nil, err
}
bktMap := make(map[string]*jcsypes.Bucket)
bktMap := make(map[string]*jcstypes.Bucket)
for _, bkt := range bkts {
b := bkt
bktMap[bkt.Name] = &b

8
client/internal/repl/load.go
View File

@ -8,7 +8,7 @@ import (
"time"
"github.com/spf13/cobra"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func init() {
@ -31,7 +31,7 @@ func init() {
fmt.Printf("Invalid user space ID: %s\n", args[1])
}
loadByID(cmdCtx, jcsypes.PackageID(pkgID), jcsypes.UserSpaceID(userSpaceID), args[2])
loadByID(cmdCtx, jcstypes.PackageID(pkgID), jcstypes.UserSpaceID(userSpaceID), args[2])
} else {
loadByPath(cmdCtx, args[0], args[1], args[2])
}
@ -42,7 +42,7 @@ func init() {
}
func loadByPath(cmdCtx *CommandContext, pkgPath string, stgName string, rootPath string) {
comps := strings.Split(strings.Trim(pkgPath, jcsypes.ObjectPathSeparator), jcsypes.ObjectPathSeparator)
comps := strings.Split(strings.Trim(pkgPath, jcstypes.ObjectPathSeparator), jcstypes.ObjectPathSeparator)
if len(comps) != 2 {
fmt.Printf("Package path must be in format of <bucket>/<package>")
return
@ -63,7 +63,7 @@ func loadByPath(cmdCtx *CommandContext, pkgPath string, stgName string, rootPath
loadByID(cmdCtx, pkg.PackageID, stg.StorageID, rootPath)
}
func loadByID(cmdCtx *CommandContext, pkgID jcsypes.PackageID, stgID jcsypes.StorageID, rootPath string) {
func loadByID(cmdCtx *CommandContext, pkgID jcstypes.PackageID, stgID jcstypes.StorageID, rootPath string) {
startTime := time.Now()
err := cmdCtx.Cmdline.Svc.StorageSvc().LoadPackage(pkgID, stgID, rootPath)

8
client/internal/repl/lsp.go
View File

@ -7,7 +7,7 @@ import (
"github.com/jedib0t/go-pretty/v6/table"
"github.com/spf13/cobra"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func init() {
@ -26,7 +26,7 @@ func init() {
return
}
lspOneByID(cmdCtx, jcsypes.PackageID(id))
lspOneByID(cmdCtx, jcstypes.PackageID(id))
} else {
lspByPath(cmdCtx, args[0])
}
@ -40,7 +40,7 @@ func init() {
func lspByPath(cmdCtx *CommandContext, path string) {
db2 := cmdCtx.repl.db
comps := strings.Split(strings.Trim(path, jcsypes.ObjectPathSeparator), jcsypes.ObjectPathSeparator)
comps := strings.Split(strings.Trim(path, jcstypes.ObjectPathSeparator), jcstypes.ObjectPathSeparator)
if len(comps) != 2 {
fmt.Printf("Package path must be in format of <bucket>/<package>")
return
@ -58,7 +58,7 @@ func lspByPath(cmdCtx *CommandContext, path string) {
fmt.Println(wr.Render())
}
func lspOneByID(cmdCtx *CommandContext, id jcsypes.PackageID) {
func lspOneByID(cmdCtx *CommandContext, id jcstypes.PackageID) {
db2 := cmdCtx.repl.db
pkg, err := db2.Package().GetByID(db2.DefCtx(), id)

52
client/internal/services/user_space.go
View File

@ -8,7 +8,7 @@ import (
"github.com/samber/lo"
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
@ -31,41 +31,41 @@ func (svc *Service) UserSpaceSvc() *UserSpaceService {
return &UserSpaceService{Service: svc}
}
func (svc *UserSpaceService) Get(userspaceID jcsypes.UserSpaceID) (jcsypes.UserSpace, error) {
func (svc *UserSpaceService) Get(userspaceID jcstypes.UserSpaceID) (jcstypes.UserSpace, error) {
return svc.DB.UserSpace().GetByID(svc.DB.DefCtx(), userspaceID)
}
func (svc *UserSpaceService) GetByName(name string) (jcsypes.UserSpace, error) {
func (svc *UserSpaceService) GetByName(name string) (jcstypes.UserSpace, error) {
return svc.DB.UserSpace().GetByName(svc.DB.DefCtx(), name)
}
func (svc *UserSpaceService) GetAll() ([]jcsypes.UserSpace, error) {
func (svc *UserSpaceService) GetAll() ([]jcstypes.UserSpace, error) {
return svc.DB.UserSpace().GetAll(svc.DB.DefCtx())
}
func (svc *UserSpaceService) Create(req cliapi.UserSpaceCreate) (*cliapi.UserSpaceCreateResp, *ecode.CodeError) {
db2 := svc.DB
space, err := db.DoTx01(db2, func(tx db.SQLContext) (jcsypes.UserSpace, error) {
space, err := db.DoTx01(db2, func(tx db.SQLContext) (jcstypes.UserSpace, error) {
space, err := db2.UserSpace().GetByName(tx, req.Name)
if err == nil {
return jcsypes.UserSpace{}, gorm.ErrDuplicatedKey
return jcstypes.UserSpace{}, gorm.ErrDuplicatedKey
}
if err != gorm.ErrRecordNotFound {
return jcsypes.UserSpace{}, err
return jcstypes.UserSpace{}, err
}
space = jcsypes.UserSpace{
space = jcstypes.UserSpace{
Name: req.Name,
Storage: req.Storage,
Credential: req.Credential,
ShardStore: req.ShardStore,
Features: req.Features,
WorkingDir: jcsypes.PathFromJcsPathString(req.WorkingDir),
WorkingDir: jcstypes.PathFromJcsPathString(req.WorkingDir),
Revision: 0,
}
err = db2.UserSpace().Create(tx, &space)
if err != nil {
return jcsypes.UserSpace{}, err
return jcstypes.UserSpace{}, err
}
return space, nil
})
@ -80,19 +80,19 @@ func (svc *UserSpaceService) Create(req cliapi.UserSpaceCreate) (*cliapi.UserSpa
func (svc *UserSpaceService) Update(req cliapi.UserSpaceUpdate) (*cliapi.UserSpaceUpdateResp, *ecode.CodeError) {
db2 := svc.DB
space, err := db.DoTx01(db2, func(tx db.SQLContext) (jcsypes.UserSpace, error) {
space, err := db.DoTx01(db2, func(tx db.SQLContext) (jcstypes.UserSpace, error) {
space, err := db2.UserSpace().GetByID(tx, req.UserSpaceID)
if err != nil {
return jcsypes.UserSpace{}, err
return jcstypes.UserSpace{}, err
}
if space.Name != req.Name {
_, err = db2.UserSpace().GetByName(tx, req.Name)
if err == nil {
return jcsypes.UserSpace{}, gorm.ErrDuplicatedKey
return jcstypes.UserSpace{}, gorm.ErrDuplicatedKey
}
if err != gorm.ErrRecordNotFound {
return jcsypes.UserSpace{}, err
return jcstypes.UserSpace{}, err
}
}
@ -110,7 +110,7 @@ func (svc *UserSpaceService) Update(req cliapi.UserSpaceUpdate) (*cliapi.UserSpa
}
// 通知元数据缓存无效
svc.UserSpaceMeta.Drop([]jcsypes.UserSpaceID{req.UserSpaceID})
svc.UserSpaceMeta.Drop([]jcstypes.UserSpaceID{req.UserSpaceID})
// 通知存储服务组件池停止组件。TODO 对于在Hub上运行的组件，需要一个机制去定时清理
svc.StgPool.Drop(stgglb.UserID, space.UserSpaceID)
@ -155,7 +155,7 @@ func (svc *UserSpaceService) Delete(req cliapi.UserSpaceDelete) (*cliapi.UserSpa
}
// 通知元数据缓存无效
svc.UserSpaceMeta.Drop([]jcsypes.UserSpaceID{req.UserSpaceID})
svc.UserSpaceMeta.Drop([]jcstypes.UserSpaceID{req.UserSpaceID})
// 通知存储服务组件池停止组件。TODO 对于在Hub上运行的组件，需要一个机制去定时清理
svc.StgPool.Drop(stgglb.UserID, req.UserSpaceID)
@ -166,13 +166,13 @@ func (svc *UserSpaceService) Delete(req cliapi.UserSpaceDelete) (*cliapi.UserSpa
}
func (svc *UserSpaceService) Test(req cliapi.UserSpaceTest) (*cliapi.UserSpaceTestResp, *ecode.CodeError) {
detail := jcsypes.UserSpaceDetail{
detail := jcstypes.UserSpaceDetail{
UserID: stgglb.UserID,
UserSpace: jcsypes.UserSpace{
UserSpace: jcstypes.UserSpace{
Name: "test",
Storage: req.Storage,
Credential: req.Credential,
WorkingDir: jcsypes.PathFromJcsPathString(req.WorikingDir),
WorkingDir: jcstypes.PathFromJcsPathString(req.WorikingDir),
},
}
blder := factory.GetBuilder(&detail)
@ -189,7 +189,7 @@ func (svc *UserSpaceService) Test(req cliapi.UserSpaceTest) (*cliapi.UserSpaceTe
return &cliapi.UserSpaceTestResp{}, nil
}
func (svc *UserSpaceService) DownloadPackage(packageID jcsypes.PackageID, userspaceID jcsypes.UserSpaceID, rootPath string) error {
func (svc *UserSpaceService) DownloadPackage(packageID jcstypes.PackageID, userspaceID jcstypes.UserSpaceID, rootPath string) error {
destSpace := svc.UserSpaceMeta.Get(userspaceID)
if destSpace == nil {
return fmt.Errorf("userspace not found: %d", userspaceID)
@ -208,10 +208,10 @@ func (svc *UserSpaceService) DownloadPackage(packageID jcsypes.PackageID, usersp
}
defer mutex.Unlock()
rootJPath := jcsypes.PathFromJcsPathString(rootPath)
rootJPath := jcstypes.PathFromJcsPathString(rootPath)
dIndex := 0
var pinned []jcsypes.PinnedObject
var pinned []jcstypes.PinnedObject
for dIndex < len(details) {
plans := exec.NewPlanBuilder()
for i := 0; i < 10 && dIndex < len(details); i++ {
@ -252,7 +252,7 @@ func (svc *UserSpaceService) DownloadPackage(packageID jcsypes.PackageID, usersp
return fmt.Errorf("unsupported download strategy: %T", strg)
}
objPath := jcsypes.PathFromJcsPathString(details[dIndex].Object.Path)
objPath := jcstypes.PathFromJcsPathString(details[dIndex].Object.Path)
dstPath := rootJPath.ConcatNew(objPath)
newDstSpace := *destSpace
@ -265,7 +265,7 @@ func (svc *UserSpaceService) DownloadPackage(packageID jcsypes.PackageID, usersp
// 顺便保存到同存储服务的分片存储中
if destSpace.UserSpace.ShardStore != nil {
ft.AddTo(ioswitch2.NewToShardStore(newDstSpace, ioswitch2.RawStream(), ""))
pinned = append(pinned, jcsypes.PinnedObject{
pinned = append(pinned, jcstypes.PinnedObject{
ObjectID: details[dIndex].Object.ObjectID,
UserSpaceID: destSpace.UserSpace.UserSpaceID,
CreateTime: time.Now(),
@ -297,7 +297,7 @@ func (svc *UserSpaceService) DownloadPackage(packageID jcsypes.PackageID, usersp
}
err = svc.DB.DoTx(func(tx db.SQLContext) error {
objIDs := make([]jcsypes.ObjectID, len(pinned))
objIDs := make([]jcstypes.ObjectID, len(pinned))
for i, obj := range pinned {
objIDs[i] = obj.ObjectID
}
@ -307,7 +307,7 @@ func (svc *UserSpaceService) DownloadPackage(packageID jcsypes.PackageID, usersp
return err
}
pinned = lo.Filter(pinned, func(p jcsypes.PinnedObject, idx int) bool { return avaiIDs[p.ObjectID] })
pinned = lo.Filter(pinned, func(p jcstypes.PinnedObject, idx int) bool { return avaiIDs[p.ObjectID] })
return svc.DB.PinnedObject().BatchTryCreate(svc.DB.DefCtx(), pinned)
})
if err != nil {

12
client/internal/services/utils.go
View File

@ -1,22 +1,22 @@
package services
import (
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
)
func getBlockTypeFromRed(red jcsypes.Redundancy) string {
func getBlockTypeFromRed(red jcstypes.Redundancy) string {
switch red.(type) {
case *jcsypes.NoneRedundancy:
case *jcstypes.NoneRedundancy:
return datamap.BlockTypeRaw
case *jcsypes.ECRedundancy:
case *jcstypes.ECRedundancy:
return datamap.BlockTypeEC
case *jcsypes.LRCRedundancy:
case *jcstypes.LRCRedundancy:
return datamap.BlockTypeEC
case *jcsypes.SegmentRedundancy:
case *jcstypes.SegmentRedundancy:
return datamap.BlockTypeSegment
}
return ""

14
client/internal/spacesyncer/execute_diff.go
View File

@ -12,10 +12,10 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func executeDiff(syncer *SpaceSyncer, task *task, mode *jcsypes.SpaceSyncModeDiff) {
func executeDiff(syncer *SpaceSyncer, task *task, mode *jcstypes.SpaceSyncModeDiff) {
log := logger.WithField("Mod", logMod).WithField("TaskID", task.Task.TaskID)
startTime := time.Now()
@ -112,7 +112,7 @@ func executeDiff(syncer *SpaceSyncer, task *task, mode *jcsypes.SpaceSyncModeDif
}
var willSync []stgtypes.DirEntry
var willMkdirs []jcsypes.JPath
var willMkdirs []jcstypes.JPath
dirTree.Iterate(func(path []string, node *trie.Node[srcDstDirEntry], isWordNode bool) trie.VisitCtrl {
if node.Value.src == nil {
@ -123,7 +123,7 @@ func executeDiff(syncer *SpaceSyncer, task *task, mode *jcsypes.SpaceSyncModeDif
if node.Value.src.IsDir {
if node.Value.dst == nil {
if node.IsEmpty() {
willMkdirs = append(willMkdirs, jcsypes.PathFromComps(path...))
willMkdirs = append(willMkdirs, jcstypes.PathFromComps(path...))
}
}
} else {
@ -193,7 +193,7 @@ func executeDiff(syncer *SpaceSyncer, task *task, mode *jcsypes.SpaceSyncModeDif
}
}
func diffCreateSrcNode(tree *trie.Trie[srcDstDirEntry], path jcsypes.JPath, e *stgtypes.DirEntry) {
func diffCreateSrcNode(tree *trie.Trie[srcDstDirEntry], path jcstypes.JPath, e *stgtypes.DirEntry) {
var ptr = &tree.Root
for _, c := range path.Comps() {
if ptr.Value.src != nil && ptr.Value.src.IsDir {
@ -205,7 +205,7 @@ func diffCreateSrcNode(tree *trie.Trie[srcDstDirEntry], path jcsypes.JPath, e *s
ptr.Value.src = e
}
func diffCreateDstNode(tree *trie.Trie[srcDstDirEntry], path jcsypes.JPath, e *stgtypes.DirEntry) {
func diffCreateDstNode(tree *trie.Trie[srcDstDirEntry], path jcstypes.JPath, e *stgtypes.DirEntry) {
var ptr = &tree.Root
for _, c := range path.Comps() {
if ptr.Value.src != nil && ptr.Value.src.IsDir {
@ -227,7 +227,7 @@ type srcDstDirEntry struct {
dst *stgtypes.DirEntry
}
func cmpFile(diff *jcsypes.SpaceSyncModeDiff, src, dst *stgtypes.DirEntry) bool {
func cmpFile(diff *jcstypes.SpaceSyncModeDiff, src, dst *stgtypes.DirEntry) bool {
if diff.IncludeSize && src.Size != dst.Size {
return false
}

6
client/internal/spacesyncer/filter.go
View File

@ -2,7 +2,7 @@ package spacesyncer
import (
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type FilterFn func(info stgtypes.DirEntry) bool
@ -11,7 +11,7 @@ func buildFilter(task *task) FilterFn {
var fns []FilterFn
for _, f := range task.Task.Filters {
switch f := f.(type) {
case *jcsypes.SpaceSyncFilterSize:
case *jcstypes.SpaceSyncFilterSize:
fns = append(fns, filterSize(f))
}
}
@ -26,7 +26,7 @@ func buildFilter(task *task) FilterFn {
}
}
func filterSize(filter *jcsypes.SpaceSyncFilterSize) FilterFn {
func filterSize(filter *jcstypes.SpaceSyncFilterSize) FilterFn {
return func(info stgtypes.DirEntry) bool {
if filter.MinSize > 0 && info.Size < filter.MinSize {
return false

32
client/internal/spacesyncer/space_syncer.go
View File

@ -10,7 +10,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/metacache"
stgpool "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
const (
@ -26,7 +26,7 @@ type SpaceSyncer struct {
stgPool *stgpool.Pool
spaceMeta *metacache.UserSpaceMeta
lock sync.Mutex
tasks map[jcsypes.SpaceSyncTaskID]*task
tasks map[jcstypes.SpaceSyncTaskID]*task
}
func New(db *db.DB, stgPool *stgpool.Pool, spaceMeta *metacache.UserSpaceMeta) *SpaceSyncer {
@ -34,7 +34,7 @@ func New(db *db.DB, stgPool *stgpool.Pool, spaceMeta *metacache.UserSpaceMeta) *
db: db,
stgPool: stgPool,
spaceMeta: spaceMeta,
tasks: make(map[jcsypes.SpaceSyncTaskID]*task),
tasks: make(map[jcstypes.SpaceSyncTaskID]*task),
}
}
@ -50,7 +50,7 @@ func (s *SpaceSyncer) Start() *async.UnboundChannel[SpaceSyncerEvent] {
if err != nil {
log.Warnf("load task from db: %v", err)
} else {
var rms []jcsypes.SpaceSyncTaskID
var rms []jcstypes.SpaceSyncTaskID
for _, t := range allTask {
ctx, cancel := context.WithCancel(context.Background())
tsk := task{
@ -60,14 +60,14 @@ func (s *SpaceSyncer) Start() *async.UnboundChannel[SpaceSyncerEvent] {
}
switch tr := t.Trigger.(type) {
case *jcsypes.SpaceSyncTriggerOnce:
case *jcstypes.SpaceSyncTriggerOnce:
// Once类型的任务没有执行完也不执行了
rms = append(rms, t.TaskID)
case *jcsypes.SpaceSyncTriggerInterval:
case *jcstypes.SpaceSyncTriggerInterval:
triggerInterval(s, &tsk, tr)
case *jcsypes.SpaceSyncTriggerAt:
case *jcstypes.SpaceSyncTriggerAt:
triggerAt(s, &tsk, tr)
}
@ -95,10 +95,10 @@ func (s *SpaceSyncer) Stop() {
t.CancelFn()
}
s.tasks = make(map[jcsypes.SpaceSyncTaskID]*task)
s.tasks = make(map[jcstypes.SpaceSyncTaskID]*task)
}
func (s *SpaceSyncer) CreateTask(t jcsypes.SpaceSyncTask) (*TaskInfo, error) {
func (s *SpaceSyncer) CreateTask(t jcstypes.SpaceSyncTask) (*TaskInfo, error) {
log := logger.WithField("Mod", logMod)
d := s.db
@ -126,13 +126,13 @@ func (s *SpaceSyncer) CreateTask(t jcsypes.SpaceSyncTask) (*TaskInfo, error) {
s.lock.Unlock()
switch tr := t.Trigger.(type) {
case *jcsypes.SpaceSyncTriggerOnce:
case *jcstypes.SpaceSyncTriggerOnce:
triggerOnce(s, &tsk)
case *jcsypes.SpaceSyncTriggerInterval:
case *jcstypes.SpaceSyncTriggerInterval:
triggerInterval(s, &tsk, tr)
case *jcsypes.SpaceSyncTriggerAt:
case *jcstypes.SpaceSyncTriggerAt:
triggerAt(s, &tsk, tr)
}
@ -143,7 +143,7 @@ func (s *SpaceSyncer) CreateTask(t jcsypes.SpaceSyncTask) (*TaskInfo, error) {
}, nil
}
func (s *SpaceSyncer) CancelTask(taskID jcsypes.SpaceSyncTaskID) {
func (s *SpaceSyncer) CancelTask(taskID jcstypes.SpaceSyncTaskID) {
log := logger.WithField("Mod", logMod)
s.lock.Lock()
@ -166,7 +166,7 @@ func (s *SpaceSyncer) CancelTask(taskID jcsypes.SpaceSyncTaskID) {
log.Infof("task %v canceled", taskID)
}
func (s *SpaceSyncer) GetTask(taskID jcsypes.SpaceSyncTaskID) *jcsypes.SpaceSyncTask {
func (s *SpaceSyncer) GetTask(taskID jcstypes.SpaceSyncTaskID) *jcstypes.SpaceSyncTask {
s.lock.Lock()
defer s.lock.Unlock()
@ -181,11 +181,11 @@ func (s *SpaceSyncer) GetTask(taskID jcsypes.SpaceSyncTaskID) *jcsypes.SpaceSync
}
type TaskInfo struct {
Task jcsypes.SpaceSyncTask
Task jcstypes.SpaceSyncTask
}
type task struct {
Task jcsypes.SpaceSyncTask
Task jcstypes.SpaceSyncTask
Context context.Context
CancelFn func()
}

6
client/internal/spacesyncer/trigger.go
View File

@ -5,7 +5,7 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/sort2"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func triggerOnce(syncer *SpaceSyncer, task *task) {
@ -31,7 +31,7 @@ func triggerOnce(syncer *SpaceSyncer, task *task) {
}()
}
func triggerInterval(syncer *SpaceSyncer, task *task, trigger *jcsypes.SpaceSyncTriggerInterval) {
func triggerInterval(syncer *SpaceSyncer, task *task, trigger *jcstypes.SpaceSyncTriggerInterval) {
go func() {
log := logger.WithField("Mod", logMod)
@ -66,7 +66,7 @@ func triggerInterval(syncer *SpaceSyncer, task *task, trigger *jcsypes.SpaceSync
}()
}
func triggerAt(syncer *SpaceSyncer, task *task, trigger *jcsypes.SpaceSyncTriggerAt) {
func triggerAt(syncer *SpaceSyncer, task *task, trigger *jcstypes.SpaceSyncTriggerAt) {
go func() {
log := logger.WithField("Mod", logMod)

12
client/internal/speedstats/speedstats.go
View File

@ -5,7 +5,7 @@ import (
"sync"
"time"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
const (
@ -121,9 +121,9 @@ func (p *SpeedStats) ShouldAtClient(size int64) bool {
return v < prob
}
func (p *SpeedStats) DumpStatus() jcsypes.SpeedStatsStatus {
return jcsypes.SpeedStatsStatus{
Below100M: []jcsypes.SpeedStatsStatusEntry{
func (p *SpeedStats) DumpStatus() jcstypes.SpeedStatsStatus {
return jcstypes.SpeedStatsStatus{
Below100M: []jcstypes.SpeedStatsStatusEntry{
{
TotalSize: p.stats100M[0].TotalSize,
TotalTime: p.stats100M[0].TotalTime,
@ -137,7 +137,7 @@ func (p *SpeedStats) DumpStatus() jcsypes.SpeedStatsStatus {
LastSpeed: p.stats100M[1].LastSpeed,
},
},
Below1G: []jcsypes.SpeedStatsStatusEntry{
Below1G: []jcstypes.SpeedStatsStatusEntry{
{
TotalSize: p.stats1G[0].TotalSize,
TotalTime: p.stats1G[0].TotalTime,
@ -151,7 +151,7 @@ func (p *SpeedStats) DumpStatus() jcsypes.SpeedStatsStatus {
LastSpeed: p.stats1G[1].LastSpeed,
},
},
Above1G: []jcsypes.SpeedStatsStatusEntry{
Above1G: []jcstypes.SpeedStatsStatusEntry{
{
TotalSize: p.statsAbove1G[0].TotalSize,
TotalTime: p.statsAbove1G[0].TotalTime,

18
client/internal/ticktock/change_redundancy.go
View File

@ -8,7 +8,7 @@ import (
"gitlink.org.cn/cloudream/common/utils/reflect2"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
)
@ -34,7 +34,7 @@ func (j *ChangeRedundancy) Execute(t *TickTock) {
ctx := &changeRedundancyContext{
ticktock: t,
allUserSpaces: make(map[jcsypes.UserSpaceID]*userSpaceUsageInfo),
allUserSpaces: make(map[jcstypes.UserSpaceID]*userSpaceUsageInfo),
}
spaceIDs, err := t.db.UserSpace().GetAllIDs(t.db.DefCtx())
@ -58,7 +58,7 @@ func (j *ChangeRedundancy) Execute(t *TickTock) {
return
}
lastPkgID := jcsypes.PackageID(0)
lastPkgID := jcstypes.PackageID(0)
loop:
for {
@ -90,16 +90,16 @@ loop:
type changeRedundancyContext struct {
ticktock *TickTock
allUserSpaces map[jcsypes.UserSpaceID]*userSpaceUsageInfo
mostBlockStgIDs []jcsypes.UserSpaceID
allUserSpaces map[jcstypes.UserSpaceID]*userSpaceUsageInfo
mostBlockStgIDs []jcstypes.UserSpaceID
}
type userSpaceUsageInfo struct {
UserSpace *jcsypes.UserSpaceDetail
UserSpace *jcstypes.UserSpaceDetail
AccessAmount float64
}
func (j *ChangeRedundancy) changeOne(ctx *changeRedundancyContext, pkg jcsypes.PackageDetail) error {
func (j *ChangeRedundancy) changeOne(ctx *changeRedundancyContext, pkg jcstypes.PackageDetail) error {
log := logger.WithType[ChangeRedundancy]("TickTock")
db2 := ctx.ticktock.db
@ -121,7 +121,7 @@ func (j *ChangeRedundancy) changeOne(ctx *changeRedundancyContext, pkg jcsypes.P
info.AccessAmount = stat.Amount
}
lastObjID := jcsypes.ObjectID(0)
lastObjID := jcstypes.ObjectID(0)
for {
objs, err := db.DoTx31(db2, db2.Object().BatchGetDetailsPaged, pkg.Package.PackageID, lastObjID, BatchGetObjectDetailCount)
if err != nil {
@ -139,7 +139,7 @@ func (j *ChangeRedundancy) changeOne(ctx *changeRedundancyContext, pkg jcsypes.P
ctx.mostBlockStgIDs = j.summaryRepObjectBlockUserSpaces(ctx, objs, 2)
var willShrinks []jcsypes.ObjectDetail
var willShrinks []jcstypes.ObjectDetail
for _, obj := range objs {
newRed, selectedSpaces := j.chooseRedundancy(ctx, obj)

18
client/internal/ticktock/check_shardstore.go
View File

@ -10,7 +10,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
// CheckShardStore 代表一个用于处理代理缓存检查事件的结构体
@ -52,8 +52,8 @@ func (j *CheckShardStore) Execute(t *TickTock) {
}
}
func (j *CheckShardStore) checkOne(t *TickTock, space *jcsypes.UserSpaceDetail) error {
// addr, ok := space.RecommendHub.Address.(*cortypes.GRPCAddressInfo)
func (j *CheckShardStore) checkOne(t *TickTock, space *jcstypes.UserSpaceDetail) error {
// addr, ok := space.RecommendHub.Address.(*jcstypes.GRPCAddressInfo)
// if !ok {
// return fmt.Errorf("master of user space %v has no grpc address", space.UserSpace)
// }
@ -80,9 +80,9 @@ func (j *CheckShardStore) checkOne(t *TickTock, space *jcsypes.UserSpaceDetail)
return fmt.Errorf("listing all files: %w", err)
}
fileHashes := lo.Map(infos, func(info stgtypes.FileInfo, _ int) jcsypes.FileHash { return info.Hash })
fileHashes := lo.Map(infos, func(info stgtypes.FileInfo, _ int) jcstypes.FileHash { return info.Hash })
realFileHashes := lo.SliceToMap(fileHashes, func(hash jcsypes.FileHash) (jcsypes.FileHash, bool) { return hash, true })
realFileHashes := lo.SliceToMap(fileHashes, func(hash jcstypes.FileHash) (jcstypes.FileHash, bool) { return hash, true })
// 在事务中执行缓存更新操作
t.db.DoTx(func(tx db.SQLContext) error {
@ -95,7 +95,7 @@ func (j *CheckShardStore) checkOne(t *TickTock, space *jcsypes.UserSpaceDetail)
}
// checkPinnedObject 对比PinnedObject表，若实际文件不存在，则进行删除操作
func (*CheckShardStore) checkPinnedObject(t *TickTock, tx db.SQLContext, space *jcsypes.UserSpaceDetail, realFileHashes map[jcsypes.FileHash]bool) {
func (*CheckShardStore) checkPinnedObject(t *TickTock, tx db.SQLContext, space *jcstypes.UserSpaceDetail, realFileHashes map[jcstypes.FileHash]bool) {
log := logger.WithType[CheckShardStore]("TickTock")
objs, err := t.db.PinnedObject().GetObjectsByUserSpaceID(tx, space.UserSpace.UserSpaceID)
@ -104,7 +104,7 @@ func (*CheckShardStore) checkPinnedObject(t *TickTock, tx db.SQLContext, space *
return
}
var rms []jcsypes.ObjectID
var rms []jcstypes.ObjectID
for _, c := range objs {
if realFileHashes[c.FileHash] {
continue
@ -121,7 +121,7 @@ func (*CheckShardStore) checkPinnedObject(t *TickTock, tx db.SQLContext, space *
}
// checkObjectBlock 对比ObjectBlock表，若实际文件不存在，则进行删除操作
func (*CheckShardStore) checkObjectBlock(t *TickTock, tx db.SQLContext, space *jcsypes.UserSpaceDetail, realFileHashes map[jcsypes.FileHash]bool) {
func (*CheckShardStore) checkObjectBlock(t *TickTock, tx db.SQLContext, space *jcstypes.UserSpaceDetail, realFileHashes map[jcstypes.FileHash]bool) {
log := logger.WithType[CheckShardStore]("TickTock")
blocks, err := t.db.ObjectBlock().GetByUserSpaceID(tx, space.UserSpace.UserSpaceID)
@ -130,7 +130,7 @@ func (*CheckShardStore) checkObjectBlock(t *TickTock, tx db.SQLContext, space *j
return
}
var rms []jcsypes.FileHash
var rms []jcstypes.FileHash
for _, b := range blocks {
if realFileHashes[b.FileHash] {
continue

177
client/internal/ticktock/redundancy_recover.go
View File

@ -16,56 +16,55 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
lrcparser "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc/parser"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
)
func (t *ChangeRedundancy) chooseRedundancy(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail) (jcsypes.Redundancy, []*userSpaceUsageInfo) {
func (t *ChangeRedundancy) chooseRedundancy(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail) (jcstypes.Redundancy, []*userSpaceUsageInfo) {
switch obj.Object.Redundancy.(type) {
case *jcsypes.NoneRedundancy:
case *jcstypes.NoneRedundancy:
if obj.Object.Size > ctx.ticktock.cfg.ECFileSizeThreshold {
newStgs := t.chooseNewUserSpacesForEC(ctx, &jcsypes.DefaultECRedundancy)
return &jcsypes.DefaultECRedundancy, newStgs
newStgs := t.chooseNewUserSpacesForEC(ctx, &jcstypes.DefaultECRedundancy)
return &jcstypes.DefaultECRedundancy, newStgs
}
return &jcsypes.DefaultRepRedundancy, t.chooseNewUserSpacesForRep(ctx, &jcsypes.DefaultRepRedundancy)
return &jcstypes.DefaultRepRedundancy, t.chooseNewUserSpacesForRep(ctx, &jcstypes.DefaultRepRedundancy)
case *jcsypes.RepRedundancy:
case *jcstypes.RepRedundancy:
if obj.Object.Size >= ctx.ticktock.cfg.ECFileSizeThreshold {
newStgs := t.chooseNewUserSpacesForEC(ctx, &jcsypes.DefaultECRedundancy)
return &jcsypes.DefaultECRedundancy, newStgs
newStgs := t.chooseNewUserSpacesForEC(ctx, &jcstypes.DefaultECRedundancy)
return &jcstypes.DefaultECRedundancy, newStgs
}
newSpaces := t.rechooseUserSpacesForRep(ctx, &jcsypes.DefaultRepRedundancy)
newSpaces := t.rechooseUserSpacesForRep(ctx, &jcstypes.DefaultRepRedundancy)
for _, s := range newSpaces {
if !obj.ContainsBlock(0, s.UserSpace.UserSpace.UserSpaceID) && !obj.ContainsPinned(s.UserSpace.UserSpace.UserSpaceID) {
return &jcsypes.DefaultRepRedundancy, newSpaces
return &jcstypes.DefaultRepRedundancy, newSpaces
}
}
return nil, nil
case *jcsypes.ECRedundancy:
case *jcstypes.ECRedundancy:
if obj.Object.Size < ctx.ticktock.cfg.ECFileSizeThreshold {
return &jcsypes.DefaultRepRedundancy, t.chooseNewUserSpacesForRep(ctx, &jcsypes.DefaultRepRedundancy)
return &jcstypes.DefaultRepRedundancy, t.chooseNewUserSpacesForRep(ctx, &jcstypes.DefaultRepRedundancy)
}
newSpaces := t.rechooseUserSpacesForEC(ctx, obj, &jcsypes.DefaultECRedundancy)
newSpaces := t.rechooseUserSpacesForEC(ctx, obj, &jcstypes.DefaultECRedundancy)
for i, s := range newSpaces {
if !obj.ContainsBlock(i, s.UserSpace.UserSpace.UserSpaceID) {
return &jcsypes.DefaultECRedundancy, newSpaces
return &jcstypes.DefaultECRedundancy, newSpaces
}
}
return nil, nil
case *jcsypes.LRCRedundancy:
newLRCStgs := t.rechooseUserSpacesForLRC(ctx, obj, &jcsypes.DefaultLRCRedundancy)
case *jcstypes.LRCRedundancy:
newLRCStgs := t.rechooseUserSpacesForLRC(ctx, obj, &jcstypes.DefaultLRCRedundancy)
for i, s := range newLRCStgs {
if !obj.ContainsBlock(i, s.UserSpace.UserSpace.UserSpaceID) {
return &jcsypes.DefaultLRCRedundancy, newLRCStgs
return &jcstypes.DefaultLRCRedundancy, newLRCStgs
}
}
@ -74,7 +73,7 @@ func (t *ChangeRedundancy) chooseRedundancy(ctx *changeRedundancyContext, obj jc
return nil, nil
}
func (t *ChangeRedundancy) doChangeRedundancy(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, newRed jcsypes.Redundancy, selectedUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) doChangeRedundancy(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, newRed jcstypes.Redundancy, selectedUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
log := logger.WithType[ChangeRedundancy]("TickTock")
var updating *db.UpdatingObjectRedundancy
@ -82,48 +81,48 @@ func (t *ChangeRedundancy) doChangeRedundancy(ctx *changeRedundancyContext, obj
var err error
switch srcRed := obj.Object.Redundancy.(type) {
case *jcsypes.NoneRedundancy:
case *jcstypes.NoneRedundancy:
switch newRed := newRed.(type) {
case *jcsypes.RepRedundancy:
case *jcstypes.RepRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> rep")
updating, evt, err = t.noneToRep(ctx, obj, newRed, selectedUserSpaces)
case *jcsypes.ECRedundancy:
case *jcstypes.ECRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> ec")
updating, evt, err = t.noneToEC(ctx, obj, newRed, selectedUserSpaces)
case *jcsypes.LRCRedundancy:
case *jcstypes.LRCRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> lrc")
updating, evt, err = t.noneToLRC(ctx, obj, newRed, selectedUserSpaces)
case *jcsypes.SegmentRedundancy:
case *jcstypes.SegmentRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> segment")
updating, evt, err = t.noneToSeg(ctx, obj, newRed, selectedUserSpaces)
}
case *jcsypes.RepRedundancy:
case *jcstypes.RepRedundancy:
switch newRed := newRed.(type) {
case *jcsypes.RepRedundancy:
case *jcstypes.RepRedundancy:
updating, evt, err = t.repToRep(ctx, obj, srcRed, selectedUserSpaces)
case *jcsypes.ECRedundancy:
case *jcstypes.ECRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: rep -> ec")
updating, evt, err = t.repToEC(ctx, obj, newRed, selectedUserSpaces)
}
case *jcsypes.ECRedundancy:
case *jcstypes.ECRedundancy:
switch newRed := newRed.(type) {
case *jcsypes.RepRedundancy:
case *jcstypes.RepRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: ec -> rep")
updating, evt, err = t.ecToRep(ctx, obj, srcRed, newRed, selectedUserSpaces)
case *jcsypes.ECRedundancy:
case *jcstypes.ECRedundancy:
updating, evt, err = t.ecToEC(ctx, obj, srcRed, newRed, selectedUserSpaces)
}
case *jcsypes.LRCRedundancy:
case *jcstypes.LRCRedundancy:
switch newRed := newRed.(type) {
case *jcsypes.LRCRedundancy:
case *jcstypes.LRCRedundancy:
updating, evt, err = t.lrcToLRC(ctx, obj, srcRed, newRed, selectedUserSpaces)
}
}
@ -132,16 +131,16 @@ func (t *ChangeRedundancy) doChangeRedundancy(ctx *changeRedundancyContext, obj
}
// 统计每个对象块所在的节点，选出块最多的不超过userspaceCnt个节点
func (t *ChangeRedundancy) summaryRepObjectBlockUserSpaces(ctx *changeRedundancyContext, objs []jcsypes.ObjectDetail, userspaceCnt int) []jcsypes.UserSpaceID {
func (t *ChangeRedundancy) summaryRepObjectBlockUserSpaces(ctx *changeRedundancyContext, objs []jcstypes.ObjectDetail, userspaceCnt int) []jcstypes.UserSpaceID {
type stgBlocks struct {
UserSpaceID jcsypes.UserSpaceID
UserSpaceID jcstypes.UserSpaceID
Count int
}
stgBlocksMap := make(map[jcsypes.UserSpaceID]*stgBlocks)
stgBlocksMap := make(map[jcstypes.UserSpaceID]*stgBlocks)
for _, obj := range objs {
shouldUseEC := obj.Object.Size > ctx.ticktock.cfg.ECFileSizeThreshold
if _, ok := obj.Object.Redundancy.(*jcsypes.RepRedundancy); ok && !shouldUseEC {
if _, ok := obj.Object.Redundancy.(*jcstypes.RepRedundancy); ok && !shouldUseEC {
for _, block := range obj.Blocks {
if _, ok := stgBlocksMap[block.UserSpaceID]; !ok {
stgBlocksMap[block.UserSpaceID] = &stgBlocks{
@ -159,14 +158,14 @@ func (t *ChangeRedundancy) summaryRepObjectBlockUserSpaces(ctx *changeRedundancy
return right.Count - left.Count
})
ids := lo.Map(userspaces, func(item *stgBlocks, idx int) jcsypes.UserSpaceID { return item.UserSpaceID })
ids := lo.Map(userspaces, func(item *stgBlocks, idx int) jcstypes.UserSpaceID { return item.UserSpaceID })
if len(ids) > userspaceCnt {
ids = ids[:userspaceCnt]
}
return ids
}
func (t *ChangeRedundancy) chooseNewUserSpacesForRep(ctx *changeRedundancyContext, red *jcsypes.RepRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) chooseNewUserSpacesForRep(ctx *changeRedundancyContext, red *jcstypes.RepRedundancy) []*userSpaceUsageInfo {
sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int {
return sort2.Cmp(right.AccessAmount, left.AccessAmount)
})
@ -174,7 +173,7 @@ func (t *ChangeRedundancy) chooseNewUserSpacesForRep(ctx *changeRedundancyContex
return t.chooseSoManyUserSpaces(red.RepCount, sortedUserSpaces)
}
func (t *ChangeRedundancy) chooseNewUserSpacesForEC(ctx *changeRedundancyContext, red *jcsypes.ECRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) chooseNewUserSpacesForEC(ctx *changeRedundancyContext, red *jcstypes.ECRedundancy) []*userSpaceUsageInfo {
sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int {
return sort2.Cmp(right.AccessAmount, left.AccessAmount)
})
@ -182,7 +181,7 @@ func (t *ChangeRedundancy) chooseNewUserSpacesForEC(ctx *changeRedundancyContext
return t.chooseSoManyUserSpaces(red.N, sortedUserSpaces)
}
func (t *ChangeRedundancy) chooseNewUserSpacesForLRC(ctx *changeRedundancyContext, red *jcsypes.LRCRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) chooseNewUserSpacesForLRC(ctx *changeRedundancyContext, red *jcstypes.LRCRedundancy) []*userSpaceUsageInfo {
sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int {
return sort2.Cmp(right.AccessAmount, left.AccessAmount)
})
@ -198,7 +197,7 @@ func (t *ChangeRedundancy) chooseNewUserSpacesForSeg(ctx *changeRedundancyContex
return t.chooseSoManyUserSpaces(segCount, sortedUserSpaces)
}
func (t *ChangeRedundancy) rechooseUserSpacesForRep(ctx *changeRedundancyContext, red *jcsypes.RepRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) rechooseUserSpacesForRep(ctx *changeRedundancyContext, red *jcstypes.RepRedundancy) []*userSpaceUsageInfo {
type rechooseUserSpace struct {
*userSpaceUsageInfo
HasBlock bool
@ -233,7 +232,7 @@ func (t *ChangeRedundancy) rechooseUserSpacesForRep(ctx *changeRedundancyContext
return t.chooseSoManyUserSpaces(red.RepCount, lo.Map(sortedStgs, func(userspace *rechooseUserSpace, idx int) *userSpaceUsageInfo { return userspace.userSpaceUsageInfo }))
}
func (t *ChangeRedundancy) rechooseUserSpacesForEC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.ECRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) rechooseUserSpacesForEC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.ECRedundancy) []*userSpaceUsageInfo {
type rechooseStg struct {
*userSpaceUsageInfo
CachedBlockIndex int
@ -269,7 +268,7 @@ func (t *ChangeRedundancy) rechooseUserSpacesForEC(ctx *changeRedundancyContext,
return t.chooseSoManyUserSpaces(red.N, lo.Map(sortedStgs, func(userspace *rechooseStg, idx int) *userSpaceUsageInfo { return userspace.userSpaceUsageInfo }))
}
func (t *ChangeRedundancy) rechooseUserSpacesForLRC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.LRCRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) rechooseUserSpacesForLRC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.LRCRedundancy) []*userSpaceUsageInfo {
type rechooseStg struct {
*userSpaceUsageInfo
CachedBlockIndex int
@ -322,7 +321,7 @@ func (t *ChangeRedundancy) chooseSoManyUserSpaces(count int, stgs []*userSpaceUs
var chosen []*userSpaceUsageInfo
for len(chosen) < count {
// 在每一轮内都选不同地区的节点，如果节点数不够，那么就再来一轮
chosenLocations := make(map[cortypes.Location]bool)
chosenLocations := make(map[jcstypes.Location]bool)
for i, stg := range extendStgs {
if stg == nil {
continue
@ -341,7 +340,7 @@ func (t *ChangeRedundancy) chooseSoManyUserSpaces(count int, stgs []*userSpaceUs
return chosen
}
func (t *ChangeRedundancy) noneToRep(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) noneToRep(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
if len(obj.Blocks) == 0 {
return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep")
}
@ -352,7 +351,7 @@ func (t *ChangeRedundancy) noneToRep(ctx *changeRedundancyContext, obj jcsypes.O
}
// 如果选择的备份节点都是同一个，那么就只要上传一次
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcsypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcstypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
ft := ioswitch2.NewFromTo()
ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.UserSpace, ioswitch2.RawStream()))
@ -374,11 +373,11 @@ func (t *ChangeRedundancy) noneToRep(ctx *changeRedundancyContext, obj jcsypes.O
return nil, nil, fmt.Errorf("executing io plan: %w", err)
}
var blocks []jcsypes.ObjectBlock
var blocks []jcstypes.ObjectBlock
var blockChgs []datamap.BlockChange
for i, stg := range uploadStgs {
r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, jcsypes.ObjectBlock{
blocks = append(blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: 0,
UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
@ -412,7 +411,7 @@ func (t *ChangeRedundancy) noneToRep(ctx *changeRedundancyContext, obj jcsypes.O
}, nil
}
func (t *ChangeRedundancy) noneToEC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.ECRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) noneToEC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.ECRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
if len(obj.Blocks) == 0 {
return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to ec")
}
@ -441,12 +440,12 @@ func (t *ChangeRedundancy) noneToEC(ctx *changeRedundancyContext, obj jcsypes.Ob
return nil, nil, fmt.Errorf("executing io plan: %w", err)
}
var blocks []jcsypes.ObjectBlock
var blocks []jcstypes.ObjectBlock
var evtTargetBlocks []datamap.Block
var evtBlockTrans []datamap.DataTransfer
for i := 0; i < red.N; i++ {
r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, jcsypes.ObjectBlock{
blocks = append(blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: i,
UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
@ -494,7 +493,7 @@ func (t *ChangeRedundancy) noneToEC(ctx *changeRedundancyContext, obj jcsypes.Ob
}, nil
}
func (t *ChangeRedundancy) noneToLRC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.LRCRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) noneToLRC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.LRCRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
if len(obj.Blocks) == 0 {
return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to ec")
}
@ -522,12 +521,12 @@ func (t *ChangeRedundancy) noneToLRC(ctx *changeRedundancyContext, obj jcsypes.O
return nil, nil, fmt.Errorf("executing io plan: %w", err)
}
var blocks []jcsypes.ObjectBlock
var blocks []jcstypes.ObjectBlock
var evtTargetBlocks []datamap.Block
var evtBlockTrans []datamap.DataTransfer
for i := 0; i < red.N; i++ {
r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, jcsypes.ObjectBlock{
blocks = append(blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: i,
UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
@ -576,7 +575,7 @@ func (t *ChangeRedundancy) noneToLRC(ctx *changeRedundancyContext, obj jcsypes.O
nil
}
func (t *ChangeRedundancy) noneToSeg(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.SegmentRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) noneToSeg(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.SegmentRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
if len(obj.Blocks) == 0 {
return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep")
}
@ -587,7 +586,7 @@ func (t *ChangeRedundancy) noneToSeg(ctx *changeRedundancyContext, obj jcsypes.O
}
// 如果选择的备份节点都是同一个，那么就只要上传一次
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcsypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcstypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
ft := ioswitch2.NewFromTo()
ft.SegmentParam = red
@ -610,12 +609,12 @@ func (t *ChangeRedundancy) noneToSeg(ctx *changeRedundancyContext, obj jcsypes.O
return nil, nil, fmt.Errorf("executing io plan: %w", err)
}
var blocks []jcsypes.ObjectBlock
var blocks []jcstypes.ObjectBlock
var evtTargetBlocks []datamap.Block
var evtBlockTrans []datamap.DataTransfer
for i, stg := range uploadStgs {
r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, jcsypes.ObjectBlock{
blocks = append(blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: i,
UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
@ -664,7 +663,7 @@ func (t *ChangeRedundancy) noneToSeg(ctx *changeRedundancyContext, obj jcsypes.O
nil
}
func (t *ChangeRedundancy) repToRep(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) repToRep(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
if len(obj.Blocks) == 0 {
return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep")
}
@ -675,7 +674,7 @@ func (t *ChangeRedundancy) repToRep(ctx *changeRedundancyContext, obj jcsypes.Ob
}
// 如果选择的备份节点都是同一个，那么就只要上传一次
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcsypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcstypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
ft := ioswitch2.NewFromTo()
ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.UserSpace, ioswitch2.RawStream()))
@ -697,11 +696,11 @@ func (t *ChangeRedundancy) repToRep(ctx *changeRedundancyContext, obj jcsypes.Ob
return nil, nil, fmt.Errorf("executing io plan: %w", err)
}
var blocks []jcsypes.ObjectBlock
var blocks []jcstypes.ObjectBlock
var blockChgs []datamap.BlockChange
for i, stg := range uploadStgs {
r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, jcsypes.ObjectBlock{
blocks = append(blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: 0,
UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
@ -737,14 +736,14 @@ func (t *ChangeRedundancy) repToRep(ctx *changeRedundancyContext, obj jcsypes.Ob
nil
}
func (t *ChangeRedundancy) repToEC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.ECRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) repToEC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.ECRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
return t.noneToEC(ctx, obj, red, uploadUserSpaces)
}
func (t *ChangeRedundancy) ecToRep(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, srcRed *jcsypes.ECRedundancy, tarRed *jcsypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
var chosenBlocks []jcsypes.GrouppedObjectBlock
func (t *ChangeRedundancy) ecToRep(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, srcRed *jcstypes.ECRedundancy, tarRed *jcstypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
var chosenBlocks []jcstypes.GrouppedObjectBlock
var chosenBlockIndexes []int
var chosenBlockStg []jcsypes.UserSpaceDetail
var chosenBlockStg []jcstypes.UserSpaceDetail
for _, block := range obj.GroupBlocks() {
if len(block.UserSpaceIDs) > 0 {
// TODO 考虑选择最优的节点
@ -768,7 +767,7 @@ func (t *ChangeRedundancy) ecToRep(ctx *changeRedundancyContext, obj jcsypes.Obj
}
// 如果选择的备份节点都是同一个，那么就只要上传一次
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcsypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcstypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
planBlder := exec.NewPlanBuilder()
ft := ioswitch2.NewFromTo()
@ -795,11 +794,11 @@ func (t *ChangeRedundancy) ecToRep(ctx *changeRedundancyContext, obj jcsypes.Obj
return nil, nil, fmt.Errorf("executing io plan: %w", err)
}
var blocks []jcsypes.ObjectBlock
var blocks []jcstypes.ObjectBlock
for i := range uploadStgs {
r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, jcsypes.ObjectBlock{
blocks = append(blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: 0,
UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
@ -866,11 +865,11 @@ func (t *ChangeRedundancy) ecToRep(ctx *changeRedundancyContext, obj jcsypes.Obj
nil
}
func (t *ChangeRedundancy) ecToEC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, srcRed *jcsypes.ECRedundancy, tarRed *jcsypes.ECRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) ecToEC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, srcRed *jcstypes.ECRedundancy, tarRed *jcstypes.ECRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
grpBlocks := obj.GroupBlocks()
var chosenBlocks []jcsypes.GrouppedObjectBlock
var chosenBlockStg []jcsypes.UserSpaceDetail
var chosenBlocks []jcstypes.GrouppedObjectBlock
var chosenBlockStg []jcstypes.UserSpaceDetail
for _, block := range grpBlocks {
if len(block.UserSpaceIDs) > 0 {
stg, ok := ctx.allUserSpaces[block.UserSpaceIDs[0]]
@ -909,16 +908,16 @@ func (t *ChangeRedundancy) ecToEC(ctx *changeRedundancyContext, obj jcsypes.Obje
})
}
var newBlocks []jcsypes.ObjectBlock
var newBlocks []jcstypes.ObjectBlock
shouldUpdateBlocks := false
for i, stg := range uploadUserSpaces {
newBlock := jcsypes.ObjectBlock{
newBlock := jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: i,
UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
}
grp, ok := lo.Find(grpBlocks, func(grp jcsypes.GrouppedObjectBlock) bool { return grp.Index == i })
grp, ok := lo.Find(grpBlocks, func(grp jcstypes.GrouppedObjectBlock) bool { return grp.Index == i })
// 如果新选中的节点已经记录在Block表中，那么就不需要任何变更
if ok && lo.Contains(grp.UserSpaceIDs, stg.UserSpace.UserSpace.UserSpaceID) {
@ -988,7 +987,7 @@ func (t *ChangeRedundancy) ecToEC(ctx *changeRedundancyContext, obj jcsypes.Obje
var blockChgs []datamap.BlockChange
for _, block := range obj.Blocks {
keep := lo.ContainsBy(newBlocks, func(newBlock jcsypes.ObjectBlock) bool {
keep := lo.ContainsBy(newBlocks, func(newBlock jcstypes.ObjectBlock) bool {
return newBlock.Index == block.Index && newBlock.UserSpaceID == block.UserSpaceID
})
if !keep {
@ -1019,7 +1018,7 @@ func (t *ChangeRedundancy) ecToEC(ctx *changeRedundancyContext, obj jcsypes.Obje
nil
}
func (t *ChangeRedundancy) lrcToLRC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, srcRed *jcsypes.LRCRedundancy, tarRed *jcsypes.LRCRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) lrcToLRC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, srcRed *jcstypes.LRCRedundancy, tarRed *jcstypes.LRCRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
blocksGrpByIndex := obj.GroupBlocks()
@ -1062,8 +1061,8 @@ func (t *ChangeRedundancy) lrcToLRC(ctx *changeRedundancyContext, obj jcsypes.Ob
/*
TODO2 修复这一块的代码
func (t *ChangeRedundancy) groupReconstructLRC(obj jcsypes.ObjectDetail, lostBlocks []int, lostBlockGrps []int, grpedBlocks []jcsypes.GrouppedObjectBlock, red *jcsypes.LRCRedundancy, uploadUserSpaces []*UserSpaceLoadInfo) (*db.UpdatingObjectRedundancy, error) {
grped := make(map[int]jcsypes.GrouppedObjectBlock)
func (t *ChangeRedundancy) groupReconstructLRC(obj jcstypes.ObjectDetail, lostBlocks []int, lostBlockGrps []int, grpedBlocks []jcstypes.GrouppedObjectBlock, red *jcstypes.LRCRedundancy, uploadUserSpaces []*UserSpaceLoadInfo) (*db.UpdatingObjectRedundancy, error) {
grped := make(map[int]jcstypes.GrouppedObjectBlock)
for _, b := range grpedBlocks {
grped[b.Index] = b
}
@ -1098,9 +1097,9 @@ TODO2 修复这一块的代码
return nil, fmt.Errorf("executing io plan: %w", err)
}
var newBlocks []jcsypes.ObjectBlock
var newBlocks []jcstypes.ObjectBlock
for _, i := range lostBlocks {
newBlocks = append(newBlocks, jcsypes.ObjectBlock{
newBlocks = append(newBlocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: i,
UserSpaceID: uploadUserSpaces[i].UserSpace.UserSpace.UserSpaceID,
@ -1109,7 +1108,7 @@ TODO2 修复这一块的代码
}
for _, b := range grpedBlocks {
for _, hubID := range b.UserSpaceIDs {
newBlocks = append(newBlocks, jcsypes.ObjectBlock{
newBlocks = append(newBlocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: b.Index,
UserSpaceID: hubID,
@ -1125,9 +1124,9 @@ TODO2 修复这一块的代码
}, nil
}
*/
func (t *ChangeRedundancy) reconstructLRC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, grpBlocks []jcsypes.GrouppedObjectBlock, red *jcsypes.LRCRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
var chosenBlocks []jcsypes.GrouppedObjectBlock
var chosenBlockStg []jcsypes.UserSpaceDetail
func (t *ChangeRedundancy) reconstructLRC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, grpBlocks []jcstypes.GrouppedObjectBlock, red *jcstypes.LRCRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
var chosenBlocks []jcstypes.GrouppedObjectBlock
var chosenBlockStg []jcstypes.UserSpaceDetail
for _, block := range grpBlocks {
if len(block.UserSpaceIDs) > 0 && block.Index < red.M() {
@ -1154,16 +1153,16 @@ func (t *ChangeRedundancy) reconstructLRC(ctx *changeRedundancyContext, obj jcsy
var froms []ioswitchlrc.From
var toes []ioswitchlrc.To
var newBlocks []jcsypes.ObjectBlock
var newBlocks []jcstypes.ObjectBlock
shouldUpdateBlocks := false
for i, userspace := range uploadUserSpaces {
newBlock := jcsypes.ObjectBlock{
newBlock := jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: i,
UserSpaceID: userspace.UserSpace.UserSpace.UserSpaceID,
}
grp, ok := lo.Find(grpBlocks, func(grp jcsypes.GrouppedObjectBlock) bool { return grp.Index == i })
grp, ok := lo.Find(grpBlocks, func(grp jcstypes.GrouppedObjectBlock) bool { return grp.Index == i })
// 如果新选中的节点已经记录在Block表中，那么就不需要任何变更
if ok && lo.Contains(grp.UserSpaceIDs, userspace.UserSpace.UserSpace.UserSpaceID) {

104
client/internal/ticktock/redundancy_shrink.go
View File

@ -21,14 +21,14 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
)
func (t *ChangeRedundancy) doRedundancyShrink(execCtx *changeRedundancyContext, pkg jcsypes.PackageDetail, objs []jcsypes.ObjectDetail, reen *publock.Reentrant) ([]db.UpdatingObjectRedundancy, []datamap.SysEventBody, error) {
func (t *ChangeRedundancy) doRedundancyShrink(execCtx *changeRedundancyContext, pkg jcstypes.PackageDetail, objs []jcstypes.ObjectDetail, reen *publock.Reentrant) ([]db.UpdatingObjectRedundancy, []datamap.SysEventBody, error) {
log := logger.WithType[ChangeRedundancy]("TickTock")
var readerStgIDs []jcsypes.UserSpaceID
var readerStgIDs []jcstypes.UserSpaceID
for _, space := range execCtx.allUserSpaces {
// TODO 可以考虑做成配置
if space.AccessAmount >= float64(pkg.ObjectCount/2) {
@ -37,17 +37,17 @@ func (t *ChangeRedundancy) doRedundancyShrink(execCtx *changeRedundancyContext,
}
// 只对ec和rep对象进行处理
var ecObjects []jcsypes.ObjectDetail
var repObjects []jcsypes.ObjectDetail
var ecObjects []jcstypes.ObjectDetail
var repObjects []jcstypes.ObjectDetail
for _, obj := range objs {
if _, ok := obj.Object.Redundancy.(*jcsypes.ECRedundancy); ok {
if _, ok := obj.Object.Redundancy.(*jcstypes.ECRedundancy); ok {
ecObjects = append(ecObjects, obj)
} else if _, ok := obj.Object.Redundancy.(*jcsypes.RepRedundancy); ok {
} else if _, ok := obj.Object.Redundancy.(*jcstypes.RepRedundancy); ok {
repObjects = append(repObjects, obj)
}
}
planningStgIDs := make(map[jcsypes.UserSpaceID]bool)
planningStgIDs := make(map[jcstypes.UserSpaceID]bool)
var sysEvents []datamap.SysEventBody
@ -80,7 +80,7 @@ func (t *ChangeRedundancy) doRedundancyShrink(execCtx *changeRedundancyContext,
// 对于ec对象，则每个对象单独进行退火算法
var ecObjectsUpdating []db.UpdatingObjectRedundancy
for i, obj := range ecObjects {
ecRed := obj.Object.Redundancy.(*jcsypes.ECRedundancy)
ecRed := obj.Object.Redundancy.(*jcstypes.ECRedundancy)
solu := t.startAnnealing(execCtx, readerStgIDs, annealingObject{
totalBlockCount: ecRed.N,
minBlockCnt: ecRed.K,
@ -105,15 +105,15 @@ func (t *ChangeRedundancy) doRedundancyShrink(execCtx *changeRedundancyContext,
return append(repObjectsUpdating, ecObjectsUpdating...), sysEvents, nil
}
func (t *ChangeRedundancy) summaryRepObjectBlockNodes(objs []jcsypes.ObjectDetail) []jcsypes.UserSpaceID {
func (t *ChangeRedundancy) summaryRepObjectBlockNodes(objs []jcstypes.ObjectDetail) []jcstypes.UserSpaceID {
type stgBlocks struct {
UserSpaceID jcsypes.UserSpaceID
UserSpaceID jcstypes.UserSpaceID
Count int
}
stgBlocksMap := make(map[jcsypes.UserSpaceID]*stgBlocks)
stgBlocksMap := make(map[jcstypes.UserSpaceID]*stgBlocks)
for _, obj := range objs {
cacheBlockStgs := make(map[jcsypes.UserSpaceID]bool)
cacheBlockStgs := make(map[jcstypes.UserSpaceID]bool)
for _, block := range obj.Blocks {
if _, ok := stgBlocksMap[block.UserSpaceID]; !ok {
stgBlocksMap[block.UserSpaceID] = &stgBlocks{
@ -153,17 +153,17 @@ func (t *ChangeRedundancy) summaryRepObjectBlockNodes(objs []jcsypes.ObjectDetai
}
}
return lo.Map(stgs, func(item *stgBlocks, idx int) jcsypes.UserSpaceID { return item.UserSpaceID })
return lo.Map(stgs, func(item *stgBlocks, idx int) jcstypes.UserSpaceID { return item.UserSpaceID })
}
type annealingState struct {
ctx *changeRedundancyContext
readerStgIDs []jcsypes.UserSpaceID // 近期可能访问此对象的节点
stgsSortedByReader map[jcsypes.UserSpaceID][]stgDist // 拥有数据的节点到每个可能访问对象的节点按距离排序
object annealingObject // 进行退火的对象
blockList []objectBlock // 排序后的块分布情况
stgBlockBitmaps map[jcsypes.UserSpaceID]*bitmap.Bitmap64 // 用位图的形式表示每一个节点上有哪些块
stgCombTree combinatorialTree // 节点组合树，用于加速计算容灾度
readerStgIDs []jcstypes.UserSpaceID // 近期可能访问此对象的节点
stgsSortedByReader map[jcstypes.UserSpaceID][]stgDist // 拥有数据的节点到每个可能访问对象的节点按距离排序
object annealingObject // 进行退火的对象
blockList []objectBlock // 排序后的块分布情况
stgBlockBitmaps map[jcstypes.UserSpaceID]*bitmap.Bitmap64 // 用位图的形式表示每一个节点上有哪些块
stgCombTree combinatorialTree // 节点组合树，用于加速计算容灾度
maxScore float64 // 搜索过程中得到过的最大分数
maxScoreRmBlocks []bool // 最大分数对应的删除方案
@ -178,30 +178,30 @@ type annealingState struct {
type objectBlock struct {
Index int
UserSpaceID jcsypes.UserSpaceID
HasEntity bool // 节点拥有实际的文件数据块
HasShadow bool // 如果节点拥有完整文件数据，那么认为这个节点拥有所有块，这些块被称为影子块
FileHash jcsypes.FileHash // 只有在拥有实际文件数据块时，这个字段才有值
Size int64 // 块大小
UserSpaceID jcstypes.UserSpaceID
HasEntity bool // 节点拥有实际的文件数据块
HasShadow bool // 如果节点拥有完整文件数据，那么认为这个节点拥有所有块，这些块被称为影子块
FileHash jcstypes.FileHash // 只有在拥有实际文件数据块时，这个字段才有值
Size int64 // 块大小
}
type stgDist struct {
UserSpaceID jcsypes.UserSpaceID
UserSpaceID jcstypes.UserSpaceID
Distance float64
}
type combinatorialTree struct {
nodes []combinatorialTreeNode
blocksMaps map[int]bitmap.Bitmap64
stgIDToLocalStgID map[jcsypes.UserSpaceID]int
localStgIDToStgID []jcsypes.UserSpaceID
stgIDToLocalStgID map[jcstypes.UserSpaceID]int
localStgIDToStgID []jcstypes.UserSpaceID
}
type annealingObject struct {
totalBlockCount int
minBlockCnt int
pinnedAt []jcsypes.UserSpaceID
blocks []jcsypes.ObjectBlock
pinnedAt []jcstypes.UserSpaceID
blocks []jcstypes.ObjectBlock
}
const (
@ -210,10 +210,10 @@ const (
iterActionBreak = 2
)
func newCombinatorialTree(stgBlocksMaps map[jcsypes.UserSpaceID]*bitmap.Bitmap64) combinatorialTree {
func newCombinatorialTree(stgBlocksMaps map[jcstypes.UserSpaceID]*bitmap.Bitmap64) combinatorialTree {
tree := combinatorialTree{
blocksMaps: make(map[int]bitmap.Bitmap64),
stgIDToLocalStgID: make(map[jcsypes.UserSpaceID]int),
stgIDToLocalStgID: make(map[jcstypes.UserSpaceID]int),
}
tree.nodes = make([]combinatorialTreeNode, (1 << len(stgBlocksMaps)))
@ -271,7 +271,7 @@ func (t *combinatorialTree) GetDepth(index int) int {
// 更新某一个算力中心节点的块分布位图，同时更新它对应组合树节点的所有子节点。
// 如果更新到某个节点时，已有K个块，那么就不会再更新它的子节点
func (t *combinatorialTree) UpdateBitmap(stgID jcsypes.UserSpaceID, mp bitmap.Bitmap64, k int) {
func (t *combinatorialTree) UpdateBitmap(stgID jcstypes.UserSpaceID, mp bitmap.Bitmap64, k int) {
t.blocksMaps[t.stgIDToLocalStgID[stgID]] = mp
// 首先定义两种遍历树节点时的移动方式：
// 1. 竖直移动（深度增加）：从一个节点移动到它最左边的子节点。每移动一步，index+1
@ -416,13 +416,13 @@ type annealingSolution struct {
minAccessCost float64 // 本方案的最小访问费用
}
func (t *ChangeRedundancy) startAnnealing(ctx *changeRedundancyContext, readerStgIDs []jcsypes.UserSpaceID, object annealingObject) annealingSolution {
func (t *ChangeRedundancy) startAnnealing(ctx *changeRedundancyContext, readerStgIDs []jcstypes.UserSpaceID, object annealingObject) annealingSolution {
state := &annealingState{
ctx: ctx,
readerStgIDs: readerStgIDs,
stgsSortedByReader: make(map[jcsypes.UserSpaceID][]stgDist),
stgsSortedByReader: make(map[jcstypes.UserSpaceID][]stgDist),
object: object,
stgBlockBitmaps: make(map[jcsypes.UserSpaceID]*bitmap.Bitmap64),
stgBlockBitmaps: make(map[jcstypes.UserSpaceID]*bitmap.Bitmap64),
}
t.initBlockList(state)
@ -486,7 +486,7 @@ func (t *ChangeRedundancy) startAnnealing(ctx *changeRedundancyContext, readerSt
}
func (t *ChangeRedundancy) initBlockList(ctx *annealingState) {
blocksMap := make(map[jcsypes.UserSpaceID][]objectBlock)
blocksMap := make(map[jcstypes.UserSpaceID][]objectBlock)
// 先生成所有的影子块
for _, pinned := range ctx.object.pinnedAt {
@ -680,7 +680,7 @@ func (t *ChangeRedundancy) alwaysAccept(curTemp float64, dScore float64, cooling
return v > rand.Float64()
}
func (t *ChangeRedundancy) makePlansForRepObject(ctx *changeRedundancyContext, solu annealingSolution, obj jcsypes.ObjectDetail, planBld *exec.PlanBuilder, planningHubIDs map[jcsypes.UserSpaceID]bool) db.UpdatingObjectRedundancy {
func (t *ChangeRedundancy) makePlansForRepObject(ctx *changeRedundancyContext, solu annealingSolution, obj jcstypes.ObjectDetail, planBld *exec.PlanBuilder, planningHubIDs map[jcstypes.UserSpaceID]bool) db.UpdatingObjectRedundancy {
entry := db.UpdatingObjectRedundancy{
ObjectID: obj.Object.ObjectID,
FileHash: obj.Object.FileHash,
@ -694,8 +694,8 @@ func (t *ChangeRedundancy) makePlansForRepObject(ctx *changeRedundancyContext, s
ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *fromStg, ioswitch2.RawStream()))
for i, f := range solu.rmBlocks {
hasCache := lo.ContainsBy(obj.Blocks, func(b jcsypes.ObjectBlock) bool { return b.UserSpaceID == solu.blockList[i].UserSpaceID }) ||
lo.ContainsBy(obj.PinnedAt, func(n jcsypes.UserSpaceID) bool { return n == solu.blockList[i].UserSpaceID })
hasCache := lo.ContainsBy(obj.Blocks, func(b jcstypes.ObjectBlock) bool { return b.UserSpaceID == solu.blockList[i].UserSpaceID }) ||
lo.ContainsBy(obj.PinnedAt, func(n jcstypes.UserSpaceID) bool { return n == solu.blockList[i].UserSpaceID })
willRm := f
if !willRm {
@ -706,7 +706,7 @@ func (t *ChangeRedundancy) makePlansForRepObject(ctx *changeRedundancyContext, s
planningHubIDs[solu.blockList[i].UserSpaceID] = true
}
entry.Blocks = append(entry.Blocks, jcsypes.ObjectBlock{
entry.Blocks = append(entry.Blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: solu.blockList[i].Index,
UserSpaceID: solu.blockList[i].UserSpaceID,
@ -724,12 +724,12 @@ func (t *ChangeRedundancy) makePlansForRepObject(ctx *changeRedundancyContext, s
return entry
}
func (t *ChangeRedundancy) generateSysEventForRepObject(solu annealingSolution, obj jcsypes.ObjectDetail) []datamap.SysEventBody {
func (t *ChangeRedundancy) generateSysEventForRepObject(solu annealingSolution, obj jcstypes.ObjectDetail) []datamap.SysEventBody {
var blockChgs []datamap.BlockChange
for i, f := range solu.rmBlocks {
hasCache := lo.ContainsBy(obj.Blocks, func(b jcsypes.ObjectBlock) bool { return b.UserSpaceID == solu.blockList[i].UserSpaceID }) ||
lo.ContainsBy(obj.PinnedAt, func(n jcsypes.UserSpaceID) bool { return n == solu.blockList[i].UserSpaceID })
hasCache := lo.ContainsBy(obj.Blocks, func(b jcstypes.ObjectBlock) bool { return b.UserSpaceID == solu.blockList[i].UserSpaceID }) ||
lo.ContainsBy(obj.PinnedAt, func(n jcstypes.UserSpaceID) bool { return n == solu.blockList[i].UserSpaceID })
willRm := f
if !willRm {
@ -782,7 +782,7 @@ func (t *ChangeRedundancy) generateSysEventForRepObject(solu annealingSolution,
return []datamap.SysEventBody{transEvt, distEvt}
}
func (t *ChangeRedundancy) makePlansForECObject(ctx *changeRedundancyContext, solu annealingSolution, obj jcsypes.ObjectDetail, planBld *exec.PlanBuilder, planningHubIDs map[jcsypes.UserSpaceID]bool) db.UpdatingObjectRedundancy {
func (t *ChangeRedundancy) makePlansForECObject(ctx *changeRedundancyContext, solu annealingSolution, obj jcstypes.ObjectDetail, planBld *exec.PlanBuilder, planningHubIDs map[jcstypes.UserSpaceID]bool) db.UpdatingObjectRedundancy {
entry := db.UpdatingObjectRedundancy{
ObjectID: obj.Object.ObjectID,
FileHash: obj.Object.FileHash,
@ -790,11 +790,11 @@ func (t *ChangeRedundancy) makePlansForECObject(ctx *changeRedundancyContext, so
Redundancy: obj.Object.Redundancy,
}
reconstrct := make(map[jcsypes.UserSpaceID]*[]int)
reconstrct := make(map[jcstypes.UserSpaceID]*[]int)
for i, f := range solu.rmBlocks {
block := solu.blockList[i]
if !f {
entry.Blocks = append(entry.Blocks, jcsypes.ObjectBlock{
entry.Blocks = append(entry.Blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: block.Index,
UserSpaceID: block.UserSpaceID,
@ -815,7 +815,7 @@ func (t *ChangeRedundancy) makePlansForECObject(ctx *changeRedundancyContext, so
}
}
ecRed := obj.Object.Redundancy.(*jcsypes.ECRedundancy)
ecRed := obj.Object.Redundancy.(*jcstypes.ECRedundancy)
for id, idxs := range reconstrct {
// 依次生成每个节点上的执行计划，因为如果放到一个计划里一起生成，不能保证每个节点上的块用的都是本节点上的副本
@ -838,10 +838,10 @@ func (t *ChangeRedundancy) makePlansForECObject(ctx *changeRedundancyContext, so
return entry
}
func (t *ChangeRedundancy) generateSysEventForECObject(solu annealingSolution, obj jcsypes.ObjectDetail) []datamap.SysEventBody {
func (t *ChangeRedundancy) generateSysEventForECObject(solu annealingSolution, obj jcstypes.ObjectDetail) []datamap.SysEventBody {
var blockChgs []datamap.BlockChange
reconstrct := make(map[jcsypes.UserSpaceID]*[]int)
reconstrct := make(map[jcstypes.UserSpaceID]*[]int)
for i, f := range solu.rmBlocks {
block := solu.blockList[i]
if !f {
@ -917,7 +917,7 @@ func (t *ChangeRedundancy) generateSysEventForECObject(solu annealingSolution, o
return []datamap.SysEventBody{transEvt, distEvt}
}
func (t *ChangeRedundancy) executePlans(ctx *changeRedundancyContext, planBld *exec.PlanBuilder, planningSpaceIDs map[jcsypes.UserSpaceID]bool, reen *publock.Reentrant) (exec.PlanResult, error) {
func (t *ChangeRedundancy) executePlans(ctx *changeRedundancyContext, planBld *exec.PlanBuilder, planningSpaceIDs map[jcstypes.UserSpaceID]bool, reen *publock.Reentrant) (exec.PlanResult, error) {
reqBlder := reqbuilder.NewBuilder()
for id, _ := range planningSpaceIDs {
reqBlder.UserSpace().Buzy(id)
@ -955,7 +955,7 @@ func (t *ChangeRedundancy) executePlans(ctx *changeRedundancyContext, planBld *e
return ioSwRets, nil
}
func (t *ChangeRedundancy) populateECObjectEntry(entry *db.UpdatingObjectRedundancy, obj jcsypes.ObjectDetail, ioRets exec.PlanResult) {
func (t *ChangeRedundancy) populateECObjectEntry(entry *db.UpdatingObjectRedundancy, obj jcstypes.ObjectDetail, ioRets exec.PlanResult) {
for i := range entry.Blocks {
if entry.Blocks[i].FileHash != "" {
continue

26
client/internal/uploader/uploader.go
View File

@ -22,7 +22,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type Uploader struct {
@ -43,7 +43,7 @@ func NewUploader(pubLock *publock.Service, connectivity *connectivity.Collector,
}
}
func (u *Uploader) BeginUpdate(pkgID jcsypes.PackageID, affinity jcsypes.UserSpaceID, copyTo []jcsypes.UserSpaceID, copyToPath []jcsypes.JPath) (*UpdateUploader, error) {
func (u *Uploader) BeginUpdate(pkgID jcstypes.PackageID, affinity jcstypes.UserSpaceID, copyTo []jcstypes.UserSpaceID, copyToPath []jcstypes.JPath) (*UpdateUploader, error) {
spaceIDs, err := u.db.UserSpace().GetAllIDs(u.db.DefCtx())
if err != nil {
return nil, fmt.Errorf("getting user space ids: %w", err)
@ -84,9 +84,9 @@ func (u *Uploader) BeginUpdate(pkgID jcsypes.PackageID, affinity jcsypes.UserSpa
return nil, fmt.Errorf("user no available userspaces")
}
copyToSpaces := make([]jcsypes.UserSpaceDetail, len(copyTo))
copyToSpaces := make([]jcstypes.UserSpaceDetail, len(copyTo))
for i, spaceID := range copyTo {
space, ok := lo.Find(spaceDetails, func(space *jcsypes.UserSpaceDetail) bool {
space, ok := lo.Find(spaceDetails, func(space *jcstypes.UserSpaceDetail) bool {
return space.UserSpace.UserSpaceID == spaceID
})
if !ok {
@ -118,7 +118,7 @@ func (u *Uploader) BeginUpdate(pkgID jcsypes.PackageID, affinity jcsypes.UserSpa
// 1. 选择设置了亲和性的节点
// 2. 从与当前客户端相同地域的节点中随机选一个
// 3. 没有的话从所有节点选择延迟最低的节点
func (w *Uploader) chooseUploadStorage(spaces []UploadSpaceInfo, spaceAffinity jcsypes.UserSpaceID) UploadSpaceInfo {
func (w *Uploader) chooseUploadStorage(spaces []UploadSpaceInfo, spaceAffinity jcstypes.UserSpaceID) UploadSpaceInfo {
if spaceAffinity > 0 {
aff, ok := lo.Find(spaces, func(space UploadSpaceInfo) bool { return space.Space.UserSpace.UserSpaceID == spaceAffinity })
if ok {
@ -137,10 +137,10 @@ func (w *Uploader) chooseUploadStorage(spaces []UploadSpaceInfo, spaceAffinity j
return spaces[0]
}
func (u *Uploader) BeginCreateUpload(bktID jcsypes.BucketID, pkgName string, copyTo []jcsypes.UserSpaceID, copyToPath []jcsypes.JPath) (*CreateUploader, error) {
func (u *Uploader) BeginCreateUpload(bktID jcstypes.BucketID, pkgName string, copyTo []jcstypes.UserSpaceID, copyToPath []jcstypes.JPath) (*CreateUploader, error) {
getSpaces := u.spaceMeta.GetMany(copyTo)
spacesStgs := make([]jcsypes.UserSpaceDetail, len(copyTo))
spacesStgs := make([]jcstypes.UserSpaceDetail, len(copyTo))
for i, stg := range getSpaces {
if stg == nil {
return nil, fmt.Errorf("storage %v not found", copyTo[i])
@ -148,10 +148,10 @@ func (u *Uploader) BeginCreateUpload(bktID jcsypes.BucketID, pkgName string, cop
spacesStgs[i] = *stg
}
pkg, err := db.DoTx01(u.db, func(tx db.SQLContext) (jcsypes.Package, error) {
pkg, err := db.DoTx01(u.db, func(tx db.SQLContext) (jcstypes.Package, error) {
_, err := u.db.Bucket().GetByID(tx, bktID)
if err != nil {
return jcsypes.Package{}, err
return jcstypes.Package{}, err
}
return u.db.Package().Create(u.db.DefCtx(), bktID, pkgName, time.Now())
@ -178,19 +178,19 @@ func (u *Uploader) BeginCreateUpload(bktID jcsypes.BucketID, pkgName string, cop
}, nil
}
func (u *Uploader) UploadPart(objID jcsypes.ObjectID, index int, stream io.Reader) error {
func (u *Uploader) UploadPart(objID jcstypes.ObjectID, index int, stream io.Reader) error {
detail, err := u.db.Object().GetDetail(u.db.DefCtx(), objID)
if err != nil {
return fmt.Errorf("getting object detail: %w", err)
}
objDe := detail
_, ok := objDe.Object.Redundancy.(*jcsypes.MultipartUploadRedundancy)
_, ok := objDe.Object.Redundancy.(*jcstypes.MultipartUploadRedundancy)
if !ok {
return fmt.Errorf("object %v is not a multipart upload", objID)
}
var space jcsypes.UserSpaceDetail
var space jcstypes.UserSpaceDetail
if len(objDe.Blocks) > 0 {
cstg := u.spaceMeta.Get(objDe.Blocks[0].UserSpaceID)
if cstg == nil {
@ -272,7 +272,7 @@ func (u *Uploader) UploadPart(objID jcsypes.ObjectID, index int, stream io.Reade
shardInfo := ret.Get("shard").(*ops2.FileInfoValue)
err = u.db.DoTx(func(tx db.SQLContext) error {
return u.db.Object().AppendPart(tx, jcsypes.ObjectBlock{
return u.db.Object().AppendPart(tx, jcstypes.ObjectBlock{
ObjectID: objID,
Index: index,
UserSpaceID: space.UserSpace.UserSpaceID,

19
client/internal/uploader/user_space_upload.go
View File

@ -17,20 +17,19 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder"
corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func (u *Uploader) UserSpaceUpload(userSpaceID jcsypes.UserSpaceID, rootPath jcsypes.JPath, targetBktID jcsypes.BucketID, newPkgName string, uploadAffinity jcsypes.UserSpaceID) (*jcsypes.Package, error) {
func (u *Uploader) UserSpaceUpload(userSpaceID jcstypes.UserSpaceID, rootPath jcstypes.JPath, targetBktID jcstypes.BucketID, newPkgName string, uploadAffinity jcstypes.UserSpaceID) (*jcstypes.Package, error) {
srcSpace := u.spaceMeta.Get(userSpaceID)
if srcSpace == nil {
return nil, fmt.Errorf("user space %d not found", userSpaceID)
}
pkg, err := db.DoTx01(u.db, func(tx db.SQLContext) (jcsypes.Package, error) {
pkg, err := db.DoTx01(u.db, func(tx db.SQLContext) (jcstypes.Package, error) {
_, err := u.db.Bucket().GetByID(tx, targetBktID)
if err != nil {
return jcsypes.Package{}, err
return jcstypes.Package{}, err
}
return u.db.Package().Create(tx, targetBktID, newPkgName, time.Now())
@ -49,7 +48,7 @@ func (u *Uploader) UserSpaceUpload(userSpaceID jcsypes.UserSpaceID, rootPath jcs
}
spaceDetails := u.spaceMeta.GetMany(spaceIDs)
spaceDetails = lo.Filter(spaceDetails, func(e *jcsypes.UserSpaceDetail, i int) bool {
spaceDetails = lo.Filter(spaceDetails, func(e *jcstypes.UserSpaceDetail, i int) bool {
return e != nil && e.UserSpace.ShardStore != nil
})
@ -59,13 +58,13 @@ func (u *Uploader) UserSpaceUpload(userSpaceID jcsypes.UserSpaceID, rootPath jcs
coorCli := stgglb.CoordinatorRPCPool.Get()
defer coorCli.Release()
resp, cerr := coorCli.GetHubConnectivities(context.Background(), corrpc.ReqGetHubConnectivities([]cortypes.HubID{srcSpace.RecommendHub.HubID}))
resp, cerr := coorCli.GetHubConnectivities(context.Background(), corrpc.ReqGetHubConnectivities([]jcstypes.HubID{srcSpace.RecommendHub.HubID}))
if cerr != nil {
delPkg()
return nil, fmt.Errorf("getting hub connectivities: %w", cerr.ToError())
}
cons := make(map[cortypes.HubID]cortypes.HubConnectivity)
cons := make(map[jcstypes.HubID]jcstypes.HubConnectivity)
for _, c := range resp.Connectivities {
cons[c.ToHubID] = c
}
@ -153,7 +152,7 @@ func (u *Uploader) UserSpaceUpload(userSpaceID jcsypes.UserSpaceID, rootPath jcs
return &pkg, nil
}
func (u *Uploader) uploadFromBaseStore(srcSpace *jcsypes.UserSpaceDetail, targetSpace *jcsypes.UserSpaceDetail, entries []types.DirEntry, rootPath jcsypes.JPath) ([]db.AddObjectEntry, error) {
func (u *Uploader) uploadFromBaseStore(srcSpace *jcstypes.UserSpaceDetail, targetSpace *jcstypes.UserSpaceDetail, entries []types.DirEntry, rootPath jcstypes.JPath) ([]db.AddObjectEntry, error) {
ft := ioswitch2.FromTo{}
for _, e := range entries {
@ -198,7 +197,7 @@ func (u *Uploader) uploadFromBaseStore(srcSpace *jcsypes.UserSpaceDetail, target
Size: info.Size,
FileHash: info.Hash,
CreateTime: time.Now(),
UserSpaceIDs: []jcsypes.UserSpaceID{targetSpace.UserSpace.UserSpaceID},
UserSpaceIDs: []jcstypes.UserSpaceID{targetSpace.UserSpace.UserSpaceID},
})
}

14
client/sdk/api/v1/bucket.go
View File

@ -4,7 +4,7 @@ import (
"net/http"
"gitlink.org.cn/cloudream/common/sdks"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type BucketService struct {
@ -18,7 +18,7 @@ func (c *Client) Bucket() *BucketService {
const BucketGetPath = "/bucket/get"
type BucketGet struct {
BucketID jcsypes.BucketID `json:"bucketID" binding:"required"`
BucketID jcstypes.BucketID `json:"bucketID" binding:"required"`
}
func (r *BucketGet) MakeParam() *sdks.RequestParam {
@ -26,7 +26,7 @@ func (r *BucketGet) MakeParam() *sdks.RequestParam {
}
type BucketGetResp struct {
Bucket jcsypes.Bucket `json:"bucket"`
Bucket jcstypes.Bucket `json:"bucket"`
}
func (r *BucketGetResp) ParseResponse(resp *http.Response) error {
@ -48,7 +48,7 @@ func (r *BucketGetByName) MakeParam() *sdks.RequestParam {
}
type BucketGetByNameResp struct {
Bucket jcsypes.Bucket `json:"bucket"`
Bucket jcstypes.Bucket `json:"bucket"`
}
func (r *BucketGetByNameResp) ParseResponse(resp *http.Response) error {
@ -70,7 +70,7 @@ func (r *BucketCreate) MakeParam() *sdks.RequestParam {
}
type BucketCreateResp struct {
Bucket jcsypes.Bucket `json:"bucket"`
Bucket jcstypes.Bucket `json:"bucket"`
}
func (r *BucketCreateResp) ParseResponse(resp *http.Response) error {
@ -84,7 +84,7 @@ func (c *BucketService) Create(req BucketCreate) (*BucketCreateResp, error) {
const BucketDeletePath = "/bucket/delete"
type BucketDelete struct {
BucketID jcsypes.BucketID `json:"bucketID" binding:"required"`
BucketID jcstypes.BucketID `json:"bucketID" binding:"required"`
}
func (r *BucketDelete) MakeParam() *sdks.RequestParam {
@ -111,7 +111,7 @@ func (r *BucketListAll) MakeParam() *sdks.RequestParam {
}
type BucketListAllResp struct {
Buckets []jcsypes.Bucket `json:"buckets"`
Buckets []jcstypes.Bucket `json:"buckets"`
}
func (r *BucketListAllResp) ParseResponse(resp *http.Response) error {

48
client/sdk/api/v1/package.go
View File

@ -13,7 +13,7 @@ import (
"gitlink.org.cn/cloudream/common/sdks"
"gitlink.org.cn/cloudream/common/utils/http2"
"gitlink.org.cn/cloudream/common/utils/serder"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type PackageService struct {
@ -27,7 +27,7 @@ func (c *Client) Package() *PackageService {
const PackageGetPath = "/package/get"
type PackageGet struct {
PackageID jcsypes.PackageID `form:"packageID" url:"packageID" binding:"required"`
PackageID jcstypes.PackageID `form:"packageID" url:"packageID" binding:"required"`
}
func (r *PackageGet) MakeParam() *sdks.RequestParam {
@ -35,7 +35,7 @@ func (r *PackageGet) MakeParam() *sdks.RequestParam {
}
type PackageGetResp struct {
Package jcsypes.Package `json:"package"`
Package jcstypes.Package `json:"package"`
}
func (r *PackageGetResp) ParseResponse(resp *http.Response) error {
@ -58,7 +58,7 @@ func (r *PackageGetByFullName) MakeParam() *sdks.RequestParam {
}
type PackageGetByFullNameResp struct {
Package jcsypes.Package `json:"package"`
Package jcstypes.Package `json:"package"`
}
func (r *PackageGetByFullNameResp) ParseResponse(resp *http.Response) error {
@ -72,8 +72,8 @@ func (c *PackageService) GetByFullName(req PackageGetByFullName) (*PackageGetByF
const PackageCreatePath = "/package/create"
type PackageCreate struct {
BucketID jcsypes.BucketID `json:"bucketID"`
Name string `json:"name"`
BucketID jcstypes.BucketID `json:"bucketID"`
Name string `json:"name"`
}
func (r *PackageCreate) MakeParam() *sdks.RequestParam {
@ -81,7 +81,7 @@ func (r *PackageCreate) MakeParam() *sdks.RequestParam {
}
type PackageCreateResp struct {
Package jcsypes.Package `json:"package"`
Package jcstypes.Package `json:"package"`
}
func (r *PackageCreateResp) ParseResponse(resp *http.Response) error {
@ -99,14 +99,14 @@ type PackageCreateUpload struct {
Files UploadObjectIterator `json:"-"`
}
type PackageCreateUploadInfo struct {
BucketID jcsypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
CopyTo []jcsypes.UserSpaceID `json:"copyTo"`
CopyToPath []string `json:"copyToPath"`
BucketID jcstypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
CopyTo []jcstypes.UserSpaceID `json:"copyTo"`
CopyToPath []string `json:"copyToPath"`
}
type PackageCreateUploadResp struct {
Package jcsypes.Package `json:"package"`
Objects []jcsypes.Object `json:"objects"`
Package jcstypes.Package `json:"package"`
Objects []jcstypes.Object `json:"objects"`
}
func (c *PackageService) CreateUpload(req PackageCreateUpload) (*PackageCreateUploadResp, error) {
@ -148,10 +148,10 @@ func (c *PackageService) CreateUpload(req PackageCreateUpload) (*PackageCreateUp
const PackageDownloadPath = "/package/download"
type PackageDownload struct {
PackageID jcsypes.PackageID `url:"packageID" form:"packageID" binding:"required"`
Prefix string `url:"prefix" form:"prefix"`
NewPrefix *string `url:"newPrefix,omitempty" form:"newPrefix"`
Zip bool `url:"zip,omitempty" form:"zip"`
PackageID jcstypes.PackageID `url:"packageID" form:"packageID" binding:"required"`
Prefix string `url:"prefix" form:"prefix"`
NewPrefix *string `url:"newPrefix,omitempty" form:"newPrefix"`
Zip bool `url:"zip,omitempty" form:"zip"`
}
func (r *PackageDownload) MakeParam() *sdks.RequestParam {
@ -208,7 +208,7 @@ func (c *PackageService) Download(req PackageDownload) (*DownloadingPackage, err
const PackageDeletePath = "/package/delete"
type PackageDelete struct {
PackageID jcsypes.PackageID `json:"packageID" binding:"required"`
PackageID jcstypes.PackageID `json:"packageID" binding:"required"`
}
func (r *PackageDelete) MakeParam() *sdks.RequestParam {
@ -228,9 +228,9 @@ func (c *PackageService) Delete(req PackageDelete) error {
const PackageClonePath = "/package/clone"
type PackageClone struct {
PackageID jcsypes.PackageID `json:"packageID" binding:"required"`
BucketID jcsypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
PackageID jcstypes.PackageID `json:"packageID" binding:"required"`
BucketID jcstypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
}
func (r *PackageClone) MakeParam() *sdks.RequestParam {
@ -238,7 +238,7 @@ func (r *PackageClone) MakeParam() *sdks.RequestParam {
}
type PackageCloneResp struct {
Package jcsypes.Package `json:"package"`
Package jcstypes.Package `json:"package"`
}
func (r *PackageCloneResp) ParseResponse(resp *http.Response) error {
@ -252,7 +252,7 @@ func (c *PackageService) Clone(req PackageClone) (*PackageCloneResp, error) {
const PackageListBucketPackagesPath = "/package/listBucketPackages"
type PackageListBucketPackages struct {
BucketID jcsypes.BucketID `form:"bucketID" url:"bucketID" binding:"required"`
BucketID jcstypes.BucketID `form:"bucketID" url:"bucketID" binding:"required"`
}
func (r *PackageListBucketPackages) MakeParam() *sdks.RequestParam {
@ -260,7 +260,7 @@ func (r *PackageListBucketPackages) MakeParam() *sdks.RequestParam {
}
type PackageListBucketPackagesResp struct {
Packages []jcsypes.Package `json:"packages"`
Packages []jcstypes.Package `json:"packages"`
}
func (r *PackageListBucketPackagesResp) ParseResponse(resp *http.Response) error {

44
client/sdk/api/v1/presigned.go
View File

@ -7,7 +7,7 @@ import (
"github.com/google/go-querystring/query"
"gitlink.org.cn/cloudream/jcs-pub/client/sdk/signer"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type PresignedService struct {
@ -33,10 +33,10 @@ func (c *PresignedService) ObjectListByPath(req PresignedObjectListByPath, expir
const PresignedObjectDownloadByPathPath = "/presigned/object/downloadByPath"
type PresignedObjectDownloadByPath struct {
PackageID jcsypes.PackageID `form:"packageID" url:"packageID" binding:"required"`
Path string `form:"path" url:"path" binding:"required"`
Offset int64 `form:"offset" url:"offset,omitempty"`
Length *int64 `form:"length" url:"length,omitempty"`
PackageID jcstypes.PackageID `form:"packageID" url:"packageID" binding:"required"`
Path string `form:"path" url:"path" binding:"required"`
Offset int64 `form:"offset" url:"offset,omitempty"`
Length *int64 `form:"length" url:"length,omitempty"`
}
func (c *PresignedService) ObjectDownloadByPath(req PresignedObjectDownloadByPath, expireIn int) (string, error) {
@ -46,9 +46,9 @@ func (c *PresignedService) ObjectDownloadByPath(req PresignedObjectDownloadByPat
const PresignedObjectDownloadPath = "/presigned/object/download"
type PresignedObjectDownload struct {
ObjectID jcsypes.ObjectID `form:"objectID" url:"objectID" binding:"required"`
Offset int64 `form:"offset" url:"offset,omitempty"`
Length *int64 `form:"length" url:"length,omitempty"`
ObjectID jcstypes.ObjectID `form:"objectID" url:"objectID" binding:"required"`
Offset int64 `form:"offset" url:"offset,omitempty"`
Length *int64 `form:"length" url:"length,omitempty"`
}
func (c *PresignedService) ObjectDownload(req PresignedObjectDownload, expireIn int) (string, error) {
@ -58,15 +58,15 @@ func (c *PresignedService) ObjectDownload(req PresignedObjectDownload, expireIn
const PresignedObjectUploadPath = "/presigned/object/upload"
type PresignedObjectUpload struct {
PackageID jcsypes.PackageID `form:"packageID" binding:"required" url:"packageID"`
Path string `form:"path" binding:"required" url:"path"`
Affinity jcsypes.UserSpaceID `form:"affinity" url:"affinity,omitempty"`
CopyTo []jcsypes.UserSpaceID `form:"copyTo" url:"copyTo,omitempty"`
CopyToPath []string `form:"copyToPath" url:"copyToPath,omitempty"`
PackageID jcstypes.PackageID `form:"packageID" binding:"required" url:"packageID"`
Path string `form:"path" binding:"required" url:"path"`
Affinity jcstypes.UserSpaceID `form:"affinity" url:"affinity,omitempty"`
CopyTo []jcstypes.UserSpaceID `form:"copyTo" url:"copyTo,omitempty"`
CopyToPath []string `form:"copyToPath" url:"copyToPath,omitempty"`
}
type PresignedObjectUploadResp struct {
Object jcsypes.Object `json:"object"`
Object jcstypes.Object `json:"object"`
}
func (c *PresignedService) ObjectUpload(req PresignedObjectUpload, expireIn int) (string, error) {
@ -76,12 +76,12 @@ func (c *PresignedService) ObjectUpload(req PresignedObjectUpload, expireIn int)
const PresignedObjectNewMultipartUploadPath = "/presigned/object/newMultipartUpload"
type PresignedObjectNewMultipartUpload struct {
PackageID jcsypes.PackageID `form:"packageID" binding:"required" url:"packageID"`
Path string `form:"path" binding:"required" url:"path"`
PackageID jcstypes.PackageID `form:"packageID" binding:"required" url:"packageID"`
Path string `form:"path" binding:"required" url:"path"`
}
type PresignedObjectNewMultipartUploadResp struct {
Object jcsypes.Object `json:"object"`
Object jcstypes.Object `json:"object"`
}
func (c *PresignedService) ObjectNewMultipartUpload(req PresignedObjectNewMultipartUpload, expireIn int) (string, error) {
@ -91,8 +91,8 @@ func (c *PresignedService) ObjectNewMultipartUpload(req PresignedObjectNewMultip
const PresignedObjectUploadPartPath = "/presigned/object/uploadPart"
type PresignedObjectUploadPart struct {
ObjectID jcsypes.ObjectID `form:"objectID" binding:"required" url:"objectID"`
Index int `form:"index" binding:"required" url:"index"`
ObjectID jcstypes.ObjectID `form:"objectID" binding:"required" url:"objectID"`
Index int `form:"index" binding:"required" url:"index"`
}
type PresignedUploadPartResp struct{}
@ -104,12 +104,12 @@ func (c *PresignedService) ObjectUploadPart(req PresignedObjectUploadPart, expir
const PresignedObjectCompleteMultipartUploadPath = "/presigned/object/completeMultipartUpload"
type PresignedObjectCompleteMultipartUpload struct {
ObjectID jcsypes.ObjectID `form:"objectID" binding:"required" url:"objectID"`
Indexes []int `form:"indexes" binding:"required" url:"indexes"`
ObjectID jcstypes.ObjectID `form:"objectID" binding:"required" url:"objectID"`
Indexes []int `form:"indexes" binding:"required" url:"indexes"`
}
type PresignedObjectCompleteMultipartUploadResp struct {
Object jcsypes.Object `json:"object"`
Object jcstypes.Object `json:"object"`
}
func (c *PresignedService) ObjectCompleteMultipartUpload(req PresignedObjectCompleteMultipartUpload, expireIn int) (string, error) {

26
client/sdk/api/v1/space_syncer.go
View File

@ -4,7 +4,7 @@ import (
"net/http"
"gitlink.org.cn/cloudream/common/sdks"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type SpaceSyncerService struct {
@ -20,14 +20,14 @@ func (c *Client) SpaceSyncer() *SpaceSyncerService {
const SpaceSyncerCreateTaskPath = "/spaceSyncer/createTask"
type SpaceSyncerCreateTask struct {
Trigger jcsypes.SpaceSyncTrigger `json:"trigger" binding:"required"`
Mode jcsypes.SpaceSyncMode `json:"mode" binding:"required"`
Filters []jcsypes.SpaceSyncFilter `json:"filters"`
Options jcsypes.SpaceSyncOptions `json:"options" binding:"required"`
SrcUserSpaceID jcsypes.UserSpaceID `json:"srcUserSpaceID" binding:"required"`
SrcPath string `json:"srcPath"`
DestUserSpaceIDs []jcsypes.UserSpaceID `json:"destUserSpaceIDs" binding:"required"`
DestPathes []string `json:"destPathes" binding:"required"`
Trigger jcstypes.SpaceSyncTrigger `json:"trigger" binding:"required"`
Mode jcstypes.SpaceSyncMode `json:"mode" binding:"required"`
Filters []jcstypes.SpaceSyncFilter `json:"filters"`
Options jcstypes.SpaceSyncOptions `json:"options" binding:"required"`
SrcUserSpaceID jcstypes.UserSpaceID `json:"srcUserSpaceID" binding:"required"`
SrcPath string `json:"srcPath"`
DestUserSpaceIDs []jcstypes.UserSpaceID `json:"destUserSpaceIDs" binding:"required"`
DestPathes []string `json:"destPathes" binding:"required"`
}
func (r *SpaceSyncerCreateTask) MakeParam() *sdks.RequestParam {
@ -35,7 +35,7 @@ func (r *SpaceSyncerCreateTask) MakeParam() *sdks.RequestParam {
}
type SpaceSyncerCreateTaskResp struct {
Task jcsypes.SpaceSyncTask `json:"task"`
Task jcstypes.SpaceSyncTask `json:"task"`
}
func (r *SpaceSyncerCreateTaskResp) ParseResponse(resp *http.Response) error {
@ -49,7 +49,7 @@ func (c *SpaceSyncerService) CreateTask(req SpaceSyncerCreateTask) (*SpaceSyncer
const SpaceSyncerGetTaskPath = "/spaceSyncer/getTask"
type SpaceSyncerGetTask struct {
TaskID jcsypes.SpaceSyncTaskID `url:"taskID" binding:"required"`
TaskID jcstypes.SpaceSyncTaskID `url:"taskID" binding:"required"`
}
func (r *SpaceSyncerGetTask) MakeParam() *sdks.RequestParam {
@ -57,7 +57,7 @@ func (r *SpaceSyncerGetTask) MakeParam() *sdks.RequestParam {
}
type SpaceSyncerGetTaskResp struct {
Task jcsypes.SpaceSyncTask `json:"task"`
Task jcstypes.SpaceSyncTask `json:"task"`
}
func (r *SpaceSyncerGetTaskResp) ParseResponse(resp *http.Response) error {
@ -71,7 +71,7 @@ func (c *SpaceSyncerService) GetTask(req SpaceSyncerGetTask) (*SpaceSyncerGetTas
const SpaceSyncerCancelTaskPath = "/spaceSyncer/cancelTask"
type SpaceSyncerCancelTask struct {
TaskID jcsypes.SpaceSyncTaskID `json:"taskID" binding:"required"`
TaskID jcstypes.SpaceSyncTaskID `json:"taskID" binding:"required"`
}
func (r *SpaceSyncerCancelTask) MakeParam() *sdks.RequestParam {

4
client/sdk/api/v1/storage_test.go
View File

@ -10,7 +10,7 @@ import (
. "github.com/smartystreets/goconvey/convey"
"gitlink.org.cn/cloudream/common/pkgs/iterator"
"gitlink.org.cn/cloudream/jcs-pub/client/sdk/api"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func Test_PackageGet(t *testing.T) {
@ -74,7 +74,7 @@ func Test_Object(t *testing.T) {
fileData[i] = byte(i)
}
stgAff := jcsypes.UserSpaceID(2)
stgAff := jcstypes.UserSpaceID(2)
pkgName := uuid.NewString()
createResp, err := cli.Package().Create(PackageCreate{

4
client/sdk/api/v1/system.go
View File

@ -4,7 +4,7 @@ import (
"net/http"
"gitlink.org.cn/cloudream/common/sdks"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type SystemService struct {
@ -25,7 +25,7 @@ func (r *SystemStatus) MakeParam() *sdks.RequestParam {
}
type SystemStatusResp struct {
SpeedStats jcsypes.SpeedStatsStatus `json:"speedStats"`
SpeedStats jcstypes.SpeedStatsStatus `json:"speedStats"`
}
func (r *SystemStatusResp) ParseResponse(resp *http.Response) error {

53
client/sdk/api/v1/user_space.go
View File

@ -4,8 +4,7 @@ import (
"net/http"
"gitlink.org.cn/cloudream/common/sdks"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type UserSpaceService struct {
@ -19,9 +18,9 @@ func (c *Client) UserSpace() *UserSpaceService {
const UserSpaceDownloadPackagePath = "/userSpace/downloadPackage"
type UserSpaceDownloadPackageReq struct {
PackageID jcsypes.PackageID `json:"packageID" binding:"required"`
UserSpaceID jcsypes.UserSpaceID `json:"userSpaceID" binding:"required"`
RootPath string `json:"rootPath"`
PackageID jcstypes.PackageID `json:"packageID" binding:"required"`
UserSpaceID jcstypes.UserSpaceID `json:"userSpaceID" binding:"required"`
RootPath string `json:"rootPath"`
}
func (r *UserSpaceDownloadPackageReq) MakeParam() *sdks.RequestParam {
@ -41,11 +40,11 @@ func (c *UserSpaceService) DownloadPackage(req UserSpaceDownloadPackageReq) (*Us
const UserSpaceCreatePackagePath = "/userSpace/createPackage"
type UserSpaceCreatePackageReq struct {
UserSpaceID jcsypes.UserSpaceID `json:"userSpaceID" binding:"required"`
Path string `json:"path" binding:"required"`
BucketID jcsypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
SpaceAffinity jcsypes.UserSpaceID `json:"spaceAffinity"`
UserSpaceID jcstypes.UserSpaceID `json:"userSpaceID" binding:"required"`
Path string `json:"path" binding:"required"`
BucketID jcstypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
SpaceAffinity jcstypes.UserSpaceID `json:"spaceAffinity"`
}
func (r *UserSpaceCreatePackageReq) MakeParam() *sdks.RequestParam {
@ -53,7 +52,7 @@ func (r *UserSpaceCreatePackageReq) MakeParam() *sdks.RequestParam {
}
type UserSpaceCreatePackageResp struct {
Package jcsypes.Package `json:"package"`
Package jcstypes.Package `json:"package"`
}
func (r *UserSpaceCreatePackageResp) ParseResponse(resp *http.Response) error {
@ -67,7 +66,7 @@ func (c *UserSpaceService) CreatePackage(req UserSpaceCreatePackageReq) (*UserSp
const UserSpaceGetPath = "/userSpace/get"
type UserSpaceGet struct {
UserSpaceID jcsypes.UserSpaceID `form:"userSpaceID" url:"userSpaceID" binding:"required"`
UserSpaceID jcstypes.UserSpaceID `form:"userSpaceID" url:"userSpaceID" binding:"required"`
}
func (r *UserSpaceGet) MakeParam() *sdks.RequestParam {
@ -75,7 +74,7 @@ func (r *UserSpaceGet) MakeParam() *sdks.RequestParam {
}
type UserSpaceGetResp struct {
UserSpace jcsypes.UserSpace `json:"userSpace"`
UserSpace jcstypes.UserSpace `json:"userSpace"`
}
func (r *UserSpaceGetResp) ParseResponse(resp *http.Response) error {
@ -97,7 +96,7 @@ func (r *UserSpaceGetByName) MakeParam() *sdks.RequestParam {
}
type UserSpaceGetByNameResp struct {
UserSpace jcsypes.UserSpace `json:"userSpace"`
UserSpace jcstypes.UserSpace `json:"userSpace"`
}
func (r *UserSpaceGetByNameResp) ParseResponse(resp *http.Response) error {
@ -117,7 +116,7 @@ func (r *UserSpaceGetAll) MakeParam() *sdks.RequestParam {
}
type UserSpaceGetAllResp struct {
UserSpaces []jcsypes.UserSpace `json:"userSpaces"`
UserSpaces []jcstypes.UserSpace `json:"userSpaces"`
}
func (r *UserSpaceGetAllResp) ParseResponse(resp *http.Response) error {
@ -134,10 +133,10 @@ const UserSpaceCreatePath = "/userSpace/create"
type UserSpaceCreate struct {
Name string `json:"name" binding:"required"`
Storage cortypes.StorageType `json:"storage" binding:"required"`
Credential cortypes.StorageCredential `json:"credential" binding:"required"`
ShardStore *cortypes.ShardStoreUserConfig `json:"shardStore"`
Features []cortypes.StorageFeature `json:"features"`
Storage jcstypes.StorageType `json:"storage" binding:"required"`
Credential jcstypes.StorageCredential `json:"credential" binding:"required"`
ShardStore *jcstypes.ShardStoreUserConfig `json:"shardStore"`
Features []jcstypes.StorageFeature `json:"features"`
WorkingDir string `json:"workingDir"`
}
@ -146,7 +145,7 @@ func (r *UserSpaceCreate) MakeParam() *sdks.RequestParam {
}
type UserSpaceCreateResp struct {
UserSpace jcsypes.UserSpace `json:"userSpace"`
UserSpace jcstypes.UserSpace `json:"userSpace"`
}
func (r *UserSpaceCreateResp) ParseResponse(resp *http.Response) error {
@ -161,10 +160,10 @@ func (c *UserSpaceService) Create(req UserSpaceCreate) (*UserSpaceCreateResp, er
const UserSpaceUpdatePath = "/userSpace/update"
type UserSpaceUpdate struct {
UserSpaceID jcsypes.UserSpaceID `json:"userSpaceID" binding:"required"`
UserSpaceID jcstypes.UserSpaceID `json:"userSpaceID" binding:"required"`
Name string `json:"name" binding:"required"`
Credential cortypes.StorageCredential `json:"credential" binding:"required"`
Features []cortypes.StorageFeature `json:"features"`
Credential jcstypes.StorageCredential `json:"credential" binding:"required"`
Features []jcstypes.StorageFeature `json:"features"`
}
func (r *UserSpaceUpdate) MakeParam() *sdks.RequestParam {
@ -172,7 +171,7 @@ func (r *UserSpaceUpdate) MakeParam() *sdks.RequestParam {
}
type UserSpaceUpdateResp struct {
UserSpace jcsypes.UserSpace `json:"userSpace"`
UserSpace jcstypes.UserSpace `json:"userSpace"`
}
func (r *UserSpaceUpdateResp) ParseResponse(resp *http.Response) error {
@ -187,7 +186,7 @@ func (c *UserSpaceService) Update(req UserSpaceUpdate) (*UserSpaceUpdateResp, er
const UserSpaceDeletePath = "/userSpace/delete"
type UserSpaceDelete struct {
UserSpaceID jcsypes.UserSpaceID `json:"userSpaceID" binding:"required"`
UserSpaceID jcstypes.UserSpaceID `json:"userSpaceID" binding:"required"`
}
func (r *UserSpaceDelete) MakeParam() *sdks.RequestParam {
@ -208,8 +207,8 @@ func (c *UserSpaceService) Delete(req UserSpaceDelete) (*UserSpaceDeleteResp, er
const UserSpaceTestPath = "/userSpace/test"
type UserSpaceTest struct {
Storage cortypes.StorageType `json:"storage" binding:"required"`
Credential cortypes.StorageCredential `json:"credential" binding:"required"`
Storage jcstypes.StorageType `json:"storage" binding:"required"`
Credential jcstypes.StorageCredential `json:"credential" binding:"required"`
WorikingDir string `json:"workingDir"`
}

2
common/globals/globals.go
View File

@ -1,7 +1,7 @@
package stgglb
import (
"gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type LocalMachineInfo struct {

4
common/globals/utils.go
View File

@ -1,9 +1,9 @@
package stgglb
import cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
import jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
// 根据当前节点与目标地址的距离关系，选择合适的地址
func SelectGRPCAddress(hub *cortypes.Hub, addr *cortypes.GRPCAddressInfo) (string, int) {
func SelectGRPCAddress(hub *jcstypes.Hub, addr *jcstypes.GRPCAddressInfo) (string, int) {
// TODO 重新设计选择LocalIP的策略
return addr.ExternalIP, addr.ExternalGRPCPort
}

12
common/pkgs/accesstoken/accesstoken.go
View File

@ -10,7 +10,7 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/async"
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type CacheEvent interface {
@ -23,17 +23,17 @@ type ExitEvent struct {
}
type CacheKey struct {
UserID cortypes.UserID
TokenID cortypes.AccessTokenID
UserID jcstypes.UserID
TokenID jcstypes.AccessTokenID
}
var ErrTokenNotFound = fmt.Errorf("token not found")
type AccessTokenLoader func(key CacheKey) (cortypes.UserAccessToken, error)
type AccessTokenLoader func(key CacheKey) (jcstypes.UserAccessToken, error)
type CacheEntry struct {
IsTokenValid bool
Token cortypes.UserAccessToken
Token jcstypes.UserAccessToken
PublicKey ed25519.PublicKey
LoadedAt time.Time
LastUsedAt time.Time
@ -227,6 +227,6 @@ func (mc *Cache) Verify(authInfo rpc.AccessTokenAuthInfo) bool {
return ed25519.Verify(token.PublicKey, []byte(MakeStringToSign(authInfo.UserID, authInfo.AccessTokenID, authInfo.Nonce)), []byte(sig))
}
func MakeStringToSign(userID cortypes.UserID, tokenID cortypes.AccessTokenID, nonce string) string {
func MakeStringToSign(userID jcstypes.UserID, tokenID jcstypes.AccessTokenID, nonce string) string {
return fmt.Sprintf("%v.%v.%v", userID, tokenID, nonce)
}

20
common/pkgs/connectivity/collector.go
View File

@ -11,7 +11,7 @@ import (
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator"
hubrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/hub"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type CollectorEvent interface {
@ -28,7 +28,7 @@ type CollectedEvent struct {
}
type Connectivity struct {
ToHubID cortypes.HubID
ToHubID jcstypes.HubID
Latency *time.Duration
TestTime time.Time
}
@ -38,7 +38,7 @@ type Collector struct {
enabled bool
collectNow chan any
done chan any
connectivities map[cortypes.HubID]Connectivity
connectivities map[jcstypes.HubID]Connectivity
lock *sync.RWMutex
}
@ -48,7 +48,7 @@ func NewEnabled(cfg Config) *Collector {
enabled: true,
collectNow: make(chan any, 1),
done: make(chan any, 1),
connectivities: make(map[cortypes.HubID]Connectivity),
connectivities: make(map[jcstypes.HubID]Connectivity),
lock: &sync.RWMutex{},
}
return &rpt
@ -58,16 +58,16 @@ func NewDisabled() *Collector {
enabled: false,
collectNow: make(chan any, 1),
done: make(chan any, 1),
connectivities: make(map[cortypes.HubID]Connectivity),
connectivities: make(map[jcstypes.HubID]Connectivity),
lock: &sync.RWMutex{},
}
}
func (r *Collector) GetAll() map[cortypes.HubID]Connectivity {
func (r *Collector) GetAll() map[jcstypes.HubID]Connectivity {
r.lock.RLock()
defer r.lock.RUnlock()
ret := make(map[cortypes.HubID]Connectivity)
ret := make(map[jcstypes.HubID]Connectivity)
for k, v := range r.connectivities {
ret[k] = v
}
@ -170,7 +170,7 @@ func (r *Collector) testing() bool {
r.lock.Lock()
// 删除所有hub的记录，然后重建，避免hub数量变化时导致残余数据
r.connectivities = make(map[cortypes.HubID]Connectivity)
r.connectivities = make(map[jcstypes.HubID]Connectivity)
for _, con := range cons {
r.connectivities[con.ToHubID] = con
}
@ -179,13 +179,13 @@ func (r *Collector) testing() bool {
return true
}
func (r *Collector) ping(hub cortypes.Hub) Connectivity {
func (r *Collector) ping(hub jcstypes.Hub) Connectivity {
log := logger.WithType[Collector]("").WithField("HubID", hub.HubID)
var ip string
var port int
switch addr := hub.Address.(type) {
case *cortypes.GRPCAddressInfo:
case *jcstypes.GRPCAddressInfo:
// TODO 重新设计选择LocalIP的策略
ip = addr.ExternalIP
port = addr.ExternalGRPCPort

30
common/pkgs/ioswitch2/fromto.go
View File

@ -4,7 +4,7 @@ import (
"gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type From interface {
@ -69,9 +69,9 @@ type FromTos []FromTo
type FromTo struct {
// 如果输入或者输出用到了EC编码的流，则需要提供EC参数。
ECParam *jcsypes.ECRedundancy
ECParam *jcstypes.ECRedundancy
// 同上
SegmentParam *jcsypes.SegmentRedundancy
SegmentParam *jcstypes.SegmentRedundancy
Froms []From
Toes []To
}
@ -110,12 +110,12 @@ func (f *FromDriver) GetStreamIndex() StreamIndex {
}
type FromShardStore struct {
FileHash jcsypes.FileHash
UserSpace jcsypes.UserSpaceDetail
FileHash jcstypes.FileHash
UserSpace jcstypes.UserSpaceDetail
StreamIndex StreamIndex
}
func NewFromShardstore(fileHash jcsypes.FileHash, space jcsypes.UserSpaceDetail, strIdx StreamIndex) *FromShardStore {
func NewFromShardstore(fileHash jcstypes.FileHash, space jcstypes.UserSpaceDetail, strIdx StreamIndex) *FromShardStore {
return &FromShardStore{
FileHash: fileHash,
UserSpace: space,
@ -128,11 +128,11 @@ func (f *FromShardStore) GetStreamIndex() StreamIndex {
}
type FromBaseStore struct {
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
}
func NewFromBaseStore(space jcsypes.UserSpaceDetail, path jcsypes.JPath) *FromBaseStore {
func NewFromBaseStore(space jcstypes.UserSpaceDetail, path jcstypes.JPath) *FromBaseStore {
return &FromBaseStore{
UserSpace: space,
Path: path,
@ -177,13 +177,13 @@ func (t *ToDriver) GetRange() math2.Range {
}
type ToShardStore struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
StreamIndex StreamIndex
Range math2.Range
ResultStoreKey string
}
func NewToShardStore(space jcsypes.UserSpaceDetail, strIdx StreamIndex, retStoreKey string) *ToShardStore {
func NewToShardStore(space jcstypes.UserSpaceDetail, strIdx StreamIndex, retStoreKey string) *ToShardStore {
return &ToShardStore{
UserSpace: space,
StreamIndex: strIdx,
@ -191,7 +191,7 @@ func NewToShardStore(space jcsypes.UserSpaceDetail, strIdx StreamIndex, retStore
}
}
func NewToShardStoreWithRange(space jcsypes.UserSpaceDetail, streamIndex StreamIndex, retStoreKey string, rng math2.Range) *ToShardStore {
func NewToShardStoreWithRange(space jcstypes.UserSpaceDetail, streamIndex StreamIndex, retStoreKey string, rng math2.Range) *ToShardStore {
return &ToShardStore{
UserSpace: space,
StreamIndex: streamIndex,
@ -209,12 +209,12 @@ func (t *ToShardStore) GetRange() math2.Range {
}
type ToBaseStore struct {
UserSpace jcsypes.UserSpaceDetail
ObjectPath jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
ObjectPath jcstypes.JPath
Option types.WriteOption
}
func NewToBaseStore(space jcsypes.UserSpaceDetail, objectPath jcsypes.JPath) *ToBaseStore {
func NewToBaseStore(space jcstypes.UserSpaceDetail, objectPath jcstypes.JPath) *ToBaseStore {
return &ToBaseStore{
UserSpace: space,
ObjectPath: objectPath,

8
common/pkgs/ioswitch2/http_hub_worker.go
View File

@ -9,16 +9,16 @@ import (
"gitlink.org.cn/cloudream/common/utils/io2"
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
hubapi "gitlink.org.cn/cloudream/jcs-pub/hub/sdk/api"
)
type HttpHubWorker struct {
Hub cortypes.Hub
Hub jcstypes.Hub
}
func (w *HttpHubWorker) NewClient() (exec.WorkerClient, error) {
addressInfo := w.Hub.Address.(*cortypes.HttpAddressInfo)
addressInfo := w.Hub.Address.(*jcstypes.HttpAddressInfo)
baseUrl := "http://" + addressInfo.ExternalIP + ":" + strconv.Itoa(addressInfo.Port)
config := hubapi.Config{
URL: baseUrl,
@ -47,7 +47,7 @@ func (w *HttpHubWorker) Equals(worker exec.WorkerInfo) bool {
}
type HttpHubWorkerClient struct {
hubID cortypes.HubID
hubID jcstypes.HubID
cli *hubapi.Client
}

8
common/pkgs/ioswitch2/hub_worker.go
View File

@ -11,7 +11,7 @@ import (
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
hubrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/hub"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
var _ = serder.UseTypeUnionExternallyTagged(types.Ref(types.NewTypeUnion[exec.WorkerInfo](
@ -20,8 +20,8 @@ var _ = serder.UseTypeUnionExternallyTagged(types.Ref(types.NewTypeUnion[exec.Wo
)))
type HubWorker struct {
Hub cortypes.Hub
Address cortypes.GRPCAddressInfo
Hub jcstypes.Hub
Address jcstypes.GRPCAddressInfo
}
func (w *HubWorker) NewClient() (exec.WorkerClient, error) {
@ -43,7 +43,7 @@ func (w *HubWorker) Equals(worker exec.WorkerInfo) bool {
}
type HubWorkerClient struct {
hubID cortypes.HubID
hubID jcstypes.HubID
cli *hubrpc.Client
}

28
common/pkgs/ioswitch2/ops2/base_store.go
View File

@ -13,7 +13,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
const (
@ -43,8 +43,8 @@ func (v *BaseReadStatsValue) Clone() exec.VarValue {
type BaseRead struct {
Output exec.VarID
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.OpenOption
}
@ -101,7 +101,7 @@ func (o *BaseRead) String() string {
}
type BaseReadDyn struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
Output exec.VarID
FileInfo exec.VarID
Option types.OpenOption
@ -167,8 +167,8 @@ func (o *BaseReadDyn) String() string {
type BaseWrite struct {
Input exec.VarID
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
FileInfo exec.VarID
Option types.WriteOption
}
@ -213,12 +213,12 @@ func (o *BaseWrite) String() string {
type BaseReadNode struct {
dag.NodeBase
From ioswitch2.From
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.OpenOption
}
func (b *GraphNodeBuilder) NewBaseRead(from ioswitch2.From, userSpace jcsypes.UserSpaceDetail, path jcsypes.JPath, opt types.OpenOption) *BaseReadNode {
func (b *GraphNodeBuilder) NewBaseRead(from ioswitch2.From, userSpace jcstypes.UserSpaceDetail, path jcstypes.JPath, opt types.OpenOption) *BaseReadNode {
node := &BaseReadNode{
From: from,
UserSpace: userSpace,
@ -254,11 +254,11 @@ func (t *BaseReadNode) GenerateOp() (exec.Op, error) {
type BaseReadDynNode struct {
dag.NodeBase
From ioswitch2.From
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
Option types.OpenOption
}
func (b *GraphNodeBuilder) NewBaseReadDyn(from ioswitch2.From, userSpace jcsypes.UserSpaceDetail, opt types.OpenOption) *BaseReadDynNode {
func (b *GraphNodeBuilder) NewBaseReadDyn(from ioswitch2.From, userSpace jcstypes.UserSpaceDetail, opt types.OpenOption) *BaseReadDynNode {
node := &BaseReadDynNode{
From: from,
UserSpace: userSpace,
@ -301,12 +301,12 @@ func (t *BaseReadDynNode) GenerateOp() (exec.Op, error) {
type BaseWriteNode struct {
dag.NodeBase
To ioswitch2.To
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.WriteOption
}
func (b *GraphNodeBuilder) NewBaseWrite(to ioswitch2.To, userSpace jcsypes.UserSpaceDetail, path jcsypes.JPath, opt types.WriteOption) *BaseWriteNode {
func (b *GraphNodeBuilder) NewBaseWrite(to ioswitch2.To, userSpace jcstypes.UserSpaceDetail, path jcstypes.JPath, opt types.WriteOption) *BaseWriteNode {
node := &BaseWriteNode{
To: to,
UserSpace: userSpace,

12
common/pkgs/ioswitch2/ops2/bypass.go
View File

@ -7,7 +7,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func init() {
@ -17,8 +17,8 @@ func init() {
// 旁路Http读取
type GetShardHTTPRequest struct {
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
UserSpace jcstypes.UserSpaceDetail
FileHash jcstypes.FileHash
Output exec.VarID
}
@ -64,11 +64,11 @@ func (o *GetShardHTTPRequest) String() string {
// 旁路Http读取
type GetShardHTTPRequestNode struct {
dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
UserSpace jcstypes.UserSpaceDetail
FileHash jcstypes.FileHash
}
func (b *GraphNodeBuilder) NewGetShardHTTPRequest(userSpace jcsypes.UserSpaceDetail, fileHash jcsypes.FileHash) *GetShardHTTPRequestNode {
func (b *GraphNodeBuilder) NewGetShardHTTPRequest(userSpace jcstypes.UserSpaceDetail, fileHash jcstypes.FileHash) *GetShardHTTPRequestNode {
node := &GetShardHTTPRequestNode{
UserSpace: userSpace,
FileHash: fileHash,

14
common/pkgs/ioswitch2/ops2/ec.go
View File

@ -13,7 +13,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/utils"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func init() {
@ -146,7 +146,7 @@ func (o *ECMultiply) String() string {
}
type CallECMultiplier struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
Coef [][]byte
Inputs []exec.VarID
Outputs []exec.VarID
@ -203,12 +203,12 @@ func (o *CallECMultiplier) String() string {
type ECMultiplyNode struct {
dag.NodeBase
EC jcsypes.ECRedundancy
EC jcstypes.ECRedundancy
InputIndexes []int
OutputIndexes []int
}
func (b *GraphNodeBuilder) NewECMultiply(ec jcsypes.ECRedundancy) *ECMultiplyNode {
func (b *GraphNodeBuilder) NewECMultiply(ec jcstypes.ECRedundancy) *ECMultiplyNode {
node := &ECMultiplyNode{
EC: ec,
}
@ -257,13 +257,13 @@ func (t *ECMultiplyNode) GenerateOp() (exec.Op, error) {
type CallECMultiplierNode struct {
dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
EC jcsypes.ECRedundancy
UserSpace jcstypes.UserSpaceDetail
EC jcstypes.ECRedundancy
InputIndexes []int
OutputIndexes []int
}
func (b *GraphNodeBuilder) NewCallECMultiplier(userSpace jcsypes.UserSpaceDetail) *CallECMultiplierNode {
func (b *GraphNodeBuilder) NewCallECMultiplier(userSpace jcstypes.UserSpaceDetail) *CallECMultiplierNode {
node := &CallECMultiplierNode{
UserSpace: userSpace,
}

14
common/pkgs/ioswitch2/ops2/multipart.go
View File

@ -9,7 +9,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func init() {
@ -40,7 +40,7 @@ func (v *UploadedPartInfoValue) Clone() exec.VarValue {
}
type MultipartInitiator struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
UploadArgs exec.VarID
UploadedParts []exec.VarID
FileOutput exec.VarID // 分片上传之后的临时文件的路径
@ -99,7 +99,7 @@ func (o *MultipartInitiator) String() string {
}
type MultipartUpload struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
UploadArgs exec.VarID
UploadResult exec.VarID
PartStream exec.VarID
@ -149,10 +149,10 @@ func (o *MultipartUpload) String() string {
type MultipartInitiatorNode struct {
dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
}
func (b *GraphNodeBuilder) NewMultipartInitiator(userSpace jcsypes.UserSpaceDetail) *MultipartInitiatorNode {
func (b *GraphNodeBuilder) NewMultipartInitiator(userSpace jcstypes.UserSpaceDetail) *MultipartInitiatorNode {
node := &MultipartInitiatorNode{
UserSpace: userSpace,
}
@ -194,12 +194,12 @@ func (n *MultipartInitiatorNode) GenerateOp() (exec.Op, error) {
type MultipartUploadNode struct {
dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
PartNumber int
PartSize int64
}
func (b *GraphNodeBuilder) NewMultipartUpload(userSpace jcsypes.UserSpaceDetail, partNumber int, partSize int64) *MultipartUploadNode {
func (b *GraphNodeBuilder) NewMultipartUpload(userSpace jcstypes.UserSpaceDetail, partNumber int, partSize int64) *MultipartUploadNode {
node := &MultipartUploadNode{
UserSpace: userSpace,
PartNumber: partNumber,

34
common/pkgs/ioswitch2/ops2/s2s.go
View File

@ -6,7 +6,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func init() {
@ -15,10 +15,10 @@ func init() {
}
type S2STransfer struct {
SrcSpace jcsypes.UserSpaceDetail
SrcPath jcsypes.JPath
DstSpace jcsypes.UserSpaceDetail
DstPath jcsypes.JPath
SrcSpace jcstypes.UserSpaceDetail
SrcPath jcstypes.JPath
DstSpace jcstypes.UserSpaceDetail
DstPath jcstypes.JPath
Output exec.VarID
}
@ -55,10 +55,10 @@ func (o *S2STransfer) String() string {
}
type S2STransferDyn struct {
SrcSpace jcsypes.UserSpaceDetail
SrcSpace jcstypes.UserSpaceDetail
SrcFileInfo exec.VarID
DstSpace jcsypes.UserSpaceDetail
DstPath jcsypes.JPath
DstSpace jcstypes.UserSpaceDetail
DstPath jcstypes.JPath
Output exec.VarID
}
@ -101,13 +101,13 @@ func (o *S2STransferDyn) String() string {
type S2STransferNode struct {
dag.NodeBase
SrcSpace jcsypes.UserSpaceDetail
SrcPath jcsypes.JPath
DstSpace jcsypes.UserSpaceDetail
DstPath jcsypes.JPath
SrcSpace jcstypes.UserSpaceDetail
SrcPath jcstypes.JPath
DstSpace jcstypes.UserSpaceDetail
DstPath jcstypes.JPath
}
func (b *GraphNodeBuilder) NewS2STransfer(srcSpace jcsypes.UserSpaceDetail, srcPath jcsypes.JPath, dstSpace jcsypes.UserSpaceDetail, dstPath jcsypes.JPath) *S2STransferNode {
func (b *GraphNodeBuilder) NewS2STransfer(srcSpace jcstypes.UserSpaceDetail, srcPath jcstypes.JPath, dstSpace jcstypes.UserSpaceDetail, dstPath jcstypes.JPath) *S2STransferNode {
n := &S2STransferNode{
SrcSpace: srcSpace,
SrcPath: srcPath,
@ -139,12 +139,12 @@ func (n *S2STransferNode) GenerateOp() (exec.Op, error) {
type S2STransferDynNode struct {
dag.NodeBase
SrcSpace jcsypes.UserSpaceDetail
DstSpace jcsypes.UserSpaceDetail
DstPath jcsypes.JPath
SrcSpace jcstypes.UserSpaceDetail
DstSpace jcstypes.UserSpaceDetail
DstPath jcstypes.JPath
}
func (b *GraphNodeBuilder) NewS2STransferDyn(srcSpace jcsypes.UserSpaceDetail, dstSpace jcsypes.UserSpaceDetail, dstPath jcsypes.JPath) *S2STransferDynNode {
func (b *GraphNodeBuilder) NewS2STransferDyn(srcSpace jcstypes.UserSpaceDetail, dstSpace jcstypes.UserSpaceDetail, dstPath jcstypes.JPath) *S2STransferDynNode {
n := &S2STransferDynNode{
SrcSpace: srcSpace,
DstSpace: dstSpace,

18
common/pkgs/ioswitch2/ops2/shard_store.go
View File

@ -6,7 +6,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func init() {
@ -15,8 +15,8 @@ func init() {
}
type GetShardInfo struct {
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
UserSpace jcstypes.UserSpaceDetail
FileHash jcstypes.FileHash
ShardInfo exec.VarID
}
@ -46,7 +46,7 @@ func (o *GetShardInfo) String() string {
}
type StoreShard struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
FileInfo exec.VarID
ShardInfo exec.VarID
}
@ -84,11 +84,11 @@ func (o *StoreShard) String() string {
type GetShardInfoNode struct {
dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
UserSpace jcstypes.UserSpaceDetail
FileHash jcstypes.FileHash
}
func (b *GraphNodeBuilder) NewGetShardInfo(userSpace jcsypes.UserSpaceDetail, fileHash jcsypes.FileHash) *GetShardInfoNode {
func (b *GraphNodeBuilder) NewGetShardInfo(userSpace jcstypes.UserSpaceDetail, fileHash jcstypes.FileHash) *GetShardInfoNode {
node := &GetShardInfoNode{
UserSpace: userSpace,
FileHash: fileHash,
@ -116,11 +116,11 @@ func (n *GetShardInfoNode) GenerateOp() (exec.Op, error) {
type StoreShardNode struct {
dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
ShardInfoKey string
}
func (b *GraphNodeBuilder) NewStoreShard(userSpace jcsypes.UserSpaceDetail, shardInfoKey string) *StoreShardNode {
func (b *GraphNodeBuilder) NewStoreShard(userSpace jcstypes.UserSpaceDetail, shardInfoKey string) *StoreShardNode {
node := &StoreShardNode{
UserSpace: userSpace,
ShardInfoKey: shardInfoKey,

9
common/pkgs/ioswitch2/parser/gen/generator.go
View File

@ -12,8 +12,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser/state"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
// 检查使用不同编码时参数是否设置到位
@ -385,17 +384,17 @@ func buildToNode(ctx *state.GenerateState, t ioswitch2.To) (ops2.ToNode, error)
}
}
func setEnvBySpace(n dag.Node, space *jcsypes.UserSpaceDetail) error {
func setEnvBySpace(n dag.Node, space *jcstypes.UserSpaceDetail) error {
if space.RecommendHub == nil {
n.Env().ToEnvDriver(true)
return nil
}
switch addr := space.RecommendHub.Address.(type) {
case *cortypes.HttpAddressInfo:
case *jcstypes.HttpAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HttpHubWorker{Hub: *space.RecommendHub}, true)
case *cortypes.GRPCAddressInfo:
case *jcstypes.GRPCAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HubWorker{Hub: *space.RecommendHub, Address: *addr}, true)
default:

9
common/pkgs/ioswitch2/parser/opt/utils.go
View File

@ -5,21 +5,20 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func setEnvBySpace(n dag.Node, space *jcsypes.UserSpaceDetail) error {
func setEnvBySpace(n dag.Node, space *jcstypes.UserSpaceDetail) error {
if space.RecommendHub == nil {
n.Env().ToEnvDriver(true)
return nil
}
switch addr := space.RecommendHub.Address.(type) {
case *cortypes.HttpAddressInfo:
case *jcstypes.HttpAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HttpHubWorker{Hub: *space.RecommendHub}, true)
case *cortypes.GRPCAddressInfo:
case *jcstypes.GRPCAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HubWorker{Hub: *space.RecommendHub, Address: *addr}, true)
default:

4
common/pkgs/ioswitch2/plans/complete_multipart.go
View File

@ -8,10 +8,10 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/plan"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func CompleteMultipart(blocks []jcsypes.ObjectBlock, blockSpaces []jcsypes.UserSpaceDetail, targetSpace jcsypes.UserSpaceDetail, shardInfoKey string, blder *exec.PlanBuilder) error {
func CompleteMultipart(blocks []jcstypes.ObjectBlock, blockSpaces []jcstypes.UserSpaceDetail, targetSpace jcstypes.UserSpaceDetail, shardInfoKey string, blder *exec.PlanBuilder) error {
da := ops2.NewGraphNodeBuilder()
sizes := make([]int64, len(blocks))

9
common/pkgs/ioswitch2/plans/utils.go
View File

@ -5,21 +5,20 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func setEnvBySpace(n dag.Node, space *jcsypes.UserSpaceDetail) error {
func setEnvBySpace(n dag.Node, space *jcstypes.UserSpaceDetail) error {
if space.RecommendHub == nil {
n.Env().ToEnvDriver(true)
return nil
}
switch addr := space.RecommendHub.Address.(type) {
case *cortypes.HttpAddressInfo:
case *jcstypes.HttpAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HttpHubWorker{Hub: *space.RecommendHub}, true)
case *cortypes.GRPCAddressInfo:
case *jcstypes.GRPCAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HubWorker{Hub: *space.RecommendHub, Address: *addr}, true)
default:

14
common/pkgs/ioswitchlrc/fromto.go
View File

@ -4,7 +4,7 @@ import (
"gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type From interface {
@ -39,12 +39,12 @@ func (f *FromDriver) GetDataIndex() int {
}
type FromNode struct {
FileHash jcsypes.FileHash
UserSpace jcsypes.UserSpaceDetail
FileHash jcstypes.FileHash
UserSpace jcstypes.UserSpaceDetail
DataIndex int
}
func NewFromStorage(fileHash jcsypes.FileHash, space jcsypes.UserSpaceDetail, dataIndex int) *FromNode {
func NewFromStorage(fileHash jcstypes.FileHash, space jcstypes.UserSpaceDetail, dataIndex int) *FromNode {
return &FromNode{
FileHash: fileHash,
DataIndex: dataIndex,
@ -88,14 +88,14 @@ func (t *ToDriver) GetRange() math2.Range {
}
type ToNode struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
DataIndex int
Range math2.Range
FileHashStoreKey string
Option types.WriteOption
}
func NewToStorage(space jcsypes.UserSpaceDetail, dataIndex int, fileHashStoreKey string) *ToNode {
func NewToStorage(space jcstypes.UserSpaceDetail, dataIndex int, fileHashStoreKey string) *ToNode {
return &ToNode{
UserSpace: space,
DataIndex: dataIndex,
@ -103,7 +103,7 @@ func NewToStorage(space jcsypes.UserSpaceDetail, dataIndex int, fileHashStoreKey
}
}
func NewToStorageWithRange(space jcsypes.UserSpaceDetail, dataIndex int, fileHashStoreKey string, rng math2.Range) *ToNode {
func NewToStorageWithRange(space jcstypes.UserSpaceDetail, dataIndex int, fileHashStoreKey string, rng math2.Range) *ToNode {
return &ToNode{
UserSpace: space,
DataIndex: dataIndex,

6
common/pkgs/ioswitchlrc/hub_worker.go
View File

@ -7,7 +7,7 @@ import (
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
hubrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/hub"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
// var _ = serder.UseTypeUnionExternallyTagged(types.Ref(types.NewTypeUnion[exec.WorkerInfo](
@ -15,8 +15,8 @@ import (
// )))
type HubWorker struct {
Hub cortypes.Hub
Address cortypes.GRPCAddressInfo
Hub jcstypes.Hub
Address jcstypes.GRPCAddressInfo
}
func (w *HubWorker) NewClient() (exec.WorkerClient, error) {

28
common/pkgs/ioswitchlrc/ops2/base_store.go
View File

@ -12,7 +12,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func init() {
@ -23,8 +23,8 @@ func init() {
type BaseRead struct {
Output exec.VarID
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.OpenOption
}
@ -67,7 +67,7 @@ func (o *BaseRead) String() string {
}
type BaseReadDyn struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
Output exec.VarID
Path exec.VarID
Option types.OpenOption
@ -118,8 +118,8 @@ func (o *BaseReadDyn) String() string {
type BaseWrite struct {
Input exec.VarID
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
WriteResult exec.VarID
Option types.WriteOption
}
@ -164,12 +164,12 @@ func (o *BaseWrite) String() string {
type BaseReadNode struct {
dag.NodeBase
From ioswitchlrc.From
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.OpenOption
}
func (b *GraphNodeBuilder) NewBaseRead(from ioswitchlrc.From, userSpace jcsypes.UserSpaceDetail, path jcsypes.JPath, opt types.OpenOption) *BaseReadNode {
func (b *GraphNodeBuilder) NewBaseRead(from ioswitchlrc.From, userSpace jcstypes.UserSpaceDetail, path jcstypes.JPath, opt types.OpenOption) *BaseReadNode {
node := &BaseReadNode{
From: from,
UserSpace: userSpace,
@ -205,11 +205,11 @@ func (t *BaseReadNode) GenerateOp() (exec.Op, error) {
type BaseReadDynNode struct {
dag.NodeBase
From ioswitchlrc.From
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
Option types.OpenOption
}
func (b *GraphNodeBuilder) NewBaseReadDyn(from ioswitchlrc.From, userSpace jcsypes.UserSpaceDetail, opt types.OpenOption) *BaseReadDynNode {
func (b *GraphNodeBuilder) NewBaseReadDyn(from ioswitchlrc.From, userSpace jcstypes.UserSpaceDetail, opt types.OpenOption) *BaseReadDynNode {
node := &BaseReadDynNode{
From: from,
UserSpace: userSpace,
@ -252,12 +252,12 @@ func (t *BaseReadDynNode) GenerateOp() (exec.Op, error) {
type BaseWriteNode struct {
dag.NodeBase
To ioswitchlrc.To
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.WriteOption
}
func (b *GraphNodeBuilder) NewBaseWrite(to ioswitchlrc.To, userSpace jcsypes.UserSpaceDetail, path jcsypes.JPath, opt types.WriteOption) *BaseWriteNode {
func (b *GraphNodeBuilder) NewBaseWrite(to ioswitchlrc.To, userSpace jcstypes.UserSpaceDetail, path jcstypes.JPath, opt types.WriteOption) *BaseWriteNode {
node := &BaseWriteNode{
To: to,
UserSpace: userSpace,

10
common/pkgs/ioswitchlrc/ops2/ec.go
View File

@ -12,7 +12,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/utils"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func init() {
@ -114,12 +114,12 @@ func (o *GalMultiply) String() string {
type LRCConstructAnyNode struct {
dag.NodeBase
LRC jcsypes.LRCRedundancy
LRC jcstypes.LRCRedundancy
InputIndexes []int
OutputIndexes []int
}
func (b *GraphNodeBuilder) NewLRCConstructAny(lrc jcsypes.LRCRedundancy) *LRCConstructAnyNode {
func (b *GraphNodeBuilder) NewLRCConstructAny(lrc jcstypes.LRCRedundancy) *LRCConstructAnyNode {
node := &LRCConstructAnyNode{
LRC: lrc,
}
@ -168,11 +168,11 @@ func (t *LRCConstructAnyNode) GenerateOp() (exec.Op, error) {
type LRCConstructGroupNode struct {
dag.NodeBase
LRC jcsypes.LRCRedundancy
LRC jcstypes.LRCRedundancy
TargetBlockIndex int
}
func (b *GraphNodeBuilder) NewLRCConstructGroup(lrc jcsypes.LRCRedundancy) *LRCConstructGroupNode {
func (b *GraphNodeBuilder) NewLRCConstructGroup(lrc jcstypes.LRCRedundancy) *LRCConstructGroupNode {
node := &LRCConstructGroupNode{
LRC: lrc,
}

18
common/pkgs/ioswitchlrc/ops2/shard_store.go
View File

@ -6,7 +6,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func init() {
@ -15,8 +15,8 @@ func init() {
}
type GetShardInfo struct {
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
UserSpace jcstypes.UserSpaceDetail
FileHash jcstypes.FileHash
ShardInfo exec.VarID
}
@ -46,7 +46,7 @@ func (o *GetShardInfo) String() string {
}
type StoreShard struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
FileInfo exec.VarID
ShardInfo exec.VarID
}
@ -84,11 +84,11 @@ func (o *StoreShard) String() string {
type GetShardInfoNode struct {
dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
UserSpace jcstypes.UserSpaceDetail
FileHash jcstypes.FileHash
}
func (b *GraphNodeBuilder) NewGetShardInfo(userSpace jcsypes.UserSpaceDetail, fileHash jcsypes.FileHash) *GetShardInfoNode {
func (b *GraphNodeBuilder) NewGetShardInfo(userSpace jcstypes.UserSpaceDetail, fileHash jcstypes.FileHash) *GetShardInfoNode {
node := &GetShardInfoNode{
UserSpace: userSpace,
FileHash: fileHash,
@ -116,11 +116,11 @@ func (n *GetShardInfoNode) GenerateOp() (exec.Op, error) {
type StoreShardNode struct {
dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
ShardInfoKey string
}
func (b *GraphNodeBuilder) NewStoreShard(userSpace jcsypes.UserSpaceDetail, shardInfoKey string) *StoreShardNode {
func (b *GraphNodeBuilder) NewStoreShard(userSpace jcstypes.UserSpaceDetail, shardInfoKey string) *StoreShardNode {
node := &StoreShardNode{
UserSpace: userSpace,
ShardInfoKey: shardInfoKey,

10
common/pkgs/ioswitchlrc/parser/generator.go
View File

@ -9,11 +9,11 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/plan"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc/ops2"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type GenerateContext struct {
LRC jcsypes.LRCRedundancy
LRC jcstypes.LRCRedundancy
DAG *ops2.GraphNodeBuilder
To []ioswitchlrc.To
ToNodes map[ioswitchlrc.To]ops2.ToNode
@ -27,7 +27,7 @@ func Encode(fr ioswitchlrc.From, toes []ioswitchlrc.To, blder *exec.PlanBuilder)
}
ctx := GenerateContext{
LRC: jcsypes.DefaultLRCRedundancy,
LRC: jcstypes.DefaultLRCRedundancy,
DAG: ops2.NewGraphNodeBuilder(),
To: toes,
ToNodes: make(map[ioswitchlrc.To]ops2.ToNode),
@ -124,7 +124,7 @@ func buildDAGEncode(ctx *GenerateContext, fr ioswitchlrc.From, toes []ioswitchlr
// 提供数据块+编码块中的k个块，重建任意块，包括完整文件。
func ReconstructAny(frs []ioswitchlrc.From, toes []ioswitchlrc.To, blder *exec.PlanBuilder) error {
ctx := GenerateContext{
LRC: jcsypes.DefaultLRCRedundancy,
LRC: jcstypes.DefaultLRCRedundancy,
DAG: ops2.NewGraphNodeBuilder(),
To: toes,
ToNodes: make(map[ioswitchlrc.To]ops2.ToNode),
@ -245,7 +245,7 @@ func buildDAGReconstructAny(ctx *GenerateContext, frs []ioswitchlrc.From, toes [
// 输入同一组的多个块，恢复出剩下缺少的一个块。
func ReconstructGroup(frs []ioswitchlrc.From, toes []ioswitchlrc.To, blder *exec.PlanBuilder) error {
ctx := GenerateContext{
LRC: jcsypes.DefaultLRCRedundancy,
LRC: jcstypes.DefaultLRCRedundancy,
DAG: ops2.NewGraphNodeBuilder(),
To: toes,
ToNodes: make(map[ioswitchlrc.To]ops2.ToNode),

9
common/pkgs/ioswitchlrc/parser/utils.go
View File

@ -5,21 +5,20 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func setEnvBySpace(n dag.Node, space *jcsypes.UserSpaceDetail) error {
func setEnvBySpace(n dag.Node, space *jcstypes.UserSpaceDetail) error {
if space.RecommendHub == nil {
n.Env().ToEnvDriver(true)
return nil
}
switch addr := space.RecommendHub.Address.(type) {
case *cortypes.HttpAddressInfo:
case *jcstypes.HttpAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HttpHubWorker{Hub: *space.RecommendHub}, true)
case *cortypes.GRPCAddressInfo:
case *jcstypes.GRPCAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HubWorker{Hub: *space.RecommendHub, Address: *addr}, true)
default:

8
common/pkgs/publock/reqbuilder/user_space.go
View File

@ -5,7 +5,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/lockprovider"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type UserSpaceLockReqBuilder struct {
@ -15,7 +15,7 @@ type UserSpaceLockReqBuilder struct {
func (b *LockRequestBuilder) UserSpace() *UserSpaceLockReqBuilder {
return &UserSpaceLockReqBuilder{LockRequestBuilder: b}
}
func (b *UserSpaceLockReqBuilder) Buzy(spaceID jcsypes.UserSpaceID) *UserSpaceLockReqBuilder {
func (b *UserSpaceLockReqBuilder) Buzy(spaceID jcstypes.UserSpaceID) *UserSpaceLockReqBuilder {
b.locks = append(b.locks, types.Lock{
Path: b.makePath(spaceID),
Name: lockprovider.UserSpaceBuzyLock,
@ -24,7 +24,7 @@ func (b *UserSpaceLockReqBuilder) Buzy(spaceID jcsypes.UserSpaceID) *UserSpaceLo
return b
}
func (b *UserSpaceLockReqBuilder) GC(spaceID jcsypes.UserSpaceID) *UserSpaceLockReqBuilder {
func (b *UserSpaceLockReqBuilder) GC(spaceID jcstypes.UserSpaceID) *UserSpaceLockReqBuilder {
b.locks = append(b.locks, types.Lock{
Path: b.makePath(spaceID),
Name: lockprovider.UserSpaceGCLock,
@ -33,6 +33,6 @@ func (b *UserSpaceLockReqBuilder) GC(spaceID jcsypes.UserSpaceID) *UserSpaceLock
return b
}
func (b *UserSpaceLockReqBuilder) makePath(hubID jcsypes.UserSpaceID) []string {
func (b *UserSpaceLockReqBuilder) makePath(hubID jcstypes.UserSpaceID) []string {
return []string{lockprovider.UserSpaceLockPathPrefix, strconv.FormatInt(int64(hubID), 10)}
}

14
common/pkgs/rpc/auth.go
View File

@ -5,7 +5,7 @@ import (
"fmt"
"strconv"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"golang.org/x/net/context"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
@ -27,8 +27,8 @@ const (
)
type AccessTokenAuthInfo struct {
UserID cortypes.UserID
AccessTokenID cortypes.AccessTokenID
UserID jcstypes.UserID
AccessTokenID jcstypes.AccessTokenID
Nonce string
Signature string
}
@ -125,8 +125,8 @@ func (s *ServerBase) authUnary(
}
authInfo := AccessTokenAuthInfo{
UserID: cortypes.UserID(userID),
AccessTokenID: cortypes.AccessTokenID(accessTokenIDs[0]),
UserID: jcstypes.UserID(userID),
AccessTokenID: jcstypes.AccessTokenID(accessTokenIDs[0]),
Nonce: nonce[0],
Signature: signature[0],
}
@ -200,8 +200,8 @@ func (s *ServerBase) authStream(
}
authInfo := AccessTokenAuthInfo{
UserID: cortypes.UserID(userID),
AccessTokenID: cortypes.AccessTokenID(accessTokenIDs[0]),
UserID: jcstypes.UserID(userID),
AccessTokenID: jcstypes.AccessTokenID(accessTokenIDs[0]),
Nonce: nonce[0],
Signature: signature[0],
}

30
common/pkgs/rpc/coordinator/hub.go
View File

@ -4,7 +4,7 @@ import (
context "context"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type HubService interface {
@ -18,18 +18,18 @@ type HubService interface {
}
type GetHubConfig struct {
HubID cortypes.HubID `json:"hubID"`
HubID jcstypes.HubID `json:"hubID"`
}
type GetHubConfigResp struct {
Hub cortypes.Hub `json:"hub"`
Hub jcstypes.Hub `json:"hub"`
}
func ReqGetHubConfig(hubID cortypes.HubID) *GetHubConfig {
func ReqGetHubConfig(hubID jcstypes.HubID) *GetHubConfig {
return &GetHubConfig{
HubID: hubID,
}
}
func RespGetHubConfig(hub cortypes.Hub) *GetHubConfigResp {
func RespGetHubConfig(hub jcstypes.Hub) *GetHubConfigResp {
return &GetHubConfigResp{
Hub: hub,
}
@ -43,23 +43,23 @@ func (s *Server) GetHubConfig(ctx context.Context, req *rpc.Request) (*rpc.Respo
// 获取指定节点的信息。如果HubIDs为nil，则返回所有Hub
type GetHubs struct {
HubIDs []cortypes.HubID `json:"hubIDs"`
HubIDs []jcstypes.HubID `json:"hubIDs"`
}
type GetHubsResp struct {
Hubs []*cortypes.Hub `json:"hubs"`
Hubs []*jcstypes.Hub `json:"hubs"`
}
func NewGetHubs(hubIDs []cortypes.HubID) *GetHubs {
func NewGetHubs(hubIDs []jcstypes.HubID) *GetHubs {
return &GetHubs{
HubIDs: hubIDs,
}
}
func NewGetHubsResp(hubs []*cortypes.Hub) *GetHubsResp {
func NewGetHubsResp(hubs []*jcstypes.Hub) *GetHubsResp {
return &GetHubsResp{
Hubs: hubs,
}
}
func (r *GetHubsResp) GetHub(id cortypes.HubID) *cortypes.Hub {
func (r *GetHubsResp) GetHub(id jcstypes.HubID) *jcstypes.Hub {
for _, n := range r.Hubs {
if n.HubID == id {
return n
@ -78,18 +78,18 @@ func (s *Server) GetHubs(ctx context.Context, req *rpc.Request) (*rpc.Response,
// 获取节点连通性信息
type GetHubConnectivities struct {
HubIDs []cortypes.HubID `json:"hubIDs"`
HubIDs []jcstypes.HubID `json:"hubIDs"`
}
type GetHubConnectivitiesResp struct {
Connectivities []cortypes.HubConnectivity `json:"hubs"`
Connectivities []jcstypes.HubConnectivity `json:"hubs"`
}
func ReqGetHubConnectivities(hubIDs []cortypes.HubID) *GetHubConnectivities {
func ReqGetHubConnectivities(hubIDs []jcstypes.HubID) *GetHubConnectivities {
return &GetHubConnectivities{
HubIDs: hubIDs,
}
}
func RespGetHubConnectivities(cons []cortypes.HubConnectivity) *GetHubConnectivitiesResp {
func RespGetHubConnectivities(cons []jcstypes.HubConnectivity) *GetHubConnectivitiesResp {
return &GetHubConnectivitiesResp{
Connectivities: cons,
}
@ -103,7 +103,7 @@ func (s *Server) GetHubConnectivities(ctx context.Context, req *rpc.Request) (*r
// 上报节点连通性信息
type ReportHubConnectivity struct {
Connecttivities []cortypes.HubConnectivity
Connecttivities []jcstypes.HubConnectivity
}
type ReportHubConnectivityResp struct {
}

6
common/pkgs/rpc/coordinator/storage.go
View File

@ -4,7 +4,7 @@ import (
context "context"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type StorageService interface {
@ -13,10 +13,10 @@ type StorageService interface {
// 为指定的Storage选择一个适合通信的Hub
type SelectStorageHub struct {
Storages []cortypes.StorageType
Storages []jcstypes.StorageType
}
type SelectStorageHubResp struct {
Hubs []*cortypes.Hub
Hubs []*jcstypes.Hub
}
var _ = TokenAuth(Coordinator_SelectStorageHub_FullMethodName)

14
common/pkgs/rpc/coordinator/user.go
View File

@ -4,7 +4,7 @@ import (
context "context"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type UserService interface {
@ -23,7 +23,7 @@ type UserLogin struct {
Password string
}
type UserLoginResp struct {
Token cortypes.UserAccessToken
Token jcstypes.UserAccessToken
PrivateKey string
}
@ -42,7 +42,7 @@ func (s *Server) UserLogin(ctx context.Context, req *rpc.Request) (*rpc.Response
// 客户端刷新Token，原始Token会继续有效。
type UserRefreshToken struct{}
type UserRefreshTokenResp struct {
Token cortypes.UserAccessToken
Token jcstypes.UserAccessToken
PrivateKey string
}
@ -76,12 +76,12 @@ func (s *Server) UserLogout(ctx context.Context, req *rpc.Request) (*rpc.Respons
// Hub服务加载AccessToken
type HubLoadAccessToken struct {
HubID cortypes.HubID
UserID cortypes.UserID
TokenID cortypes.AccessTokenID
HubID jcstypes.HubID
UserID jcstypes.UserID
TokenID jcstypes.AccessTokenID
}
type HubLoadAccessTokenResp struct {
Token cortypes.UserAccessToken
Token jcstypes.UserAccessToken
}
func (c *Client) HubLoadAccessToken(ctx context.Context, msg *HubLoadAccessToken) (*HubLoadAccessTokenResp, *rpc.CodeError) {

10
common/pkgs/rpc/hub/cache.go
View File

@ -4,7 +4,7 @@ package hubrpc
import (
"context"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
)
@ -15,10 +15,10 @@ type CacheSvc interface {
// 获取Cache中文件列表
type CheckCache struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
}
type CheckCacheResp struct {
FileHashes []jcsypes.FileHash
FileHashes []jcstypes.FileHash
}
func (c *Client) CheckCache(ctx context.Context, req *CheckCache) (*CheckCacheResp, *rpc.CodeError) {
@ -33,8 +33,8 @@ func (s *Server) CheckCache(ctx context.Context, req *rpc.Request) (*rpc.Respons
// 清理Cache中不用的文件
type CacheGC struct {
UserSpace jcsypes.UserSpaceDetail
Availables []jcsypes.FileHash
UserSpace jcstypes.UserSpaceDetail
Availables []jcstypes.FileHash
}
type CacheGCResp struct{}

6
common/pkgs/rpc/hub/user.go
View File

@ -4,7 +4,7 @@ import (
context "context"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type UserSvc interface {
@ -13,8 +13,8 @@ type UserSvc interface {
// 通知用户的Token登出
type NotifyUserAccessTokenInvalid struct {
UserID cortypes.UserID
TokenID cortypes.AccessTokenID
UserID jcstypes.UserID
TokenID jcstypes.AccessTokenID
}
type NotifyUserAccessTokenInvalidResp struct{}

6
common/pkgs/rpc/hub/user_space.go
View File

@ -4,7 +4,7 @@ package hubrpc
import (
"context"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
)
@ -16,7 +16,7 @@ type UserSpaceSvc interface {
// 列出指定BaseStore的指定位置内的所有文件
type BaseStoreListAll struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
Path string
}
type BaseStoreListAllResp struct {
@ -35,7 +35,7 @@ func (s *Server) BaseStoreListAll(ctx context.Context, req *rpc.Request) (*rpc.R
// 批量在指定BaseStore中创建文件夹
type BaseStoreMkdirs struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
Pathes []string
}

16
common/pkgs/servicestats/hub_strorage_transfer.go
View File

@ -6,22 +6,22 @@ import (
"time"
"gitlink.org.cn/cloudream/common/utils/math2"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type HubStorageTransferStats struct {
data HubStorageTransferStatsData
fromHubID cortypes.HubID
fromHubID jcstypes.HubID
lock *sync.Mutex
}
type HubStorageTransferStatsData struct {
Entries map[cortypes.StorageID]*HubStorageTransferStatsEntry
Entries map[jcstypes.StorageID]*HubStorageTransferStatsEntry
StartTime time.Time
}
type HubStorageTransferStatsEntry struct {
DestStorageID cortypes.StorageID
DestStorageID jcstypes.StorageID
OutputBytes int64
MaxOutputBytes int64
@ -36,7 +36,7 @@ type HubStorageTransferStatsEntry struct {
SuccessInput int64
}
func (s *HubStorageTransferStats) RecordUpload(dstStorageID cortypes.StorageID, transferBytes int64, isSuccess bool) {
func (s *HubStorageTransferStats) RecordUpload(dstStorageID jcstypes.StorageID, transferBytes int64, isSuccess bool) {
s.lock.Lock()
defer s.lock.Unlock()
@ -58,7 +58,7 @@ func (s *HubStorageTransferStats) RecordUpload(dstStorageID cortypes.StorageID,
e.TotalOutput++
}
func (s *HubStorageTransferStats) RecordDownload(dstStorageID cortypes.StorageID, transferBytes int64, isSuccess bool) {
func (s *HubStorageTransferStats) RecordDownload(dstStorageID jcstypes.StorageID, transferBytes int64, isSuccess bool) {
s.lock.Lock()
defer s.lock.Unlock()
@ -83,7 +83,7 @@ func (s *HubStorageTransferStats) Reset() time.Time {
s.lock.Lock()
defer s.lock.Unlock()
s.data.Entries = make(map[cortypes.StorageID]*HubStorageTransferStatsEntry)
s.data.Entries = make(map[jcstypes.StorageID]*HubStorageTransferStatsEntry)
s.data.StartTime = time.Now()
return s.data.StartTime
}
@ -93,7 +93,7 @@ func (s *HubStorageTransferStats) DumpData() HubStorageTransferStatsData {
defer s.lock.Unlock()
data := s.data
data.Entries = make(map[cortypes.StorageID]*HubStorageTransferStatsEntry)
data.Entries = make(map[jcstypes.StorageID]*HubStorageTransferStatsEntry)
for k, v := range s.data.Entries {
v2 := *v
data.Entries[k] = &v2

16
common/pkgs/servicestats/hub_transfter.go
View File

@ -6,22 +6,22 @@ import (
"time"
"gitlink.org.cn/cloudream/common/utils/math2"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type HubTransferStats struct {
data HubTransferStatsData
fromHubID cortypes.HubID
fromHubID jcstypes.HubID
lock *sync.Mutex
}
type HubTransferStatsData struct {
Entries map[cortypes.HubID]*HubTransferStatsEntry
Entries map[jcstypes.HubID]*HubTransferStatsEntry
StartTime time.Time
}
type HubTransferStatsEntry struct {
DestHubID cortypes.HubID
DestHubID jcstypes.HubID
OutputBytes int64
MaxOutputBytes int64
@ -36,7 +36,7 @@ type HubTransferStatsEntry struct {
SuccessInput int64
}
func (s *HubTransferStats) RecordOutput(dstHubID cortypes.HubID, transferBytes int64, isSuccess bool) {
func (s *HubTransferStats) RecordOutput(dstHubID jcstypes.HubID, transferBytes int64, isSuccess bool) {
s.lock.Lock()
defer s.lock.Unlock()
@ -58,7 +58,7 @@ func (s *HubTransferStats) RecordOutput(dstHubID cortypes.HubID, transferBytes i
e.TotalOutput++
}
func (s *HubTransferStats) RecordInput(dstHubID cortypes.HubID, transferBytes int64, isSuccess bool) {
func (s *HubTransferStats) RecordInput(dstHubID jcstypes.HubID, transferBytes int64, isSuccess bool) {
s.lock.Lock()
defer s.lock.Unlock()
@ -85,7 +85,7 @@ func (s *HubTransferStats) Reset() time.Time {
defer s.lock.Unlock()
s.data.StartTime = time.Now()
s.data.Entries = make(map[cortypes.HubID]*HubTransferStatsEntry)
s.data.Entries = make(map[jcstypes.HubID]*HubTransferStatsEntry)
return s.data.StartTime
}
@ -94,7 +94,7 @@ func (s *HubTransferStats) DumpData() HubTransferStatsData {
defer s.lock.Unlock()
data := s.data
data.Entries = make(map[cortypes.HubID]*HubTransferStatsEntry)
data.Entries = make(map[jcstypes.HubID]*HubTransferStatsEntry)
for k, v := range s.data.Entries {
v2 := *v
data.Entries[k] = &v2

10
common/pkgs/servicestats/service_stats.go
View File

@ -4,7 +4,7 @@ import (
"sync"
"time"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type StatsHost struct {
@ -14,24 +14,24 @@ type StatsHost struct {
HubStorageTransfer *HubStorageTransferStats
}
func (h *StatsHost) SetupHubTransfer(fromHubID cortypes.HubID) {
func (h *StatsHost) SetupHubTransfer(fromHubID jcstypes.HubID) {
h.HubTransfer = &HubTransferStats{
fromHubID: fromHubID,
lock: &sync.Mutex{},
data: HubTransferStatsData{
StartTime: time.Now(),
Entries: make(map[cortypes.HubID]*HubTransferStatsEntry),
Entries: make(map[jcstypes.HubID]*HubTransferStatsEntry),
},
}
}
func (h *StatsHost) SetupHubStorageTransfer(fromHubID cortypes.HubID) {
func (h *StatsHost) SetupHubStorageTransfer(fromHubID jcstypes.HubID) {
h.HubStorageTransfer = &HubStorageTransferStats{
fromHubID: fromHubID,
lock: &sync.Mutex{},
data: HubStorageTransferStatsData{
StartTime: time.Now(),
Entries: make(map[cortypes.StorageID]*HubStorageTransferStatsEntry),
Entries: make(map[jcstypes.StorageID]*HubStorageTransferStatsEntry),
},
}
}

9
common/pkgs/storage/efile/ec_multiplier.go
View File

@ -10,14 +10,13 @@ import (
"gitlink.org.cn/cloudream/common/utils/os2"
"gitlink.org.cn/cloudream/common/utils/serder"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type ECMultiplier struct {
blder *builder
url string
feat *cortypes.ECMultiplierFeature
feat *jcstypes.ECMultiplierFeature
outputs []string
}
@ -98,9 +97,9 @@ func (m *ECMultiplier) Multiply(coef [][]byte, inputs []types.HTTPRequest, chunk
for i, data := range r.Data {
ret[i] = types.FileInfo{
// TODO 要确认一下output的格式
Path: jcsypes.PathFromJcsPathString(m.outputs[i]),
Path: jcstypes.PathFromJcsPathString(m.outputs[i]),
Size: data.Size,
Hash: jcsypes.NewFullHashFromString(data.Sha256),
Hash: jcstypes.NewFullHashFromString(data.Sha256),
}
}

15
common/pkgs/storage/efile/efile.go
View File

@ -10,12 +10,11 @@ import (
"gitlink.org.cn/cloudream/common/utils/serder"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func init() {
reg.RegisterBuilder[*cortypes.EFileType](func(detail *jcsypes.UserSpaceDetail) types.StorageBuilder {
reg.RegisterBuilder[*jcstypes.EFileType](func(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
return &builder{
detail: detail,
}
@ -24,15 +23,15 @@ func init() {
type builder struct {
types.EmptyBuilder
detail *jcsypes.UserSpaceDetail
detail *jcstypes.UserSpaceDetail
token string
tokenLock sync.Mutex
getTokenTime time.Time
}
func (b *builder) getToken() (string, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.EFileType)
cred := b.detail.UserSpace.Credential.(*cortypes.EFileCred)
stgType := b.detail.UserSpace.Storage.(*jcstypes.EFileType)
cred := b.detail.UserSpace.Credential.(*jcstypes.EFileCred)
b.tokenLock.Lock()
defer b.tokenLock.Unlock()
@ -91,12 +90,12 @@ func (b *builder) getToken() (string, error) {
}
func (b *builder) CreateECMultiplier(typeOnly bool) (types.ECMultiplier, error) {
feat := types.FindFeature[*cortypes.ECMultiplierFeature](b.detail)
feat := types.FindFeature[*jcstypes.ECMultiplierFeature](b.detail)
if feat == nil {
return nil, fmt.Errorf("feature ECMultiplier not found")
}
cred, ok := b.detail.UserSpace.Credential.(*cortypes.EFileCred)
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.EFileCred)
if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for efile storage", b.detail.UserSpace.Credential)
}

4
common/pkgs/storage/factory/factory.go
View File

@ -6,12 +6,12 @@ import (
_ "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
// 此函数永远不会返回nil。如果找不到对应的Builder，则会返回EmptyBuilder，
// 此Builder的所有函数都会返回否定值或者封装后的ErrUnsupported错误（需要使用errors.Is检查）
func GetBuilder(detail *jcsypes.UserSpaceDetail) types.StorageBuilder {
func GetBuilder(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
typ := reflect.TypeOf(detail.UserSpace.Storage)
ctor, ok := reg.StorageBuilders[typ]

9
common/pkgs/storage/factory/reg/reg.go
View File

@ -5,23 +5,22 @@ import (
"gitlink.org.cn/cloudream/common/utils/reflect2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type BuilderCtor func(detail *jcsypes.UserSpaceDetail) types.StorageBuilder
type BuilderCtor func(detail *jcstypes.UserSpaceDetail) types.StorageBuilder
var StorageBuilders = make(map[reflect.Type]BuilderCtor)
// 注册针对指定存储服务类型的Builder
func RegisterBuilder[T cortypes.StorageType](ctor BuilderCtor) {
func RegisterBuilder[T jcstypes.StorageType](ctor BuilderCtor) {
StorageBuilders[reflect2.TypeOf[T]()] = ctor
}
// 注：此函数只给storage包内部使用，外部包请使用外层的factory.GetBuilder
// 此函数永远不会返回nil。如果找不到对应的Builder，则会返回EmptyBuilder，
// 此Builder的所有函数都会返回否定值或者封装后的ErrUnsupported错误（需要使用errors.Is检查）
func GetBuilderInternal(detail *jcsypes.UserSpaceDetail) types.StorageBuilder {
func GetBuilderInternal(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
typ := reflect.TypeOf(detail.UserSpace.Storage)
ctor, ok := StorageBuilders[typ]

16
common/pkgs/storage/local/base_store.go
View File

@ -9,22 +9,22 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/io2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type BaseStore struct {
root string
detail *jcsypes.UserSpaceDetail
detail *jcstypes.UserSpaceDetail
}
func NewBaseStore(root string, detail *jcsypes.UserSpaceDetail) (*BaseStore, error) {
func NewBaseStore(root string, detail *jcstypes.UserSpaceDetail) (*BaseStore, error) {
return &BaseStore{
root: root,
detail: detail,
}, nil
}
func (s *BaseStore) Write(pat jcsypes.JPath, stream io.Reader, opt types.WriteOption) (types.FileInfo, error) {
func (s *BaseStore) Write(pat jcstypes.JPath, stream io.Reader, opt types.WriteOption) (types.FileInfo, error) {
log := s.getLogger()
absObjPath := filepath.Join(s.root, pat.String())
@ -58,11 +58,11 @@ func (s *BaseStore) Write(pat jcsypes.JPath, stream io.Reader, opt types.WriteOp
return types.FileInfo{
Path: pat,
Size: counter.Count(),
Hash: jcsypes.NewFullHash(hasher.Sum()),
Hash: jcstypes.NewFullHash(hasher.Sum()),
}, nil
}
func (s *BaseStore) Read(objPath jcsypes.JPath, opt types.OpenOption) (io.ReadCloser, error) {
func (s *BaseStore) Read(objPath jcstypes.JPath, opt types.OpenOption) (io.ReadCloser, error) {
absObjPath := filepath.Join(s.root, objPath.JoinOSPath())
file, err := os.Open(absObjPath)
if err != nil {
@ -86,7 +86,7 @@ func (s *BaseStore) Read(objPath jcsypes.JPath, opt types.OpenOption) (io.ReadCl
return ret, nil
}
func (s *BaseStore) Mkdir(path jcsypes.JPath) error {
func (s *BaseStore) Mkdir(path jcstypes.JPath) error {
absObjPath := filepath.Join(s.root, path.JoinOSPath())
err := os.MkdirAll(absObjPath, 0755)
if err != nil {
@ -96,7 +96,7 @@ func (s *BaseStore) Mkdir(path jcsypes.JPath) error {
return nil
}
func (s *BaseStore) ReadDir(pat jcsypes.JPath) types.DirReader {
func (s *BaseStore) ReadDir(pat jcstypes.JPath) types.DirReader {
return &DirReader{
absRootPath: filepath.Join(s.root, pat.JoinOSPath()),
rootJPath: pat.Clone(),

8
common/pkgs/storage/local/dir_reader.go
View File

@ -6,14 +6,14 @@ import (
"path/filepath"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type DirReader struct {
// 完整的根路径（包括ReadDir的path参数），比如包括了盘符
absRootPath string
// ReadDir函数传递进来的path参数
rootJPath jcsypes.JPath
rootJPath jcstypes.JPath
init bool
curEntries []dirEntry
}
@ -42,7 +42,7 @@ func (r *DirReader) Next() (types.DirEntry, error) {
for _, e := range es {
r.curEntries = append(r.curEntries, dirEntry{
dir: jcsypes.JPath{},
dir: jcstypes.JPath{},
entry: e,
})
}
@ -103,7 +103,7 @@ func (r *DirReader) Close() {
}
type dirEntry struct {
dir jcsypes.JPath
dir jcstypes.JPath
entry os.DirEntry
}

19
common/pkgs/storage/local/local.go
View File

@ -5,12 +5,11 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func init() {
reg.RegisterBuilder[*cortypes.LocalType](func(detail *jcsypes.UserSpaceDetail) types.StorageBuilder {
reg.RegisterBuilder[*jcstypes.LocalType](func(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
return &builder{
detail: detail,
}
@ -19,7 +18,7 @@ func init() {
type builder struct {
types.EmptyBuilder
detail *jcsypes.UserSpaceDetail
detail *jcstypes.UserSpaceDetail
}
func (b *builder) FeatureDesc() types.FeatureDesc {
@ -27,7 +26,7 @@ func (b *builder) FeatureDesc() types.FeatureDesc {
}
func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
cred, ok := b.detail.UserSpace.Credential.(*cortypes.LocalCred)
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.LocalCred)
if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for local storage", b.detail.UserSpace.Credential)
}
@ -40,7 +39,7 @@ func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
}
func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) {
cred, ok := b.detail.UserSpace.Credential.(*cortypes.LocalCred)
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.LocalCred)
if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for local storage", b.detail.UserSpace.Credential)
}
@ -53,9 +52,9 @@ func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) {
}
func (b *builder) CreateMultiparter(typeOnly bool) (types.Multiparter, error) {
feat := types.FindFeature[*cortypes.MultipartUploadFeature](b.detail)
feat := types.FindFeature[*jcstypes.MultipartUploadFeature](b.detail)
if feat == nil {
return nil, fmt.Errorf("feature %T not found", cortypes.MultipartUploadFeature{})
return nil, fmt.Errorf("feature %T not found", jcstypes.MultipartUploadFeature{})
}
if typeOnly {
@ -68,9 +67,9 @@ func (b *builder) CreateMultiparter(typeOnly bool) (types.Multiparter, error) {
}
func (b *builder) CreateS2STransfer(typeOnly bool) (types.S2STransfer, error) {
feat := types.FindFeature[*cortypes.S2STransferFeature](b.detail)
feat := types.FindFeature[*jcstypes.S2STransferFeature](b.detail)
if feat == nil {
return nil, fmt.Errorf("feature %T not found", cortypes.S2STransferFeature{})
return nil, fmt.Errorf("feature %T not found", jcstypes.S2STransferFeature{})
}
if typeOnly {

13
common/pkgs/storage/local/multipart_upload.go
View File

@ -13,14 +13,13 @@ import (
"gitlink.org.cn/cloudream/common/utils/os2"
"gitlink.org.cn/cloudream/common/utils/sort2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type Multiparter struct {
detail *jcsypes.UserSpaceDetail
localStg *cortypes.LocalCred
feat *cortypes.MultipartUploadFeature
detail *jcstypes.UserSpaceDetail
localStg *jcstypes.LocalCred
feat *jcstypes.MultipartUploadFeature
}
func (*Multiparter) MinPartSize() int64 {
@ -80,7 +79,7 @@ type MultipartTask struct {
absTempDir string // 应该要是绝对路径
tempFileName string
tempPartsDir string
joinedFileJPath jcsypes.JPath
joinedFileJPath jcstypes.JPath
absJoinedFilePath string
uploadID string
}
@ -118,7 +117,7 @@ func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPar
return types.FileInfo{
Path: i.joinedFileJPath,
Size: size,
Hash: jcsypes.NewFullHash(h),
Hash: jcstypes.NewFullHash(h),
}, nil
}

19
common/pkgs/storage/local/s2s.go
View File

@ -7,24 +7,23 @@ import (
"path/filepath"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type S2STransfer struct {
feat *cortypes.S2STransferFeature
detail *jcsypes.UserSpaceDetail
localStg *cortypes.LocalCred
dstPath jcsypes.JPath
feat *jcstypes.S2STransferFeature
detail *jcstypes.UserSpaceDetail
localStg *jcstypes.LocalCred
dstPath jcstypes.JPath
}
// 只有同一个机器的存储之间才可以进行数据直传
func (*S2STransfer) CanTransfer(src, dst *jcsypes.UserSpaceDetail) bool {
if types.FindFeature[*cortypes.S2STransferFeature](dst) == nil {
func (*S2STransfer) CanTransfer(src, dst *jcstypes.UserSpaceDetail) bool {
if types.FindFeature[*jcstypes.S2STransferFeature](dst) == nil {
return false
}
_, ok := src.UserSpace.Storage.(*cortypes.LocalType)
_, ok := src.UserSpace.Storage.(*jcstypes.LocalType)
if !ok {
return false
}
@ -37,7 +36,7 @@ func (*S2STransfer) CanTransfer(src, dst *jcsypes.UserSpaceDetail) bool {
}
// 执行数据直传
func (s *S2STransfer) Transfer(ctx context.Context, src *jcsypes.UserSpaceDetail, srcPath jcsypes.JPath, dstPath jcsypes.JPath) (types.FileInfo, error) {
func (s *S2STransfer) Transfer(ctx context.Context, src *jcstypes.UserSpaceDetail, srcPath jcstypes.JPath, dstPath jcstypes.JPath) (types.FileInfo, error) {
s.dstPath = dstPath
copy, err := os.OpenFile(filepath.Join(s.localStg.RootDir, s.dstPath.JoinOSPath()), os.O_WRONLY|os.O_CREATE, 0644)

24
common/pkgs/storage/local/shard_store.go
View File

@ -10,18 +10,18 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type ShardStore struct {
detail *jcsypes.UserSpaceDetail
detail *jcstypes.UserSpaceDetail
stgRoot string
storeAbsRoot string
lock sync.Mutex
done chan any
}
func NewShardStore(root string, detail *jcsypes.UserSpaceDetail) (*ShardStore, error) {
func NewShardStore(root string, detail *jcstypes.UserSpaceDetail) (*ShardStore, error) {
storeAbsRoot, err := filepath.Abs(filepath.Join(root, detail.UserSpace.WorkingDir.JoinOSPath(), types.ShardStoreWorkingDir))
if err != nil {
return nil, fmt.Errorf("get abs root: %w", err)
@ -43,7 +43,7 @@ func (s *ShardStore) Stop() {
s.getLogger().Infof("component stop")
}
func (s *ShardStore) Store(path jcsypes.JPath, hash jcsypes.FileHash, size int64) (types.FileInfo, error) {
func (s *ShardStore) Store(path jcstypes.JPath, hash jcstypes.FileHash, size int64) (types.FileInfo, error) {
fullTempPath := filepath.Join(s.stgRoot, path.JoinOSPath())
s.lock.Lock()
@ -81,7 +81,7 @@ func (s *ShardStore) Store(path jcsypes.JPath, hash jcsypes.FileHash, size int64
}, nil
}
func (s *ShardStore) Info(hash jcsypes.FileHash) (types.FileInfo, error) {
func (s *ShardStore) Info(hash jcstypes.FileHash) (types.FileInfo, error) {
s.lock.Lock()
defer s.lock.Unlock()
@ -118,7 +118,7 @@ func (s *ShardStore) ListAll() ([]types.FileInfo, error) {
return err
}
fileHash, err := jcsypes.ParseHash(filepath.Base(info.Name()))
fileHash, err := jcstypes.ParseHash(filepath.Base(info.Name()))
if err != nil {
return nil
}
@ -137,11 +137,11 @@ func (s *ShardStore) ListAll() ([]types.FileInfo, error) {
return infos, nil
}
func (s *ShardStore) GC(avaiables []jcsypes.FileHash) error {
func (s *ShardStore) GC(avaiables []jcstypes.FileHash) error {
s.lock.Lock()
defer s.lock.Unlock()
avais := make(map[jcsypes.FileHash]bool)
avais := make(map[jcstypes.FileHash]bool)
for _, hash := range avaiables {
avais[hash] = true
}
@ -162,7 +162,7 @@ func (s *ShardStore) GC(avaiables []jcsypes.FileHash) error {
return err
}
fileHash, err := jcsypes.ParseHash(filepath.Base(info.Name()))
fileHash, err := jcstypes.ParseHash(filepath.Base(info.Name()))
if err != nil {
return nil
}
@ -199,14 +199,14 @@ func (s *ShardStore) getLogger() logger.Logger {
return logger.WithField("ShardStore", "Local").WithField("Storage", s.detail.UserSpace.Storage.String())
}
func (s *ShardStore) getFileDirFromHash(hash jcsypes.FileHash) string {
func (s *ShardStore) getFileDirFromHash(hash jcstypes.FileHash) string {
return filepath.Join(s.storeAbsRoot, hash.GetHashPrefix(2))
}
func (s *ShardStore) getFilePathFromHash(hash jcsypes.FileHash) string {
func (s *ShardStore) getFilePathFromHash(hash jcstypes.FileHash) string {
return filepath.Join(s.storeAbsRoot, hash.GetHashPrefix(2), string(hash))
}
func (s *ShardStore) getJPathFromHash(hash jcsypes.FileHash) jcsypes.JPath {
func (s *ShardStore) getJPathFromHash(hash jcstypes.FileHash) jcstypes.JPath {
return s.detail.UserSpace.WorkingDir.ConcatCompsNew(types.ShardStoreWorkingDir, hash.GetHashPrefix(2), string(hash))
}

35
common/pkgs/storage/obs/obs.go
View File

@ -9,20 +9,19 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
s3stg "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/s3"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func init() {
reg.RegisterBuilder[*cortypes.OBSType](newBuilder)
reg.RegisterBuilder[*jcstypes.OBSType](newBuilder)
}
type builder struct {
types.EmptyBuilder
detail *jcsypes.UserSpaceDetail
detail *jcstypes.UserSpaceDetail
}
func newBuilder(detail *jcsypes.UserSpaceDetail) types.StorageBuilder {
func newBuilder(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
return &builder{
detail: detail,
}
@ -33,8 +32,8 @@ func (b *builder) FeatureDesc() types.FeatureDesc {
}
func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.OBSType)
cred, ok := b.detail.UserSpace.Credential.(*cortypes.OBSCred)
stgType := b.detail.UserSpace.Storage.(*jcstypes.OBSType)
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.OBSCred)
if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for obs storage", b.detail.UserSpace.Credential)
}
@ -52,8 +51,8 @@ func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
}
func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.OBSType)
cred, ok := b.detail.UserSpace.Credential.(*cortypes.OBSCred)
stgType := b.detail.UserSpace.Storage.(*jcstypes.OBSType)
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.OBSCred)
if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for obs storage", b.detail.UserSpace.Credential)
}
@ -70,7 +69,7 @@ func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) {
return s3stg.NewBaseStore(b.detail, cli, bucket, s3stg.BaseStoreOption{UseAWSSha256: false})
}
func createClient(stgType *cortypes.OBSType, cred *cortypes.OBSCred) (*s3.Client, string, error) {
func createClient(stgType *jcstypes.OBSType, cred *jcstypes.OBSCred) (*s3.Client, string, error) {
awsConfig := aws.Config{}
cre := aws.Credentials{
@ -91,17 +90,17 @@ func createClient(stgType *cortypes.OBSType, cred *cortypes.OBSCred) (*s3.Client
}
func (b *builder) CreateMultiparter(typeOnly bool) (types.Multiparter, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.OBSType)
feat := types.FindFeature[*cortypes.MultipartUploadFeature](b.detail)
stgType := b.detail.UserSpace.Storage.(*jcstypes.OBSType)
feat := types.FindFeature[*jcstypes.MultipartUploadFeature](b.detail)
if feat == nil {
return nil, fmt.Errorf("feature %T not found", cortypes.MultipartUploadFeature{})
return nil, fmt.Errorf("feature %T not found", jcstypes.MultipartUploadFeature{})
}
if typeOnly {
return (*s3stg.Multiparter)(nil), nil
}
cred, ok := b.detail.UserSpace.Credential.(*cortypes.OBSCred)
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.OBSCred)
if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for obs storage", b.detail.UserSpace.Credential)
}
@ -120,17 +119,17 @@ func (b *builder) CreateMultiparter(typeOnly bool) (types.Multiparter, error) {
}
func (b *builder) CreateS2STransfer(typeOnly bool) (types.S2STransfer, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.OBSType)
feat := types.FindFeature[*cortypes.S2STransferFeature](b.detail)
stgType := b.detail.UserSpace.Storage.(*jcstypes.OBSType)
feat := types.FindFeature[*jcstypes.S2STransferFeature](b.detail)
if feat == nil {
return nil, fmt.Errorf("feature %T not found", cortypes.S2STransferFeature{})
return nil, fmt.Errorf("feature %T not found", jcstypes.S2STransferFeature{})
}
if typeOnly {
return (*S2STransfer)(nil), nil
}
cred, ok := b.detail.UserSpace.Credential.(*cortypes.OBSCred)
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.OBSCred)
if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for obs storage", b.detail.UserSpace.Credential)
}

19
common/pkgs/storage/obs/obs_test.go
View File

@ -5,40 +5,39 @@ import (
"testing"
. "github.com/smartystreets/goconvey/convey"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func Test_S2S(t *testing.T) {
Convey("OBS", t, func() {
s2s := S2STransfer{
stgType: &cortypes.OBSType{
stgType: &jcstypes.OBSType{
Region: "cn-north-4",
Endpoint: "obs.cn-north-4.myhuaweicloud.com",
Bucket: "pcm3-bucket3",
ProjectID: "",
},
cred: &cortypes.OBSCred{
cred: &jcstypes.OBSCred{
AK: "",
SK: "",
},
feat: &cortypes.S2STransferFeature{},
feat: &jcstypes.S2STransferFeature{},
}
_, err := s2s.Transfer(context.TODO(), &jcsypes.UserSpaceDetail{
UserSpace: jcsypes.UserSpace{
Storage: &cortypes.OBSType{
_, err := s2s.Transfer(context.TODO(), &jcstypes.UserSpaceDetail{
UserSpace: jcstypes.UserSpace{
Storage: &jcstypes.OBSType{
Region: "cn-north-4",
Endpoint: "obs.cn-north-4.myhuaweicloud.com",
Bucket: "pcm2-bucket2",
ProjectID: "",
},
Credential: &cortypes.OBSCred{
Credential: &jcstypes.OBSCred{
AK: "",
SK: "",
},
},
}, jcsypes.PathFromComps("test_data/test03.txt"), jcsypes.PathFromComps("atest.txt"))
}, jcstypes.PathFromComps("test_data/test03.txt"), jcstypes.PathFromComps("atest.txt"))
defer s2s.Close()
So(err, ShouldEqual, nil)

25
common/pkgs/storage/obs/s2s.go
View File

@ -15,20 +15,19 @@ import (
"gitlink.org.cn/cloudream/common/utils/os2"
stgs3 "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/s3"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type S2STransfer struct {
detail *jcsypes.UserSpaceDetail
stgType *cortypes.OBSType
cred *cortypes.OBSCred
feat *cortypes.S2STransferFeature
detail *jcstypes.UserSpaceDetail
stgType *jcstypes.OBSType
cred *jcstypes.OBSCred
feat *jcstypes.S2STransferFeature
taskID *int64
omsCli *oms.OmsClient
}
func NewS2STransfer(detail *jcsypes.UserSpaceDetail, stgType *cortypes.OBSType, cred *cortypes.OBSCred, feat *cortypes.S2STransferFeature) *S2STransfer {
func NewS2STransfer(detail *jcstypes.UserSpaceDetail, stgType *jcstypes.OBSType, cred *jcstypes.OBSCred, feat *jcstypes.S2STransferFeature) *S2STransfer {
return &S2STransfer{
detail: detail,
stgType: stgType,
@ -38,13 +37,13 @@ func NewS2STransfer(detail *jcsypes.UserSpaceDetail, stgType *cortypes.OBSType,
}
// 判断是否能从指定的源存储中直传到当前存储的目的路径
func (*S2STransfer) CanTransfer(src, dst *jcsypes.UserSpaceDetail) bool {
req := makeRequest(src, jcsypes.JPath{})
func (*S2STransfer) CanTransfer(src, dst *jcstypes.UserSpaceDetail) bool {
req := makeRequest(src, jcstypes.JPath{})
return req != nil
}
// 执行数据直传。返回传输后的文件路径
func (s *S2STransfer) Transfer(ctx context.Context, src *jcsypes.UserSpaceDetail, srcPath jcsypes.JPath, dstPath jcsypes.JPath) (types.FileInfo, error) {
func (s *S2STransfer) Transfer(ctx context.Context, src *jcstypes.UserSpaceDetail, srcPath jcstypes.JPath, dstPath jcstypes.JPath) (types.FileInfo, error) {
req := makeRequest(src, srcPath)
if req == nil {
return types.FileInfo{}, fmt.Errorf("unsupported source storage type: %T", src.UserSpace.Storage)
@ -178,12 +177,12 @@ func (s *S2STransfer) Close() {
}
}
func makeRequest(srcStg *jcsypes.UserSpaceDetail, srcPath jcsypes.JPath) *model.SrcNodeReq {
func makeRequest(srcStg *jcstypes.UserSpaceDetail, srcPath jcstypes.JPath) *model.SrcNodeReq {
switch srcType := srcStg.UserSpace.Storage.(type) {
case *cortypes.OBSType:
case *jcstypes.OBSType:
cloudType := "HuaweiCloud"
cred, ok := srcStg.UserSpace.Credential.(*cortypes.OBSCred)
cred, ok := srcStg.UserSpace.Credential.(*jcstypes.OBSCred)
if !ok {
return nil
}

11
common/pkgs/storage/obs/shard_store.go
View File

@ -5,17 +5,16 @@ import (
"github.com/huaweicloud/huaweicloud-sdk-go-obs/obs"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/s3"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type ShardStore struct {
*s3.ShardStore
stgType *cortypes.OBSType
cred *cortypes.OBSCred
stgType *jcstypes.OBSType
cred *jcstypes.OBSCred
}
func NewShardStore(detail *jcsypes.UserSpaceDetail, stgType *cortypes.OBSType, cred *cortypes.OBSCred, s3Cli *awss3.Client, bkt string) (*ShardStore, error) {
func NewShardStore(detail *jcstypes.UserSpaceDetail, stgType *jcstypes.OBSType, cred *jcstypes.OBSCred, s3Cli *awss3.Client, bkt string) (*ShardStore, error) {
sd := ShardStore{
stgType: stgType,
cred: cred,
@ -32,7 +31,7 @@ func NewShardStore(detail *jcsypes.UserSpaceDetail, stgType *cortypes.OBSType, c
return &sd, nil
}
func (s *ShardStore) MakeHTTPReadRequest(fileHash jcsypes.FileHash) (types.HTTPRequest, error) {
func (s *ShardStore) MakeHTTPReadRequest(fileHash jcstypes.FileHash) (types.HTTPRequest, error) {
cli, err := obs.New(s.cred.AK, s.cred.SK, s.stgType.Endpoint)
if err != nil {
return types.HTTPRequest{}, err

21
common/pkgs/storage/pool/pool.go
View File

@ -6,12 +6,11 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/async"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type userSpace struct {
detail *jcsypes.UserSpaceDetail
detail *jcstypes.UserSpaceDetail
store types.ShardStore
}
@ -23,8 +22,8 @@ func (u *userSpace) Drop() {
}
type userSpaceKey struct {
UserID cortypes.UserID
UserSpaceID jcsypes.UserSpaceID
UserID jcstypes.UserID
UserSpaceID jcstypes.UserSpaceID
}
type Pool struct {
@ -40,7 +39,7 @@ func NewPool() *Pool {
}
}
func (p *Pool) Drop(userID cortypes.UserID, spaceID jcsypes.UserSpaceID) {
func (p *Pool) Drop(userID jcstypes.UserID, spaceID jcstypes.UserSpaceID) {
p.lock.Lock()
defer p.lock.Unlock()
@ -56,7 +55,7 @@ func (p *Pool) Drop(userID cortypes.UserID, spaceID jcsypes.UserSpaceID) {
delete(p.spaces, key)
}
func (p *Pool) GetShardStore(spaceDetail *jcsypes.UserSpaceDetail) (types.ShardStore, error) {
func (p *Pool) GetShardStore(spaceDetail *jcstypes.UserSpaceDetail) (types.ShardStore, error) {
p.lock.Lock()
defer p.lock.Unlock()
@ -91,18 +90,18 @@ func (p *Pool) GetShardStore(spaceDetail *jcsypes.UserSpaceDetail) (types.ShardS
return space.store, nil
}
func (p *Pool) GetBaseStore(spaceDetail *jcsypes.UserSpaceDetail) (types.BaseStore, error) {
func (p *Pool) GetBaseStore(spaceDetail *jcstypes.UserSpaceDetail) (types.BaseStore, error) {
return factory.GetBuilder(spaceDetail).CreateBaseStore(false)
}
func (p *Pool) GetMultiparter(spaceDetail *jcsypes.UserSpaceDetail) (types.Multiparter, error) {
func (p *Pool) GetMultiparter(spaceDetail *jcstypes.UserSpaceDetail) (types.Multiparter, error) {
return factory.GetBuilder(spaceDetail).CreateMultiparter(false)
}
func (p *Pool) GetS2STransfer(spaceDetail *jcsypes.UserSpaceDetail) (types.S2STransfer, error) {
func (p *Pool) GetS2STransfer(spaceDetail *jcstypes.UserSpaceDetail) (types.S2STransfer, error) {
return factory.GetBuilder(spaceDetail).CreateS2STransfer(false)
}
func (p *Pool) GetECMultiplier(spaceDetail *jcsypes.UserSpaceDetail) (types.ECMultiplier, error) {
func (p *Pool) GetECMultiplier(spaceDetail *jcstypes.UserSpaceDetail) (types.ECMultiplier, error) {
return factory.GetBuilder(spaceDetail).CreateECMultiplier(false)
}

18
common/pkgs/storage/s3/base_store.go
View File

@ -16,7 +16,7 @@ import (
"gitlink.org.cn/cloudream/common/utils/io2"
"gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
const (
@ -24,7 +24,7 @@ const (
)
type BaseStore struct {
Detail *jcsypes.UserSpaceDetail
Detail *jcstypes.UserSpaceDetail
Bucket string
cli *s3.Client
opt BaseStoreOption
@ -34,7 +34,7 @@ type BaseStoreOption struct {
UseAWSSha256 bool // 能否直接使用AWS提供的SHA256校验，如果不行，则使用本地计算。默认使用本地计算。
}
func NewBaseStore(detail *jcsypes.UserSpaceDetail, cli *s3.Client, bkt string, opt BaseStoreOption) (*BaseStore, error) {
func NewBaseStore(detail *jcstypes.UserSpaceDetail, cli *s3.Client, bkt string, opt BaseStoreOption) (*BaseStore, error) {
return &BaseStore{
Detail: detail,
Bucket: bkt,
@ -43,7 +43,7 @@ func NewBaseStore(detail *jcsypes.UserSpaceDetail, cli *s3.Client, bkt string, o
}, nil
}
func (s *BaseStore) Write(pat jcsypes.JPath, stream io.Reader, opt types.WriteOption) (types.FileInfo, error) {
func (s *BaseStore) Write(pat jcstypes.JPath, stream io.Reader, opt types.WriteOption) (types.FileInfo, error) {
key := pat
meta := make(map[string]string)
if opt.ModTime.IsZero() {
@ -81,7 +81,7 @@ func (s *BaseStore) Write(pat jcsypes.JPath, stream io.Reader, opt types.WriteOp
return types.FileInfo{
Path: key,
Hash: jcsypes.NewFullHash(hash),
Hash: jcstypes.NewFullHash(hash),
Size: counter.Count(),
}, nil
}
@ -99,12 +99,12 @@ func (s *BaseStore) Write(pat jcsypes.JPath, stream io.Reader, opt types.WriteOp
return types.FileInfo{
Path: key,
Hash: jcsypes.NewFullHash(hashStr.Sum()),
Hash: jcstypes.NewFullHash(hashStr.Sum()),
Size: counter.Count(),
}, nil
}
func (s *BaseStore) Read(pat jcsypes.JPath, opt types.OpenOption) (io.ReadCloser, error) {
func (s *BaseStore) Read(pat jcstypes.JPath, opt types.OpenOption) (io.ReadCloser, error) {
key := pat
input := &s3.GetObjectInput{
@ -128,7 +128,7 @@ func (s *BaseStore) Read(pat jcsypes.JPath, opt types.OpenOption) (io.ReadCloser
return resp.Body, nil
}
func (s *BaseStore) Mkdir(path jcsypes.JPath) error {
func (s *BaseStore) Mkdir(path jcstypes.JPath) error {
_, err := s.cli.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(s.Bucket),
Key: aws.String(path.String() + "/"),
@ -137,7 +137,7 @@ func (s *BaseStore) Mkdir(path jcsypes.JPath) error {
return err
}
func (s *BaseStore) ReadDir(path jcsypes.JPath) types.DirReader {
func (s *BaseStore) ReadDir(path jcstypes.JPath) types.DirReader {
return &DirReader{
cli: s.cli,
bucket: s.Bucket,

6
common/pkgs/storage/s3/dir_reader.go
View File

@ -7,13 +7,13 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type DirReader struct {
cli *s3.Client
bucket string
rootPath jcsypes.JPath
rootPath jcstypes.JPath
marker *string
curInfos []types.DirEntry
eof bool
@ -39,7 +39,7 @@ func (r *DirReader) Next() (types.DirEntry, error) {
}
for _, obj := range resp.Contents {
key := jcsypes.PathFromJcsPathString(*obj.Key)
key := jcstypes.PathFromJcsPathString(*obj.Key)
r.curInfos = append(r.curInfos, types.DirEntry{
Path: key,

15
common/pkgs/storage/s3/multipart_upload.go
View File

@ -12,18 +12,17 @@ import (
"gitlink.org.cn/cloudream/common/utils/os2"
"gitlink.org.cn/cloudream/common/utils/sort2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type Multiparter struct {
detail *jcsypes.UserSpaceDetail
feat *cortypes.MultipartUploadFeature
detail *jcstypes.UserSpaceDetail
feat *jcstypes.MultipartUploadFeature
bucket string
cli *s3.Client
}
func NewMultiparter(detail *jcsypes.UserSpaceDetail, feat *cortypes.MultipartUploadFeature, bkt string, cli *s3.Client) *Multiparter {
func NewMultiparter(detail *jcstypes.UserSpaceDetail, feat *jcstypes.MultipartUploadFeature, bkt string, cli *s3.Client) *Multiparter {
return &Multiparter{
detail: detail,
feat: feat,
@ -89,9 +88,9 @@ func (m *Multiparter) UploadPart(ctx context.Context, init types.MultipartInitSt
type MultipartTask struct {
cli *s3.Client
bucket string
tempDir jcsypes.JPath
tempDir jcstypes.JPath
tempFileName string
tempFilePath jcsypes.JPath
tempFilePath jcstypes.JPath
uploadID string
}
@ -140,7 +139,7 @@ func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPar
return types.FileInfo{}, err
}
hash := jcsypes.CalculateCompositeHash(partHashes)
hash := jcstypes.CalculateCompositeHash(partHashes)
return types.FileInfo{
Path: i.tempFilePath,

27
common/pkgs/storage/s3/s3.go
View File

@ -8,20 +8,19 @@ import (
"github.com/aws/aws-sdk-go-v2/service/s3"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func init() {
reg.RegisterBuilder[*cortypes.S3Type](newBuilder)
reg.RegisterBuilder[*jcstypes.S3Type](newBuilder)
}
type builder struct {
types.EmptyBuilder
detail *jcsypes.UserSpaceDetail
detail *jcstypes.UserSpaceDetail
}
func newBuilder(detail *jcsypes.UserSpaceDetail) types.StorageBuilder {
func newBuilder(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
return &builder{
detail: detail,
}
@ -32,8 +31,8 @@ func (b *builder) FeatureDesc() types.FeatureDesc {
}
func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.S3Type)
s3Cred, ok := b.detail.UserSpace.Credential.(*cortypes.S3Cred)
stgType := b.detail.UserSpace.Storage.(*jcstypes.S3Type)
s3Cred, ok := b.detail.UserSpace.Credential.(*jcstypes.S3Cred)
if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for s3 storage", b.detail.UserSpace.Credential)
}
@ -51,8 +50,8 @@ func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
}
func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.S3Type)
s3Cred, ok := b.detail.UserSpace.Credential.(*cortypes.S3Cred)
stgType := b.detail.UserSpace.Storage.(*jcstypes.S3Type)
s3Cred, ok := b.detail.UserSpace.Credential.(*jcstypes.S3Cred)
if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for s3 storage", b.detail.UserSpace.Credential)
}
@ -69,7 +68,7 @@ func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) {
return NewBaseStore(b.detail, cli, bkt, BaseStoreOption{UseAWSSha256: false})
}
func createClient(stgType *cortypes.S3Type, cred *cortypes.S3Cred) (*s3.Client, string, error) {
func createClient(stgType *jcstypes.S3Type, cred *jcstypes.S3Cred) (*s3.Client, string, error) {
awsConfig := aws.Config{}
if cred.AK != "" && cred.SK != "" {
@ -92,13 +91,13 @@ func createClient(stgType *cortypes.S3Type, cred *cortypes.S3Cred) (*s3.Client,
}
func (b *builder) CreateMultiparter(typeOnly bool) (types.Multiparter, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.S3Type)
feat := types.FindFeature[*cortypes.MultipartUploadFeature](b.detail)
stgType := b.detail.UserSpace.Storage.(*jcstypes.S3Type)
feat := types.FindFeature[*jcstypes.MultipartUploadFeature](b.detail)
if feat == nil {
return nil, fmt.Errorf("feature %T not found", cortypes.MultipartUploadFeature{})
return nil, fmt.Errorf("feature %T not found", jcstypes.MultipartUploadFeature{})
}
s3Cred, ok := b.detail.UserSpace.Credential.(*cortypes.S3Cred)
s3Cred, ok := b.detail.UserSpace.Credential.(*jcstypes.S3Cred)
if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for s3 base store", b.detail.UserSpace.Credential)
}

Some files were not shown because too many files have changed in this diff Show More