JointCloud
/
JCS-pub
24
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ShardStore只负责管理元数据，读写功能统一交给BaseStore

This commit is contained in:
Sydonian 2025-05-22 16:43:53 +08:00
parent 682c3f5eb3
commit 9cd5631627
44 changed files with 1381 additions and 1988 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
client/internal/services/user_space.go
View File

@ -160,7 +160,7 @@ func (svc *UserSpaceService) SpaceToSpace(srcSpaceID clitypes.UserSpaceID, srcPa
srcPathComps := clitypes.SplitObjectPath(srcPath)
srcDirCompLen := len(srcPathComps) - 1
entryTree := trie.NewTrie[*types.BaseStoreEntry]()
entryTree := trie.NewTrie[*types.ListEntry]()
for _, e := range listAllResp.Entries {
pa, ok := strings.CutSuffix(e.Path, clitypes.ObjectPathSeparator)
comps := clitypes.SplitObjectPath(pa)
@ -171,7 +171,7 @@ func (svc *UserSpaceService) SpaceToSpace(srcSpaceID clitypes.UserSpaceID, srcPa
e2.IsDir = e2.IsDir || ok
}
entryTree.Iterate(func(path []string, node *trie.Node[*types.BaseStoreEntry], isWordNode bool) trie.VisitCtrl {
entryTree.Iterate(func(path []string, node *trie.Node[*types.ListEntry], isWordNode bool) trie.VisitCtrl {
if node.Value == nil {
return trie.VisitContinue
}
@ -190,7 +190,7 @@ func (svc *UserSpaceService) SpaceToSpace(srcSpaceID clitypes.UserSpaceID, srcPa
var filePathes []string
var dirPathes []string
entryTree.Iterate(func(path []string, node *trie.Node[*types.BaseStoreEntry], isWordNode bool) trie.VisitCtrl {
entryTree.Iterate(func(path []string, node *trie.Node[*types.ListEntry], isWordNode bool) trie.VisitCtrl {
if node.Value == nil {
return trie.VisitContinue
}

2
client/internal/uploader/user_space_upload.go
View File

@ -122,7 +122,7 @@ func (u *Uploader) UserSpaceUpload(userSpaceID clitypes.UserSpaceID, rootPath st
return &pkg, nil
}
func (u *Uploader) uploadFromBaseStore(srcSpace *clitypes.UserSpaceDetail, targetSpace *clitypes.UserSpaceDetail, entries []types.BaseStoreEntry, rootPath string) ([]db.AddObjectEntry, error) {
func (u *Uploader) uploadFromBaseStore(srcSpace *clitypes.UserSpaceDetail, targetSpace *clitypes.UserSpaceDetail, entries []types.ListEntry, rootPath string) ([]db.AddObjectEntry, error) {
ft := ioswitch2.FromTo{}
for _, e := range entries {

34
common/pkgs/ioswitch2/fromto.go
View File

@ -108,21 +108,21 @@ func (f *FromDriver) GetStreamIndex() StreamIndex {
return f.StreamIndex
}
type FromShardstore struct {
type FromShardStore struct {
FileHash clitypes.FileHash
UserSpace clitypes.UserSpaceDetail
StreamIndex StreamIndex
}
func NewFromShardstore(fileHash clitypes.FileHash, space clitypes.UserSpaceDetail, strIdx StreamIndex) *FromShardstore {
return &FromShardstore{
func NewFromShardstore(fileHash clitypes.FileHash, space clitypes.UserSpaceDetail, strIdx StreamIndex) *FromShardStore {
return &FromShardStore{
FileHash: fileHash,
UserSpace: space,
StreamIndex: strIdx,
}
}
func (f *FromShardstore) GetStreamIndex() StreamIndex {
func (f *FromShardStore) GetStreamIndex() StreamIndex {
return f.StreamIndex
}
@ -176,26 +176,26 @@ func (t *ToDriver) GetRange() math2.Range {
}
type ToShardStore struct {
Space clitypes.UserSpaceDetail
StreamIndex StreamIndex
Range math2.Range
FileHashStoreKey string
Space clitypes.UserSpaceDetail
StreamIndex StreamIndex
Range math2.Range
ResultStoreKey string
}
func NewToShardStore(space clitypes.UserSpaceDetail, strIdx StreamIndex, fileHashStoreKey string) *ToShardStore {
func NewToShardStore(space clitypes.UserSpaceDetail, strIdx StreamIndex, retStoreKey string) *ToShardStore {
return &ToShardStore{
Space: space,
StreamIndex: strIdx,
FileHashStoreKey: fileHashStoreKey,
Space: space,
StreamIndex: strIdx,
ResultStoreKey: retStoreKey,
}
}
func NewToShardStoreWithRange(space clitypes.UserSpaceDetail, streamIndex StreamIndex, fileHashStoreKey string, rng math2.Range) *ToShardStore {
func NewToShardStoreWithRange(space clitypes.UserSpaceDetail, streamIndex StreamIndex, retStoreKey string, rng math2.Range) *ToShardStore {
return &ToShardStore{
Space: space,
StreamIndex: streamIndex,
FileHashStoreKey: fileHashStoreKey,
Range: rng,
Space: space,
StreamIndex: streamIndex,
ResultStoreKey: retStoreKey,
Range: rng,
}
}

144
common/pkgs/ioswitch2/ops2/base_store.go
View File

@ -12,27 +12,19 @@ import (
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
)
func init() {
exec.UseOp[*BaseWrite]()
exec.UseOp[*BaseRead]()
exec.UseVarValue[*FileInfoValue]()
}
type FileInfoValue struct {
Hash clitypes.FileHash `json:"hash"`
Size int64 `json:"size"`
}
func (v *FileInfoValue) Clone() exec.VarValue {
return &FileInfoValue{Hash: v.Hash, Size: v.Size}
}
type BaseRead struct {
Output exec.VarID
UserSpace clitypes.UserSpaceDetail
Path string
Option types.OpenOption
}
func (o *BaseRead) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
@ -53,7 +45,7 @@ func (o *BaseRead) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
return fmt.Errorf("getting base store of storage %v: %w", o.UserSpace, err)
}
stream, err := store.Read(o.Path)
stream, err := store.Read(o.Path, o.Option)
if err != nil {
return fmt.Errorf("reading object %v: %w", o.Path, err)
}
@ -73,6 +65,56 @@ func (o *BaseRead) String() string {
return fmt.Sprintf("PublicRead %v:%v -> %v", o.UserSpace, o.Path, o.Output)
}
type BaseReadDyn struct {
UserSpace clitypes.UserSpaceDetail
Output exec.VarID
FileInfo exec.VarID
Option types.OpenOption
}
func (o *BaseReadDyn) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
logger.
WithField("Output", o.Output).
WithField("UserSpace", o.UserSpace).
WithField("Path", o.FileInfo).
Debug("base read dynamic")
defer logger.Debug("base read dynamic end")
stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
if err != nil {
return fmt.Errorf("getting storage pool: %w", err)
}
info, err := exec.BindVar[*FileInfoValue](e, ctx.Context, o.FileInfo)
if err != nil {
return err
}
store, err := stgPool.GetBaseStore(&o.UserSpace)
if err != nil {
return fmt.Errorf("getting base store of storage %v: %w", o.UserSpace, err)
}
stream, err := store.Read(info.Path, o.Option)
if err != nil {
return fmt.Errorf("reading object %v: %w", o.FileInfo, err)
}
fut := future.NewSetVoid()
output := &exec.StreamValue{
Stream: io2.AfterReadClosed(stream, func(closer io.ReadCloser) {
fut.SetVoid()
}),
}
e.PutVar(o.Output, output)
return fut.Wait(ctx.Context)
}
func (o *BaseReadDyn) String() string {
return fmt.Sprintf("BaseReadDyn %v:%v -> %v", o.UserSpace, o.FileInfo, o.Output)
}
type BaseWrite struct {
Input exec.VarID
UserSpace clitypes.UserSpaceDetail
@ -102,14 +144,13 @@ func (o *BaseWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
}
defer input.Stream.Close()
info, err := store.Write(o.Path, input.Stream)
ret, err := store.Write(o.Path, input.Stream)
if err != nil {
return err
}
e.PutVar(o.FileInfo, &FileInfoValue{
Hash: info.Hash,
Size: info.Size,
FileInfo: ret,
})
return nil
}
@ -123,13 +164,15 @@ type BaseReadNode struct {
From ioswitch2.From
UserSpace clitypes.UserSpaceDetail
Path string
Option types.OpenOption
}
func (b *GraphNodeBuilder) NewPublicRead(from ioswitch2.From, userSpace clitypes.UserSpaceDetail, path string) *BaseReadNode {
func (b *GraphNodeBuilder) NewBaseRead(from ioswitch2.From, userSpace clitypes.UserSpaceDetail, path string, opt types.OpenOption) *BaseReadNode {
node := &BaseReadNode{
From: from,
UserSpace: userSpace,
Path: path,
Option: opt,
}
b.AddNode(node)
@ -153,18 +196,65 @@ func (t *BaseReadNode) GenerateOp() (exec.Op, error) {
Output: t.Output().Var().VarID,
UserSpace: t.UserSpace,
Path: t.Path,
Option: t.Option,
}, nil
}
type BaseReadDynNode struct {
dag.NodeBase
From ioswitch2.From
UserSpace clitypes.UserSpaceDetail
Option types.OpenOption
}
func (b *GraphNodeBuilder) NewBaseReadDyn(from ioswitch2.From, userSpace clitypes.UserSpaceDetail, opt types.OpenOption) *BaseReadDynNode {
node := &BaseReadDynNode{
From: from,
UserSpace: userSpace,
Option: opt,
}
b.AddNode(node)
node.OutputStreams().Init(node, 1)
node.InputValues().Init(1)
return node
}
func (t *BaseReadDynNode) GetFrom() ioswitch2.From {
return t.From
}
func (t *BaseReadDynNode) FileInfoSlot() dag.ValueInputSlot {
return dag.ValueInputSlot{
Node: t,
Index: 0,
}
}
func (t *BaseReadDynNode) Output() dag.StreamOutputSlot {
return dag.StreamOutputSlot{
Node: t,
Index: 0,
}
}
func (t *BaseReadDynNode) GenerateOp() (exec.Op, error) {
return &BaseReadDyn{
UserSpace: t.UserSpace,
Output: t.Output().Var().VarID,
FileInfo: t.FileInfoSlot().Var().VarID,
Option: t.Option,
}, nil
}
type BaseWriteNode struct {
dag.NodeBase
To ioswitch2.To
UserSpace clitypes.UserSpaceDetail
Path string
FileInfoStoreKey string
To ioswitch2.To
UserSpace clitypes.UserSpaceDetail
Path string
}
func (b *GraphNodeBuilder) NewPublicWrite(to ioswitch2.To, userSpace clitypes.UserSpaceDetail, path string) *BaseWriteNode {
func (b *GraphNodeBuilder) NewBaseWrite(to ioswitch2.To, userSpace clitypes.UserSpaceDetail, path string) *BaseWriteNode {
node := &BaseWriteNode{
To: to,
UserSpace: userSpace,
@ -173,6 +263,7 @@ func (b *GraphNodeBuilder) NewPublicWrite(to ioswitch2.To, userSpace clitypes.Us
b.AddNode(node)
node.InputStreams().Init(1)
node.OutputValues().Init(node, 1)
return node
}
@ -180,10 +271,6 @@ func (t *BaseWriteNode) GetTo() ioswitch2.To {
return t.To
}
func (t *BaseWriteNode) SetInput(input *dag.StreamVar) {
input.To(t, 0)
}
func (t *BaseWriteNode) Input() dag.StreamInputSlot {
return dag.StreamInputSlot{
Node: t,
@ -191,8 +278,11 @@ func (t *BaseWriteNode) Input() dag.StreamInputSlot {
}
}
func (t *BaseWriteNode) FileInfoVar() *dag.ValueVar {
return t.OutputValues().Get(0)
func (t *BaseWriteNode) FileInfoVar() dag.ValueOutputSlot {
return dag.ValueOutputSlot{
Node: t,
Index: 0,
}
}
func (t *BaseWriteNode) GenerateOp() (exec.Op, error) {
@ -200,6 +290,6 @@ func (t *BaseWriteNode) GenerateOp() (exec.Op, error) {
Input: t.InputStreams().Get(0).VarID,
UserSpace: t.UserSpace,
Path: t.Path,
FileInfo: t.FileInfoVar().VarID,
FileInfo: t.FileInfoVar().Var().VarID,
}, nil
}

374
common/pkgs/ioswitch2/ops2/bypass.go
View File

@ -11,205 +11,12 @@ import (
)
func init() {
exec.UseOp[*BypassToShardStore]()
exec.UseOp[*BypassToBaseStore]()
exec.UseVarValue[*BypassedFileInfoValue]()
exec.UseVarValue[*BypassHandleResultValue]()
exec.UseOp[*BypassFromShardStore]()
exec.UseOp[*BypassFromBaseStore]()
exec.UseVarValue[*BypassFilePathValue]()
exec.UseOp[*BypassFromShardStoreHTTP]()
exec.UseOp[*GetShardHTTPRequest]()
exec.UseVarValue[*HTTPRequestValue]()
}
type BypassedFileInfoValue struct {
types.BypassedFileInfo
}
func (v *BypassedFileInfoValue) Clone() exec.VarValue {
return &BypassedFileInfoValue{
BypassedFileInfo: v.BypassedFileInfo,
}
}
type BypassHandleResultValue struct {
Commited bool
}
func (r *BypassHandleResultValue) Clone() exec.VarValue {
return &BypassHandleResultValue{
Commited: r.Commited,
}
}
type BypassToShardStore struct {
UserSpace clitypes.UserSpaceDetail
BypassFileInfo exec.VarID
BypassCallback exec.VarID
FileInfo exec.VarID
}
func (o *BypassToShardStore) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
if err != nil {
return err
}
shardStore, err := stgPool.GetShardStore(&o.UserSpace)
if err != nil {
return err
}
br, ok := shardStore.(types.BypassShardWrite)
if !ok {
return fmt.Errorf("shard store %v not support bypass write", o.UserSpace)
}
fileInfo, err := exec.BindVar[*BypassedFileInfoValue](e, ctx.Context, o.BypassFileInfo)
if err != nil {
return err
}
err = br.BypassedShard(fileInfo.BypassedFileInfo)
if err != nil {
return err
}
e.PutVar(o.BypassCallback, &BypassHandleResultValue{Commited: true})
e.PutVar(o.FileInfo, &FileInfoValue{Hash: fileInfo.Hash, Size: fileInfo.Size})
return nil
}
func (o *BypassToShardStore) String() string {
return fmt.Sprintf("BypassToShardStore[UserSpace:%v] Info: %v, Callback: %v", o.UserSpace, o.BypassFileInfo, o.BypassCallback)
}
type BypassToBaseStore struct {
UserSpace clitypes.UserSpaceDetail
BypassFileInfo exec.VarID
BypassCallback exec.VarID
DestPath string
}
func (o *BypassToBaseStore) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
if err != nil {
return err
}
store, err := stgPool.GetBaseStore(&o.UserSpace)
if err != nil {
return err
}
br, ok := store.(types.BypassPublicWrite)
if !ok {
return fmt.Errorf("base store %v not support bypass write", o.UserSpace)
}
fileInfo, err := exec.BindVar[*BypassedFileInfoValue](e, ctx.Context, o.BypassFileInfo)
if err != nil {
return err
}
err = br.BypassedPublic(fileInfo.BypassedFileInfo, o.DestPath)
if err != nil {
return err
}
e.PutVar(o.BypassCallback, &BypassHandleResultValue{Commited: true})
return nil
}
func (o *BypassToBaseStore) String() string {
return fmt.Sprintf("BypassToBaseStore[UserSpace:%v] Info: %v, Callback: %v", o.UserSpace, o.BypassFileInfo, o.BypassCallback)
}
type BypassFilePathValue struct {
types.BypassFilePath
}
func (v *BypassFilePathValue) Clone() exec.VarValue {
return &BypassFilePathValue{
BypassFilePath: v.BypassFilePath,
}
}
type BypassFromShardStore struct {
UserSpace clitypes.UserSpaceDetail
FileHash clitypes.FileHash
Output exec.VarID
}
func (o *BypassFromShardStore) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
if err != nil {
return err
}
shardStore, err := stgPool.GetShardStore(&o.UserSpace)
if err != nil {
return err
}
br, ok := shardStore.(types.BypassShardRead)
if !ok {
return fmt.Errorf("shard store %v not support bypass read", o.UserSpace)
}
path, err := br.BypassShardRead(o.FileHash)
if err != nil {
return err
}
e.PutVar(o.Output, &BypassFilePathValue{BypassFilePath: path})
return nil
}
func (o *BypassFromShardStore) String() string {
return fmt.Sprintf("BypassFromShardStore[UserSpace:%v] FileHash: %v, Output: %v", o.UserSpace, o.FileHash, o.Output)
}
type BypassFromBaseStore struct {
UserSpace clitypes.UserSpaceDetail
Path string
Output exec.VarID
}
func (o *BypassFromBaseStore) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
if err != nil {
return err
}
store, err := stgPool.GetBaseStore(&o.UserSpace)
if err != nil {
return err
}
br, ok := store.(types.BypassPublicRead)
if !ok {
return fmt.Errorf("base store %v not support bypass read", o.UserSpace)
}
path, err := br.BypassPublicRead(o.Path)
if err != nil {
return err
}
e.PutVar(o.Output, &BypassFilePathValue{BypassFilePath: path})
return nil
}
func (o *BypassFromBaseStore) String() string {
return fmt.Sprintf("BypassFromBaseStore[UserSpace:%v] Path: %v, Output: %v", o.UserSpace, o.Path, o.Output)
}
// 旁路Http读取
type BypassFromShardStoreHTTP struct {
type GetShardHTTPRequest struct {
UserSpace clitypes.UserSpaceDetail
FileHash clitypes.FileHash
Output exec.VarID
@ -225,7 +32,7 @@ func (v *HTTPRequestValue) Clone() exec.VarValue {
}
}
func (o *BypassFromShardStoreHTTP) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
func (o *GetShardHTTPRequest) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
if err != nil {
return err
@ -236,12 +43,12 @@ func (o *BypassFromShardStoreHTTP) Execute(ctx *exec.ExecContext, e *exec.Execut
return err
}
br, ok := shardStore.(types.HTTPBypassShardRead)
br, ok := shardStore.(types.HTTPShardRead)
if !ok {
return fmt.Errorf("shard store %v not support bypass read", o.UserSpace)
}
req, err := br.HTTPBypassRead(o.FileHash)
req, err := br.MakeHTTPReadRequest(o.FileHash)
if err != nil {
return err
}
@ -250,174 +57,19 @@ func (o *BypassFromShardStoreHTTP) Execute(ctx *exec.ExecContext, e *exec.Execut
return nil
}
func (o *BypassFromShardStoreHTTP) String() string {
return fmt.Sprintf("BypassFromShardStoreHTTP[UserSpace:%v] FileHash: %v, Output: %v", o.UserSpace, o.FileHash, o.Output)
}
// 旁路写入
type BypassToShardStoreNode struct {
dag.NodeBase
UserSpace clitypes.UserSpaceDetail
FileHashStoreKey string
}
func (b *GraphNodeBuilder) NewBypassToShardStore(userSpace clitypes.UserSpaceDetail, fileHashStoreKey string) *BypassToShardStoreNode {
node := &BypassToShardStoreNode{
UserSpace: userSpace,
FileHashStoreKey: fileHashStoreKey,
}
b.AddNode(node)
node.InputValues().Init(1)
node.OutputValues().Init(node, 2)
return node
}
func (n *BypassToShardStoreNode) BypassFileInfoSlot() dag.ValueInputSlot {
return dag.ValueInputSlot{
Node: n,
Index: 0,
}
}
func (n *BypassToShardStoreNode) BypassCallbackVar() dag.ValueOutputSlot {
return dag.ValueOutputSlot{
Node: n,
Index: 0,
}
}
func (n *BypassToShardStoreNode) FileHashVar() dag.ValueOutputSlot {
return dag.ValueOutputSlot{
Node: n,
Index: 1,
}
}
func (t *BypassToShardStoreNode) GenerateOp() (exec.Op, error) {
return &BypassToShardStore{
UserSpace: t.UserSpace,
BypassFileInfo: t.BypassFileInfoSlot().Var().VarID,
BypassCallback: t.BypassCallbackVar().Var().VarID,
FileInfo: t.FileHashVar().Var().VarID,
}, nil
}
type BypassToBaseStoreNode struct {
dag.NodeBase
UserSpace clitypes.UserSpaceDetail
DestPath string
}
func (b *GraphNodeBuilder) NewBypassToBaseStore(userSpace clitypes.UserSpaceDetail, dstPath string) *BypassToBaseStoreNode {
node := &BypassToBaseStoreNode{
UserSpace: userSpace,
DestPath: dstPath,
}
b.AddNode(node)
node.InputValues().Init(1)
node.OutputValues().Init(node, 1)
return node
}
func (n *BypassToBaseStoreNode) BypassFileInfoSlot() dag.ValueInputSlot {
return dag.ValueInputSlot{
Node: n,
Index: 0,
}
}
func (n *BypassToBaseStoreNode) BypassCallbackVar() dag.ValueOutputSlot {
return dag.ValueOutputSlot{
Node: n,
Index: 0,
}
}
func (t *BypassToBaseStoreNode) GenerateOp() (exec.Op, error) {
return &BypassToBaseStore{
UserSpace: t.UserSpace,
BypassFileInfo: t.BypassFileInfoSlot().Var().VarID,
BypassCallback: t.BypassCallbackVar().Var().VarID,
DestPath: t.DestPath,
}, nil
}
// 旁路读取
type BypassFromShardStoreNode struct {
dag.NodeBase
UserSpace clitypes.UserSpaceDetail
FileHash clitypes.FileHash
}
func (b *GraphNodeBuilder) NewBypassFromShardStore(userSpace clitypes.UserSpaceDetail, fileHash clitypes.FileHash) *BypassFromShardStoreNode {
node := &BypassFromShardStoreNode{
UserSpace: userSpace,
FileHash: fileHash,
}
b.AddNode(node)
node.OutputValues().Init(node, 1)
return node
}
func (n *BypassFromShardStoreNode) FilePathVar() dag.ValueOutputSlot {
return dag.ValueOutputSlot{
Node: n,
Index: 0,
}
}
func (n *BypassFromShardStoreNode) GenerateOp() (exec.Op, error) {
return &BypassFromShardStore{
UserSpace: n.UserSpace,
FileHash: n.FileHash,
Output: n.FilePathVar().Var().VarID,
}, nil
}
type BypassFromBaseStoreNode struct {
dag.NodeBase
UserSpace clitypes.UserSpaceDetail
Path string
}
func (b *GraphNodeBuilder) NewBypassFromBaseStore(userSpace clitypes.UserSpaceDetail, path string) *BypassFromBaseStoreNode {
node := &BypassFromBaseStoreNode{
UserSpace: userSpace,
Path: path,
}
b.AddNode(node)
node.OutputValues().Init(node, 1)
return node
}
func (n *BypassFromBaseStoreNode) FilePathVar() dag.ValueOutputSlot {
return dag.ValueOutputSlot{
Node: n,
Index: 0,
}
}
func (n *BypassFromBaseStoreNode) GenerateOp() (exec.Op, error) {
return &BypassFromBaseStore{
UserSpace: n.UserSpace,
Path: n.Path,
Output: n.FilePathVar().Var().VarID,
}, nil
func (o *GetShardHTTPRequest) String() string {
return fmt.Sprintf("GetShardHTTPRequest[UserSpace:%v] FileHash: %v, Output: %v", o.UserSpace, o.FileHash, o.Output)
}
// 旁路Http读取
type BypassFromShardStoreHTTPNode struct {
type GetShardHTTPRequestNode struct {
dag.NodeBase
UserSpace clitypes.UserSpaceDetail
FileHash clitypes.FileHash
}
func (b *GraphNodeBuilder) NewBypassFromShardStoreHTTP(userSpace clitypes.UserSpaceDetail, fileHash clitypes.FileHash) *BypassFromShardStoreHTTPNode {
node := &BypassFromShardStoreHTTPNode{
func (b *GraphNodeBuilder) NewGetShardHTTPRequest(userSpace clitypes.UserSpaceDetail, fileHash clitypes.FileHash) *GetShardHTTPRequestNode {
node := &GetShardHTTPRequestNode{
UserSpace: userSpace,
FileHash: fileHash,
}
@ -427,15 +79,15 @@ func (b *GraphNodeBuilder) NewBypassFromShardStoreHTTP(userSpace clitypes.UserSp
return node
}
func (n *BypassFromShardStoreHTTPNode) HTTPRequestVar() dag.ValueOutputSlot {
func (n *GetShardHTTPRequestNode) HTTPRequestVar() dag.ValueOutputSlot {
return dag.ValueOutputSlot{
Node: n,
Index: 0,
}
}
func (n *BypassFromShardStoreHTTPNode) GenerateOp() (exec.Op, error) {
return &BypassFromShardStoreHTTP{
func (n *GetShardHTTPRequestNode) GenerateOp() (exec.Op, error) {
return &GetShardHTTPRequest{
UserSpace: n.UserSpace,
FileHash: n.FileHash,
Output: n.HTTPRequestVar().Var().VarID,

59
common/pkgs/ioswitch2/ops2/ec.go
View File

@ -158,12 +158,11 @@ func (o *ECMultiply) String() string {
}
type CallECMultiplier struct {
UserSpace clitypes.UserSpaceDetail
Coef [][]byte
Inputs []exec.VarID
Outputs []exec.VarID
BypassCallbacks []exec.VarID
ChunkSize int
UserSpace clitypes.UserSpaceDetail
Coef [][]byte
Inputs []exec.VarID
Outputs []exec.VarID
ChunkSize int
}
func (o *CallECMultiplier) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
@ -176,6 +175,7 @@ func (o *CallECMultiplier) Execute(ctx *exec.ExecContext, e *exec.Executor) erro
if err != nil {
return err
}
defer ecMul.Close()
inputs, err := exec.BindArray[*HTTPRequestValue](e, ctx.Context, o.Inputs)
if err != nil {
@ -191,32 +191,15 @@ func (o *CallECMultiplier) Execute(ctx *exec.ExecContext, e *exec.Executor) erro
if err != nil {
return err
}
defer ecMul.Abort()
outputVals := make([]*BypassedFileInfoValue, 0, len(outputs))
outputVals := make([]*FileInfoValue, 0, len(outputs))
for _, output := range outputs {
outputVals = append(outputVals, &BypassedFileInfoValue{
BypassedFileInfo: output,
outputVals = append(outputVals, &FileInfoValue{
FileInfo: output,
})
}
exec.PutArray(e, o.Outputs, outputVals)
callbacks, err := exec.BindArray[*BypassHandleResultValue](e, ctx.Context, o.BypassCallbacks)
if err != nil {
return err
}
allSuc := true
for _, callback := range callbacks {
if !callback.Commited {
allSuc = false
}
}
if allSuc {
ecMul.Complete()
}
return nil
}
@ -305,31 +288,24 @@ func (t *CallECMultiplierNode) InitFrom(node *ECMultiplyNode) {
t.InputIndexes = node.InputIndexes
t.OutputIndexes = node.OutputIndexes
t.InputValues().Init(len(t.InputIndexes) + len(t.OutputIndexes)) // 流的输出+回调的输入
t.InputValues().Init(len(t.InputIndexes))
t.OutputValues().Init(t, len(t.OutputIndexes))
}
func (t *CallECMultiplierNode) InputSlot(idx int) dag.ValueInputSlot {
func (t *CallECMultiplierNode) HTTPRequestSlot(idx int) dag.ValueInputSlot {
return dag.ValueInputSlot{
Node: t,
Index: idx,
}
}
func (t *CallECMultiplierNode) OutputVar(idx int) dag.ValueOutputSlot {
func (t *CallECMultiplierNode) FileInfoVar(idx int) dag.ValueOutputSlot {
return dag.ValueOutputSlot{
Node: t,
Index: idx,
}
}
func (t *CallECMultiplierNode) BypassCallbackSlot(idx int) dag.ValueInputSlot {
return dag.ValueInputSlot{
Node: t,
Index: idx + len(t.InputIndexes),
}
}
func (t *CallECMultiplierNode) GenerateOp() (exec.Op, error) {
rs, err := ec.NewRs(t.EC.K, t.EC.N)
if err != nil {
@ -341,11 +317,10 @@ func (t *CallECMultiplierNode) GenerateOp() (exec.Op, error) {
}
return &CallECMultiplier{
UserSpace: t.UserSpace,
Coef: coef,
Inputs: t.InputValues().GetVarIDsRanged(0, len(t.InputIndexes)),
Outputs: t.OutputValues().GetVarIDs(),
BypassCallbacks: t.InputValues().GetVarIDsStart(len(t.InputIndexes)),
ChunkSize: t.EC.ChunkSize,
UserSpace: t.UserSpace,
Coef: coef,
Inputs: t.InputValues().GetVarIDs(),
Outputs: t.OutputValues().GetVarIDs(),
ChunkSize: t.EC.ChunkSize,
}, nil
}

48
common/pkgs/ioswitch2/ops2/multipart.go
View File

@ -40,11 +40,10 @@ func (v *UploadedPartInfoValue) Clone() exec.VarValue {
}
type MultipartInitiator struct {
UserSpace clitypes.UserSpaceDetail
UploadArgs exec.VarID
UploadedParts []exec.VarID
BypassFileOutput exec.VarID // 分片上传之后的临时文件的路径
BypassCallback exec.VarID // 临时文件使用结果，用于告知Initiator如何处理临时文件
UserSpace clitypes.UserSpaceDetail
UploadArgs exec.VarID
UploadedParts []exec.VarID
FileOutput exec.VarID // 分片上传之后的临时文件的路径
}
func (o *MultipartInitiator) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
@ -63,7 +62,7 @@ func (o *MultipartInitiator) Execute(ctx *exec.ExecContext, e *exec.Executor) er
if err != nil {
return err
}
defer multiTask.Abort()
defer multiTask.Close()
// 分发上传参数
e.PutVar(o.UploadArgs, &MultipartUploadArgsValue{
@ -88,25 +87,15 @@ func (o *MultipartInitiator) Execute(ctx *exec.ExecContext, e *exec.Executor) er
}
// 告知后续Op临时文件的路径
e.PutVar(o.BypassFileOutput, &BypassedFileInfoValue{
BypassedFileInfo: fileInfo,
e.PutVar(o.FileOutput, &FileInfoValue{
FileInfo: fileInfo,
})
// 等待后续Op处理临时文件
cb, err := exec.BindVar[*BypassHandleResultValue](e, ctx.Context, o.BypassCallback)
if err != nil {
return fmt.Errorf("getting temp file callback: %v", err)
}
if cb.Commited {
multiTask.Complete()
}
return nil
}
func (o *MultipartInitiator) String() string {
return fmt.Sprintf("MultipartInitiator Args: %v, Parts: %v, BypassFileOutput: %v, BypassCallback: %v", o.UploadArgs, o.UploadedParts, o.BypassFileOutput, o.BypassCallback)
return fmt.Sprintf("MultipartInitiator Args: %v, Parts: %v, BypassFileOutput: %v", o.UploadArgs, o.UploadedParts, o.FileOutput)
}
type MultipartUpload struct {
@ -170,7 +159,6 @@ func (b *GraphNodeBuilder) NewMultipartInitiator(userSpace clitypes.UserSpaceDet
b.AddNode(node)
node.OutputValues().Init(node, 2)
node.InputValues().Init(1)
return node
}
@ -181,21 +169,14 @@ func (n *MultipartInitiatorNode) UploadArgsVar() dag.ValueOutputSlot {
}
}
func (n *MultipartInitiatorNode) BypassFileInfoVar() dag.ValueOutputSlot {
func (n *MultipartInitiatorNode) FileInfoVar() dag.ValueOutputSlot {
return dag.ValueOutputSlot{
Node: n,
Index: 1,
}
}
func (n *MultipartInitiatorNode) BypassCallbackSlot() dag.ValueInputSlot {
return dag.ValueInputSlot{
Node: n,
Index: 0,
}
}
func (n *MultipartInitiatorNode) AppendPartInfoSlot() dag.ValueInputSlot {
func (n *MultipartInitiatorNode) PartInfoSlot() dag.ValueInputSlot {
return dag.ValueInputSlot{
Node: n,
Index: n.InputValues().EnlargeOne(),
@ -204,11 +185,10 @@ func (n *MultipartInitiatorNode) AppendPartInfoSlot() dag.ValueInputSlot {
func (n *MultipartInitiatorNode) GenerateOp() (exec.Op, error) {
return &MultipartInitiator{
UserSpace: n.UserSpace,
UploadArgs: n.UploadArgsVar().Var().VarID,
UploadedParts: n.InputValues().GetVarIDsStart(1),
BypassFileOutput: n.BypassFileInfoVar().Var().VarID,
BypassCallback: n.BypassCallbackSlot().Var().VarID,
UserSpace: n.UserSpace,
UploadArgs: n.UploadArgsVar().Var().VarID,
UploadedParts: n.InputValues().GetVarIDs(),
FileOutput: n.FileInfoVar().Var().VarID,
}, nil
}

11
common/pkgs/ioswitch2/ops2/ops.go
View File

@ -2,8 +2,10 @@ package ops2
import (
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/plan/ops"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
)
type GraphNodeBuilder struct {
@ -24,5 +26,12 @@ type ToNode interface {
dag.Node
GetTo() ioswitch2.To
Input() dag.StreamInputSlot
SetInput(input *dag.StreamVar)
}
type FileInfoValue struct {
types.FileInfo
}
func (v *FileInfoValue) Clone() exec.VarValue {
return &FileInfoValue{v.FileInfo}
}

171
common/pkgs/ioswitch2/ops2/s2s.go
View File

@ -7,7 +7,6 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
)
func init() {
@ -15,97 +14,102 @@ func init() {
}
type S2STransfer struct {
Src clitypes.UserSpaceDetail
SrcPath exec.VarID
Dst clitypes.UserSpaceDetail
Output exec.VarID
BypassCallback exec.VarID
S2SOption types.S2SOption
SrcSpace clitypes.UserSpaceDetail
SrcPath string
DstSpace clitypes.UserSpaceDetail
DstPath string
Output exec.VarID
}
func (o *S2STransfer) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
srcPath, err := exec.BindVar[*BypassFilePathValue](e, ctx.Context, o.SrcPath)
if err != nil {
return err
}
stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
if err != nil {
return fmt.Errorf("getting storage pool: %w", err)
}
s2s, err := stgPool.GetS2STransfer(&o.Dst)
s2s, err := stgPool.GetS2STransfer(&o.DstSpace)
if err != nil {
return err
}
// 传输文件
dstPath, err := s2s.Transfer(ctx.Context, &o.Src, srcPath.Path, o.S2SOption)
fileInfo, err := s2s.Transfer(ctx.Context, &o.SrcSpace, o.SrcPath, o.DstPath)
if err != nil {
return err
}
defer s2s.Abort()
defer s2s.Close()
// 告知后续Op处理临时文件
e.PutVar(o.Output, &BypassedFileInfoValue{BypassedFileInfo: types.BypassedFileInfo{
Path: dstPath,
Hash: srcPath.Info.Hash,
Size: srcPath.Info.Size,
}})
// 等待后续Op处理临时文件
cb, err := exec.BindVar[*BypassHandleResultValue](e, ctx.Context, o.BypassCallback)
if err != nil {
return fmt.Errorf("getting temp file callback: %v", err)
}
if cb.Commited {
s2s.Complete()
}
e.PutVar(o.Output, &FileInfoValue{FileInfo: fileInfo})
return nil
}
func (o *S2STransfer) String() string {
return fmt.Sprintf("S2STransfer %v:%v -> %v:%v, Callback: %v", o.Src.UserSpace.Storage.String(), o.SrcPath, o.Dst.UserSpace.Storage.String(), o.Output, o.BypassCallback)
return fmt.Sprintf("S2STransfer %v:%v -> %v:%v", o.SrcSpace.UserSpace.Storage.String(), o.SrcPath, o.DstSpace.UserSpace.Storage.String(), o.Output)
}
type S2STransferDyn struct {
SrcSpace clitypes.UserSpaceDetail
SrcFileInfo exec.VarID
DstSpace clitypes.UserSpaceDetail
DstPath string
Output exec.VarID
}
func (o *S2STransferDyn) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
if err != nil {
return fmt.Errorf("getting storage pool: %w", err)
}
srcInfo, err := exec.BindVar[*FileInfoValue](e, ctx.Context, o.SrcFileInfo)
if err != nil {
return err
}
s2s, err := stgPool.GetS2STransfer(&o.DstSpace)
if err != nil {
return err
}
// 传输文件
fileInfo, err := s2s.Transfer(ctx.Context, &o.SrcSpace, srcInfo.Path, o.DstPath)
if err != nil {
return err
}
defer s2s.Close()
// 告知后续Op处理临时文件
e.PutVar(o.Output, &FileInfoValue{FileInfo: fileInfo})
return nil
}
func (o *S2STransferDyn) String() string {
return fmt.Sprintf("S2STransferDyn %v:%v -> %v:%v", o.SrcSpace.UserSpace.Storage.String(), o.SrcFileInfo, o.DstSpace.UserSpace.Storage.String(), o.Output)
}
type S2STransferNode struct {
dag.NodeBase
Src clitypes.UserSpaceDetail
Dst clitypes.UserSpaceDetail
Opt types.S2SOption
SrcSpace clitypes.UserSpaceDetail
SrcPath string
DstSpace clitypes.UserSpaceDetail
DstPath string
}
func (b *GraphNodeBuilder) NewS2STransfer(src, dst clitypes.UserSpaceDetail, opt types.S2SOption) *S2STransferNode {
func (b *GraphNodeBuilder) NewS2STransfer(srcSpace clitypes.UserSpaceDetail, srcPath string, dstSpace clitypes.UserSpaceDetail, dstPath string) *S2STransferNode {
n := &S2STransferNode{
Src: src,
Dst: dst,
Opt: opt,
SrcSpace: srcSpace,
SrcPath: srcPath,
DstSpace: dstSpace,
DstPath: dstPath,
}
b.AddNode(n)
n.OutputValues().Init(n, 1)
n.InputValues().Init(2)
return n
}
func (n *S2STransferNode) SrcPathSlot() dag.ValueInputSlot {
return dag.ValueInputSlot{
Node: n,
Index: 0,
}
}
func (n *S2STransferNode) BypassCallbackSlot() dag.ValueInputSlot {
return dag.ValueInputSlot{
Node: n,
Index: 1,
}
}
func (n *S2STransferNode) BypassFileInfoVar() dag.ValueOutputSlot {
func (n *S2STransferNode) FileInfoVar() dag.ValueOutputSlot {
return dag.ValueOutputSlot{
Node: n,
Index: 0,
@ -114,11 +118,54 @@ func (n *S2STransferNode) BypassFileInfoVar() dag.ValueOutputSlot {
func (n *S2STransferNode) GenerateOp() (exec.Op, error) {
return &S2STransfer{
Src: n.Src,
SrcPath: n.SrcPathSlot().Var().VarID,
Dst: n.Dst,
Output: n.BypassFileInfoVar().Var().VarID,
BypassCallback: n.BypassCallbackSlot().Var().VarID,
S2SOption: n.Opt,
SrcSpace: n.SrcSpace,
SrcPath: n.SrcPath,
DstSpace: n.DstSpace,
DstPath: n.DstPath,
Output: n.FileInfoVar().Var().VarID,
}, nil
}
type S2STransferDynNode struct {
dag.NodeBase
SrcSpace clitypes.UserSpaceDetail
DstSpace clitypes.UserSpaceDetail
DstPath string
}
func (b *GraphNodeBuilder) NewS2STransferDyn(srcSpace clitypes.UserSpaceDetail, dstSpace clitypes.UserSpaceDetail, dstPath string) *S2STransferDynNode {
n := &S2STransferDynNode{
SrcSpace: srcSpace,
DstSpace: dstSpace,
DstPath: dstPath,
}
b.AddNode(n)
n.InputValues().Init(1)
n.OutputValues().Init(n, 1)
return n
}
func (n *S2STransferDynNode) SrcFileInfoSlot() dag.ValueInputSlot {
return dag.ValueInputSlot{
Node: n,
Index: 0,
}
}
func (n *S2STransferDynNode) FileInfoVar() dag.ValueOutputSlot {
return dag.ValueOutputSlot{
Node: n,
Index: 0,
}
}
func (n *S2STransferDynNode) GenerateOp() (exec.Op, error) {
return &S2STransferDyn{
SrcSpace: n.SrcSpace,
SrcFileInfo: n.SrcFileInfoSlot().Var().VarID,
DstSpace: n.DstSpace,
DstPath: n.DstPath,
Output: n.FileInfoVar().Var().VarID,
}, nil
}

223
common/pkgs/ioswitch2/ops2/shard_store.go
View File

@ -2,199 +2,154 @@ package ops2
import (
"fmt"
"io"
"gitlink.org.cn/cloudream/common/pkgs/future"
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/io2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
)
func init() {
exec.UseOp[*ShardRead]()
exec.UseOp[*ShardWrite]()
exec.UseOp[*GetShardInfo]()
exec.UseOp[*StoreShard]()
}
type ShardRead struct {
Output exec.VarID
type GetShardInfo struct {
UserSpace clitypes.UserSpaceDetail
Open types.OpenOption
FileHash clitypes.FileHash
ShardInfo exec.VarID
}
func (o *ShardRead) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
logger.
WithField("Open", o.Open.String()).
Debugf("reading from shard store")
defer logger.Debugf("reading from shard store finished")
func (o *GetShardInfo) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
if err != nil {
return fmt.Errorf("getting storage pool: %w", err)
return fmt.Errorf("getting shard store: %w", err)
}
store, err := stgPool.GetShardStore(&o.UserSpace)
if err != nil {
return fmt.Errorf("getting shard store of user space %v: %w", o.UserSpace, err)
return fmt.Errorf("getting shard store: %w", err)
}
file, err := store.Open(o.Open)
info, err := store.Info(o.FileHash)
if err != nil {
return fmt.Errorf("opening shard store file: %w", err)
return fmt.Errorf("getting shard info: %w", err)
}
fut := future.NewSetVoid()
e.PutVar(o.Output, &exec.StreamValue{
Stream: io2.AfterReadClosedOnce(file, func(closer io.ReadCloser) {
fut.SetVoid()
}),
})
return fut.Wait(ctx.Context)
}
func (o *ShardRead) String() string {
return fmt.Sprintf("ShardRead %v -> %v", o.Open.String(), o.Output)
}
type ShardWrite struct {
Input exec.VarID
FileHashVar exec.VarID
UserSpace clitypes.UserSpaceDetail
}
func (o *ShardWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
logger.
WithField("Input", o.Input).
WithField("FileHash", o.FileHashVar).
Debugf("writting file to shard store")
defer logger.Debugf("write to shard store finished")
stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
if err != nil {
return fmt.Errorf("getting storage pool: %w", err)
}
store, err := stgPool.GetShardStore(&o.UserSpace)
if err != nil {
return fmt.Errorf("getting shard store of user space %v: %w", o.UserSpace, err)
}
input, err := exec.BindVar[*exec.StreamValue](e, ctx.Context, o.Input)
if err != nil {
return err
}
defer input.Stream.Close()
fileInfo, err := store.Create(input.Stream)
if err != nil {
return fmt.Errorf("writing file to shard store: %w", err)
}
e.PutVar(o.FileHashVar, &FileInfoValue{
Hash: fileInfo.Hash,
Size: fileInfo.Size,
e.PutVar(o.ShardInfo, &FileInfoValue{
FileInfo: info,
})
return nil
}
func (o *ShardWrite) String() string {
return fmt.Sprintf("ShardWrite %v -> %v", o.Input, o.FileHashVar)
func (o *GetShardInfo) String() string {
return fmt.Sprintf("GetShardInfo(%v)", o.FileHash)
}
type ShardReadNode struct {
dag.NodeBase
From *ioswitch2.FromShardstore
type StoreShard struct {
UserSpace clitypes.UserSpaceDetail
Open types.OpenOption
FileInfo exec.VarID
ShardInfo exec.VarID
}
func (b *GraphNodeBuilder) NewShardRead(fr *ioswitch2.FromShardstore, userSpace clitypes.UserSpaceDetail, open types.OpenOption) *ShardReadNode {
node := &ShardReadNode{
From: fr,
UserSpace: userSpace,
Open: open,
func (o *StoreShard) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
if err != nil {
return fmt.Errorf("getting storage pool: %w", err)
}
b.AddNode(node)
node.OutputStreams().Init(node, 1)
return node
}
func (t *ShardReadNode) GetFrom() ioswitch2.From {
return t.From
}
func (t *ShardReadNode) Output() dag.StreamOutputSlot {
return dag.StreamOutputSlot{
Node: t,
Index: 0,
store, err := stgPool.GetShardStore(&o.UserSpace)
if err != nil {
return fmt.Errorf("getting shard store: %w", err)
}
info, err := exec.BindVar[*FileInfoValue](e, ctx.Context, o.FileInfo)
if err != nil {
return err
}
stored, err := store.Store(info.Path, info.Hash, info.Size)
if err != nil {
return fmt.Errorf("adding shard: %w", err)
}
e.PutVar(o.ShardInfo, &FileInfoValue{
FileInfo: stored,
})
return nil
}
func (t *ShardReadNode) GenerateOp() (exec.Op, error) {
return &ShardRead{
Output: t.OutputStreams().Get(0).VarID,
UserSpace: t.UserSpace,
Open: t.Open,
}, nil
func (o *StoreShard) String() string {
return fmt.Sprintf("StoreShard: addInfo=%v, shardInfo=%v", o.FileInfo, o.ShardInfo)
}
// func (t *IPFSReadType) String() string {
// return fmt.Sprintf("IPFSRead[%s,%v+%v]%v%v", t.FileHash, t.Option.Offset, t.Option.Length, formatStreamIO(node), formatValueIO(node))
// }
type ShardWriteNode struct {
type GetShardInfoNode struct {
dag.NodeBase
To *ioswitch2.ToShardStore
UserSpace clitypes.UserSpaceDetail
FileHashStoreKey string
UserSpace clitypes.UserSpaceDetail
FileHash clitypes.FileHash
}
func (b *GraphNodeBuilder) NewShardWrite(to *ioswitch2.ToShardStore, userSpace clitypes.UserSpaceDetail, fileHashStoreKey string) *ShardWriteNode {
node := &ShardWriteNode{
To: to,
UserSpace: userSpace,
FileHashStoreKey: fileHashStoreKey,
func (b *GraphNodeBuilder) NewGetShardInfo(userSpace clitypes.UserSpaceDetail, fileHash clitypes.FileHash) *GetShardInfoNode {
node := &GetShardInfoNode{
UserSpace: userSpace,
FileHash: fileHash,
}
b.AddNode(node)
node.InputStreams().Init(1)
node.OutputValues().Init(node, 1)
return node
}
func (t *ShardWriteNode) GetTo() ioswitch2.To {
return t.To
func (n *GetShardInfoNode) FileInfoVar() dag.ValueOutputSlot {
return dag.ValueOutputSlot{
Node: n,
Index: 0,
}
}
func (t *ShardWriteNode) SetInput(input *dag.StreamVar) {
input.To(t, 0)
func (n *GetShardInfoNode) GenerateOp() (exec.Op, error) {
return &GetShardInfo{
UserSpace: n.UserSpace,
FileHash: n.FileHash,
ShardInfo: n.FileInfoVar().Var().VarID,
}, nil
}
func (t *ShardWriteNode) Input() dag.StreamInputSlot {
return dag.StreamInputSlot{
type StoreShardNode struct {
dag.NodeBase
UserSpace clitypes.UserSpaceDetail
ShardInfoKey string
}
func (b *GraphNodeBuilder) NewStoreShard(userSpace clitypes.UserSpaceDetail, shardInfoKey string) *StoreShardNode {
node := &StoreShardNode{
UserSpace: userSpace,
ShardInfoKey: shardInfoKey,
}
b.AddNode(node)
node.InputValues().Init(1)
node.OutputValues().Init(node, 1)
return node
}
func (t *StoreShardNode) FileInfoSlot() dag.ValueInputSlot {
return dag.ValueInputSlot{
Node: t,
Index: 0,
}
}
func (t *ShardWriteNode) FileHashVar() *dag.ValueVar {
return t.OutputValues().Get(0)
func (t *StoreShardNode) ShardInfoVar() dag.ValueOutputSlot {
return dag.ValueOutputSlot{
Node: t,
Index: 0,
}
}
func (t *ShardWriteNode) GenerateOp() (exec.Op, error) {
return &ShardWrite{
Input: t.InputStreams().Get(0).VarID,
FileHashVar: t.OutputValues().Get(0).VarID,
UserSpace: t.UserSpace,
func (t *StoreShardNode) GenerateOp() (exec.Op, error) {
return &StoreShard{
UserSpace: t.UserSpace,
FileInfo: t.FileInfoSlot().Var().VarID,
ShardInfo: t.ShardInfoVar().Var().VarID,
}, nil
}
// func (t *IPFSWriteType) String() string {
// return fmt.Sprintf("IPFSWrite[%s,%v+%v]%v%v", t.FileHashStoreKey, t.Range.Offset, t.Range.Length, formatStreamIO(node), formatValueIO(node))
// }

58
common/pkgs/ioswitch2/parser/gen/generator.go
View File

@ -7,6 +7,7 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/common/utils/lo2"
"gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/common/utils/os2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser/state"
@ -234,7 +235,7 @@ func Extend(ctx *state.GenerateState) error {
return fmt.Errorf("no output stream found for data index %d", to.GetStreamIndex())
}
toNode.SetInput(str)
str.ToSlot(toNode.Input())
}
return nil
@ -258,13 +259,17 @@ func buildFromNode(ctx *state.GenerateState, f ioswitch2.From) (ops2.FromNode, e
}
switch f := f.(type) {
case *ioswitch2.FromShardstore:
t := ctx.DAG.NewShardRead(f, f.UserSpace, types.NewOpen(f.FileHash))
case *ioswitch2.FromShardStore:
getShard := ctx.DAG.NewGetShardInfo(f.UserSpace, f.FileHash)
getShard.Env().ToEnvDriver()
getShard.Env().Pinned = true
read := ctx.DAG.NewBaseReadDyn(f, f.UserSpace, types.DefaultOpen())
if f.StreamIndex.IsRaw() {
t.Open.WithNullableLength(repRange.Offset, repRange.Length)
read.Option.WithNullableLength(repRange.Offset, repRange.Length)
} else if f.StreamIndex.IsEC() {
t.Open.WithNullableLength(blkRange.Offset, blkRange.Length)
read.Option.WithNullableLength(blkRange.Offset, blkRange.Length)
} else if f.StreamIndex.IsSegment() {
segStart := ctx.Ft.SegmentParam.CalcSegmentStart(f.StreamIndex.Index)
segLen := ctx.Ft.SegmentParam.Segments[f.StreamIndex.Index]
@ -283,23 +288,23 @@ func buildFromNode(ctx *state.GenerateState, f ioswitch2.From) (ops2.FromNode, e
openLen = openEnd - segStart - openOff
}
t.Open.WithNullableLength(openOff, &openLen)
read.Option.WithNullableLength(openOff, &openLen)
}
switch addr := f.UserSpace.RecommendHub.Address.(type) {
case *cortypes.HttpAddressInfo:
t.Env().ToEnvWorker(&ioswitch2.HttpHubWorker{Hub: f.UserSpace.RecommendHub})
t.Env().Pinned = true
read.Env().ToEnvWorker(&ioswitch2.HttpHubWorker{Hub: f.UserSpace.RecommendHub})
read.Env().Pinned = true
case *cortypes.GRPCAddressInfo:
t.Env().ToEnvWorker(&ioswitch2.HubWorker{Hub: f.UserSpace.RecommendHub, Address: *addr})
t.Env().Pinned = true
read.Env().ToEnvWorker(&ioswitch2.HubWorker{Hub: f.UserSpace.RecommendHub, Address: *addr})
read.Env().Pinned = true
default:
return nil, fmt.Errorf("unsupported node address type %T", addr)
}
return t, nil
return read, nil
case *ioswitch2.FromDriver:
n := ctx.DAG.NewFromDriver(f, f.Handle)
@ -338,18 +343,9 @@ func buildFromNode(ctx *state.GenerateState, f ioswitch2.From) (ops2.FromNode, e
case *ioswitch2.FromBaseStore:
// TODO 可以考虑支持设置读取范围
n := ctx.DAG.NewPublicRead(f, f.UserSpace, f.Path)
switch addr := f.UserSpace.RecommendHub.Address.(type) {
case *cortypes.HttpAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HttpHubWorker{Hub: f.UserSpace.RecommendHub})
n.Env().Pinned = true
case *cortypes.GRPCAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HubWorker{Hub: f.UserSpace.RecommendHub, Address: *addr})
n.Env().Pinned = true
default:
return nil, fmt.Errorf("unsupported node address type %T", addr)
n := ctx.DAG.NewBaseRead(f, f.UserSpace, f.Path, types.DefaultOpen())
if err := setEnvByAddress(n, f.UserSpace.RecommendHub, f.UserSpace.RecommendHub.Address); err != nil {
return nil, err
}
return n, nil
@ -362,15 +358,21 @@ func buildFromNode(ctx *state.GenerateState, f ioswitch2.From) (ops2.FromNode, e
func buildToNode(ctx *state.GenerateState, t ioswitch2.To) (ops2.ToNode, error) {
switch t := t.(type) {
case *ioswitch2.ToShardStore:
n := ctx.DAG.NewShardWrite(t, t.Space, t.FileHashStoreKey)
tempFileName := os2.GenerateRandomFileName(20)
if err := setEnvByAddress(n, t.Space.RecommendHub, t.Space.RecommendHub.Address); err != nil {
write := ctx.DAG.NewBaseWrite(t, t.Space, tempFileName)
if err := setEnvByAddress(write, t.Space.RecommendHub, t.Space.RecommendHub.Address); err != nil {
return nil, err
}
write.Env().Pinned = true
n.Env().Pinned = true
add := ctx.DAG.NewStoreShard(t.Space, t.ResultStoreKey)
add.Env().ToEnvDriver()
add.Env().Pinned = true
return n, nil
write.FileInfoVar().ToSlot(add.FileInfoSlot())
return write, nil
case *ioswitch2.ToDriver:
n := ctx.DAG.NewToDriver(t, t.Handle)
@ -380,7 +382,7 @@ func buildToNode(ctx *state.GenerateState, t ioswitch2.To) (ops2.ToNode, error)
return n, nil
case *ioswitch2.ToBaseStore:
n := ctx.DAG.NewPublicWrite(t, t.Space, t.ObjectPath)
n := ctx.DAG.NewBaseWrite(t, t.Space, t.ObjectPath)
if err := setEnvByAddress(n, t.Space.RecommendHub, t.Space.RecommendHub.Address); err != nil {
return nil, err

59
common/pkgs/ioswitch2/parser/opt/ec.go
View File

@ -53,7 +53,7 @@ func UseECMultiplier(ctx *state.GenerateState) {
}
var to *ioswitch2.ToShardStore
var swNodes []*ops2.ShardWriteNode
var bwNodes []*ops2.BaseWriteNode
// 所有的输出流必须有且只有一个相同的目的地
// 暂时只支持分片存储
for i := 0; i < mulNode.OutputStreams().Len(); i++ {
@ -62,17 +62,22 @@ func UseECMultiplier(ctx *state.GenerateState) {
}
dstNode := mulNode.OutputStreams().Get(i).Dst.Get(0)
swNode, ok := dstNode.(*ops2.ShardWriteNode)
swNode, ok := dstNode.(*ops2.BaseWriteNode)
if !ok {
return true
}
toShard, ok := swNode.To.(*ioswitch2.ToShardStore)
if !ok {
return true
}
if to == nil {
to = swNode.To
} else if to.Space.UserSpace.Storage.Equals(swNode.UserSpace.UserSpace.Storage) {
to = toShard
} else if to.Space.UserSpace.UserSpaceID != swNode.UserSpace.UserSpace.UserSpaceID {
return true
}
swNodes = append(swNodes, swNode)
bwNodes = append(bwNodes, swNode)
}
_, err := factory.GetBuilder(&to.Space).CreateECMultiplier()
if err != nil {
@ -80,19 +85,25 @@ func UseECMultiplier(ctx *state.GenerateState) {
}
// 每一个输入流都必须直接来自于存储服务，而且要支持通过HTTP读取文件
var srNodes []*ops2.ShardReadNode
var brNodes []*ops2.BaseReadDynNode
var fromShards []*ioswitch2.FromShardStore
for i := 0; i < mulNode.InputStreams().Len(); i++ {
inNode := mulNode.InputStreams().Get(i).Src
srNode, ok := inNode.(*ops2.ShardReadNode)
brNode, ok := inNode.(*ops2.BaseReadDynNode)
if !ok {
return true
}
fromShard, ok := brNode.From.(*ioswitch2.FromShardStore)
if !ok {
return true
}
if !factory.GetBuilder(&srNode.From.UserSpace).FeatureDesc().HasBypassHTTPRead {
if !factory.GetBuilder(&brNode.UserSpace).FeatureDesc().HasBypassHTTPRead {
return true
}
srNodes = append(srNodes, srNode)
brNodes = append(brNodes, brNode)
fromShards = append(fromShards, fromShard)
}
// 检查满足条件后，替换ECMultiply指令
@ -111,28 +122,26 @@ func UseECMultiplier(ctx *state.GenerateState) {
}
callMul.InitFrom(mulNode)
for i, srNode := range srNodes {
srNode.Output().Var().NotTo(mulNode)
for i, brNode := range brNodes {
brNode.Output().Var().NotTo(mulNode)
// 只有完全没有输出的ShardReadNode才可以被删除
if srNode.Output().Var().Dst.Len() == 0 {
ctx.DAG.RemoveNode(srNode)
delete(ctx.FromNodes, srNode.From)
if brNode.Output().Var().Dst.Len() == 0 {
ctx.DAG.RemoveNode(brNode)
delete(ctx.FromNodes, brNode.From)
}
hbr := ctx.DAG.NewBypassFromShardStoreHTTP(srNode.UserSpace, srNode.From.FileHash)
hbr.Env().CopyFrom(srNode.Env())
hbr.HTTPRequestVar().ToSlot(callMul.InputSlot(i))
hbr := ctx.DAG.NewGetShardHTTPRequest(brNode.UserSpace, fromShards[i].FileHash)
hbr.Env().CopyFrom(brNode.Env())
hbr.HTTPRequestVar().ToSlot(callMul.HTTPRequestSlot(i))
}
for i, swNode := range swNodes {
ctx.DAG.RemoveNode(swNode)
delete(ctx.ToNodes, swNode.To)
for i, bwNode := range bwNodes {
ctx.DAG.RemoveNode(bwNode)
delete(ctx.ToNodes, bwNode.To)
bs := ctx.DAG.NewBypassToShardStore(to.Space, swNode.FileHashStoreKey)
bs.Env().CopyFrom(swNode.Env())
callMul.OutputVar(i).ToSlot(bs.BypassFileInfoSlot())
bs.BypassCallbackVar().ToSlot(callMul.BypassCallbackSlot(i))
for _, dstSlot := range bwNode.FileInfoVar().ListDstSlots() {
callMul.FileInfoVar(i).ToSlot(dstSlot)
}
}
ctx.DAG.RemoveNode(mulNode)

22
common/pkgs/ioswitch2/parser/opt/misc.go
View File

@ -37,27 +37,15 @@ func DropUnused(ctx *state.GenerateState) {
// 为IPFS写入指令存储结果
func StoreShardWriteResult(ctx *state.GenerateState) {
dag.WalkOnlyType[*ops2.ShardWriteNode](ctx.DAG.Graph, func(n *ops2.ShardWriteNode) bool {
if n.FileHashStoreKey == "" {
dag.WalkOnlyType[*ops2.StoreShardNode](ctx.DAG.Graph, func(n *ops2.StoreShardNode) bool {
if n.ShardInfoKey == "" {
return true
}
storeNode := ctx.DAG.NewStore()
storeNode.Env().ToEnvDriver()
storeNode.Store(n.FileHashStoreKey, n.FileHashVar())
return true
})
dag.WalkOnlyType[*ops2.BypassToShardStoreNode](ctx.DAG.Graph, func(n *ops2.BypassToShardStoreNode) bool {
if n.FileHashStoreKey == "" {
return true
}
storeNode := ctx.DAG.NewStore()
storeNode.Env().ToEnvDriver()
storeNode.Store(n.FileHashStoreKey, n.FileHashVar().Var())
storeNode.Store(n.ShardInfoKey, n.ShardInfoVar().Var())
return true
})
}
@ -77,7 +65,7 @@ func GenerateRange(ctx *state.GenerateState) {
Length: toRng.Length,
})
toInput.Var().NotTo(toNode)
toNode.SetInput(rnged)
rnged.ToSlot(toInput)
} else if toStrIdx.IsEC() {
stripSize := int64(ctx.Ft.ECParam.ChunkSize * ctx.Ft.ECParam.K)
@ -93,7 +81,7 @@ func GenerateRange(ctx *state.GenerateState) {
Length: toRng.Length,
})
toInput.Var().NotTo(toNode)
toNode.SetInput(rnged)
rnged.ToSlot(toInput)
} else if toStrIdx.IsSegment() {
// if frNode, ok := toNode.Input().Var().From().Node.(ops2.FromNode); ok {
// // 目前只有To也是分段时，才可能对接一个提供分段的From，此时不需要再生成Range指令

36
common/pkgs/ioswitch2/parser/opt/multipart.go
View File

@ -9,22 +9,22 @@ import (
)
// 将SegmentJoin指令替换成分片上传指令
func UseMultipartUpcopyToShardStore(ctx *state.GenerateState) {
func UseMultipartUploadToShardStore(ctx *state.GenerateState) {
dag.WalkOnlyType[*ops2.SegmentJoinNode](ctx.DAG.Graph, func(joinNode *ops2.SegmentJoinNode) bool {
if joinNode.Joined().Dst.Len() != 1 {
return true
}
joinDst := joinNode.Joined().Dst.Get(0)
shardNode, ok := joinDst.(*ops2.ShardWriteNode)
bwNode, ok := joinDst.(*ops2.BaseWriteNode)
if !ok {
return true
}
// SegmentJoin的输出流的范围必须与ToShardStore的输入流的范围相同，
// 虽然可以通过调整SegmentJoin的输入流来调整范围，但太复杂，暂不支持
toStrIdx := shardNode.GetTo().GetStreamIndex()
toStrRng := shardNode.GetTo().GetRange()
toStrIdx := bwNode.GetTo().GetStreamIndex()
toStrRng := bwNode.GetTo().GetRange()
if toStrIdx.IsRaw() {
if !toStrRng.Equals(ctx.StreamRange) {
return true
@ -34,7 +34,7 @@ func UseMultipartUpcopyToShardStore(ctx *state.GenerateState) {
}
// Join的目的地必须支持MultipartUpload功能才能替换成分片上传
multiUpload, err := factory.GetBuilder(&shardNode.UserSpace).CreateMultiparter()
multiUpload, err := factory.GetBuilder(&bwNode.UserSpace).CreateMultiparter()
if err != nil {
return true
}
@ -47,8 +47,8 @@ func UseMultipartUpcopyToShardStore(ctx *state.GenerateState) {
}
}
initNode := ctx.DAG.NewMultipartInitiator(shardNode.UserSpace)
initNode.Env().CopyFrom(shardNode.Env())
initNode := ctx.DAG.NewMultipartInitiator(bwNode.UserSpace)
initNode.Env().CopyFrom(bwNode.Env())
partNumber := 1
for i, size := range joinNode.Segments {
@ -64,24 +64,24 @@ func UseMultipartUpcopyToShardStore(ctx *state.GenerateState) {
joinInput.Var().ToSlot(splitNode.InputSlot())
for i2 := 0; i2 < len(splits); i2++ {
uploadNode := ctx.DAG.NewMultipartUpload(shardNode.UserSpace, partNumber, splits[i2])
uploadNode := ctx.DAG.NewMultipartUpload(bwNode.UserSpace, partNumber, splits[i2])
uploadNode.Env().CopyFrom(joinInput.Var().Src.Env())
initNode.UploadArgsVar().ToSlot(uploadNode.UploadArgsSlot())
splitNode.SegmentVar(i2).ToSlot(uploadNode.PartStreamSlot())
uploadNode.UploadResultVar().ToSlot(initNode.AppendPartInfoSlot())
uploadNode.UploadResultVar().ToSlot(initNode.PartInfoSlot())
partNumber++
}
} else {
// 否则直接上传整个分段
uploadNode := ctx.DAG.NewMultipartUpload(shardNode.UserSpace, partNumber, size)
uploadNode := ctx.DAG.NewMultipartUpload(bwNode.UserSpace, partNumber, size)
// 上传指令直接在流的产生节点执行
uploadNode.Env().CopyFrom(joinInput.Var().Src.Env())
initNode.UploadArgsVar().ToSlot(uploadNode.UploadArgsSlot())
joinInput.Var().ToSlot(uploadNode.PartStreamSlot())
uploadNode.UploadResultVar().ToSlot(initNode.AppendPartInfoSlot())
uploadNode.UploadResultVar().ToSlot(initNode.PartInfoSlot())
partNumber++
}
@ -89,17 +89,15 @@ func UseMultipartUpcopyToShardStore(ctx *state.GenerateState) {
joinInput.Var().NotTo(joinNode)
}
bypassNode := ctx.DAG.NewBypassToShardStore(shardNode.UserSpace, shardNode.FileHashStoreKey)
bypassNode.Env().CopyFrom(shardNode.Env())
// 分片上传Node产生的结果送到bypassNode，bypassNode将处理结果再送回分片上传Node
initNode.BypassFileInfoVar().ToSlot(bypassNode.BypassFileInfoSlot())
bypassNode.BypassCallbackVar().ToSlot(initNode.BypassCallbackSlot())
// BaseWriteNode的FileInfoVar替换为MultipartInitiatorNode的FileInfoVar
for _, dstSlot := range bwNode.FileInfoVar().ListDstSlots() {
initNode.FileInfoVar().ToSlot(dstSlot)
}
// 最后删除Join指令和ToShardStore指令
ctx.DAG.RemoveNode(joinNode)
ctx.DAG.RemoveNode(shardNode)
delete(ctx.ToNodes, shardNode.GetTo())
ctx.DAG.RemoveNode(bwNode)
delete(ctx.ToNodes, bwNode.GetTo())
return true
})
}

115
common/pkgs/ioswitch2/parser/opt/s2s.go
View File

@ -5,7 +5,6 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser/state"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
)
// 将直接从一个存储服务传到另一个存储服务的过程换成S2S传输
@ -17,7 +16,7 @@ func UseS2STransfer(ctx *state.GenerateState) {
for fr, frNode := range ctx.FromNodes {
switch fr := fr.(type) {
case *ioswitch2.FromShardstore:
case *ioswitch2.FromShardStore:
s2sFromShardStore(ctx, fr, frNode)
case *ioswitch2.FromBaseStore:
s2sFromBaseStore(ctx, fr, frNode)
@ -25,7 +24,7 @@ func UseS2STransfer(ctx *state.GenerateState) {
}
}
func s2sFromShardStore(ctx *state.GenerateState, fromShard *ioswitch2.FromShardstore, frNode ops2.FromNode) {
func s2sFromShardStore(ctx *state.GenerateState, fromShard *ioswitch2.FromShardStore, frNode ops2.FromNode) {
fromStgBld := factory.GetBuilder(&fromShard.UserSpace)
if !fromStgBld.FeatureDesc().HasBypassShardRead {
return
@ -43,31 +42,16 @@ func s2sFromShardStore(ctx *state.GenerateState, fromShard *ioswitch2.FromShards
}
failed := false
var toShards []*ops2.ShardWriteNode
var toPublics []*ops2.BaseWriteNode
var toBases []*ops2.BaseWriteNode
loop:
for i := 0; i < outVar.Dst.Len(); i++ {
dstNode := outVar.Dst.Get(i)
switch dstNode := dstNode.(type) {
case *ops2.ShardWriteNode:
dstStgBld := factory.GetBuilder(&dstNode.UserSpace)
if !dstStgBld.FeatureDesc().HasBypassShardWrite {
failed = true
break
}
if !s2s.CanTransfer(&dstNode.UserSpace) {
failed = true
break
}
toShards = append(toShards, dstNode)
case *ops2.BaseWriteNode:
dstStgBld := factory.GetBuilder(&dstNode.UserSpace)
if !dstStgBld.FeatureDesc().HasBypassPublicWrite {
if !dstStgBld.FeatureDesc().HasBypassBaseWrite {
failed = true
break
}
@ -77,7 +61,7 @@ loop:
break
}
toPublics = append(toPublics, dstNode)
toBases = append(toBases, dstNode)
default:
failed = true
@ -88,50 +72,24 @@ loop:
return
}
for _, toShard := range toShards {
s2sNode := ctx.DAG.NewS2STransfer(fromShard.UserSpace, toShard.UserSpace, types.S2SOption{})
for _, toBase := range toBases {
s2sNode := ctx.DAG.NewS2STransferDyn(fromShard.UserSpace, toBase.UserSpace, toBase.Path)
// 直传指令在目的地Hub上执行
s2sNode.Env().CopyFrom(toShard.Env())
s2sNode.Env().CopyFrom(toBase.Env())
// 先获取文件路径，送到S2S节点
brNode := ctx.DAG.NewBypassFromShardStore(fromShard.UserSpace, fromShard.FileHash)
brNode.Env().CopyFrom(frNode.Env())
brNode.FilePathVar().ToSlot(s2sNode.SrcPathSlot())
gsNode := ctx.DAG.NewGetShardInfo(fromShard.UserSpace, fromShard.FileHash)
gsNode.Env().CopyFrom(toBase.Env())
gsNode.FileInfoVar().ToSlot(s2sNode.SrcFileInfoSlot())
// 传输结果通知目的节点
bwNode := ctx.DAG.NewBypassToShardStore(toShard.UserSpace, toShard.To.FileHashStoreKey)
bwNode.Env().CopyFrom(toShard.Env())
s2sNode.BypassFileInfoVar().ToSlot(bwNode.BypassFileInfoSlot())
bwNode.BypassCallbackVar().ToSlot(s2sNode.BypassCallbackSlot())
// 原本BaseWriteNode的FileInfoVar被替换成S2SNode的FileInfoVar
for _, dstSlot := range toBase.FileInfoVar().ListDstSlots() {
s2sNode.FileInfoVar().ToSlot(dstSlot)
}
// 从计划中删除目标节点
ctx.DAG.RemoveNode(toShard)
delete(ctx.ToNodes, toShard.To)
}
for _, toPub := range toPublics {
s2sNode := ctx.DAG.NewS2STransfer(fromShard.UserSpace, toPub.UserSpace, types.S2SOption{
DestPathHint: toPub.Path,
})
// 直传指令在目的地Hub上执行
s2sNode.Env().CopyFrom(toPub.Env())
// 先获取文件路径，送到S2S节点
brNode := ctx.DAG.NewBypassFromShardStore(fromShard.UserSpace, fromShard.FileHash)
brNode.Env().CopyFrom(toPub.Env())
brNode.FilePathVar().ToSlot(s2sNode.SrcPathSlot())
// 传输结果通知目的节点
bwNode := ctx.DAG.NewBypassToBaseStore(toPub.UserSpace, toPub.Path)
bwNode.Env().CopyFrom(toPub.Env())
s2sNode.BypassFileInfoVar().ToSlot(bwNode.BypassFileInfoSlot())
bwNode.BypassCallbackVar().ToSlot(s2sNode.BypassCallbackSlot())
// 从计划中删除目标节点
ctx.DAG.RemoveNode(toPub)
delete(ctx.ToNodes, toPub.To)
ctx.DAG.RemoveNode(toBase)
delete(ctx.ToNodes, toBase.To)
}
// 从计划中删除源节点
@ -139,9 +97,9 @@ loop:
delete(ctx.FromNodes, frNode.GetFrom())
}
func s2sFromBaseStore(ctx *state.GenerateState, fromPub *ioswitch2.FromBaseStore, frNode ops2.FromNode) {
fromStgBld := factory.GetBuilder(&fromPub.UserSpace)
if !fromStgBld.FeatureDesc().HasBypassPublicRead {
func s2sFromBaseStore(ctx *state.GenerateState, fromBase *ioswitch2.FromBaseStore, frNode ops2.FromNode) {
fromStgBld := factory.GetBuilder(&fromBase.UserSpace)
if !fromStgBld.FeatureDesc().HasBypassBaseRead {
return
}
@ -157,7 +115,7 @@ func s2sFromBaseStore(ctx *state.GenerateState, fromPub *ioswitch2.FromBaseStore
}
failed := false
var toPublics []*ops2.BaseWriteNode
var toBases []*ops2.BaseWriteNode
loop:
for i := 0; i < outVar.Dst.Len(); i++ {
@ -166,7 +124,7 @@ loop:
switch dstNode := dstNode.(type) {
case *ops2.BaseWriteNode:
dstStgBld := factory.GetBuilder(&dstNode.UserSpace)
if !dstStgBld.FeatureDesc().HasBypassPublicWrite {
if !dstStgBld.FeatureDesc().HasBypassBaseWrite {
failed = true
break
}
@ -176,7 +134,7 @@ loop:
break
}
toPublics = append(toPublics, dstNode)
toBases = append(toBases, dstNode)
default:
failed = true
@ -187,28 +145,19 @@ loop:
return
}
for _, toPub := range toPublics {
s2sNode := ctx.DAG.NewS2STransfer(fromPub.UserSpace, toPub.UserSpace, types.S2SOption{
DestPathHint: toPub.Path,
})
for _, toBase := range toBases {
s2sNode := ctx.DAG.NewS2STransfer(fromBase.UserSpace, fromBase.Path, toBase.UserSpace, toBase.Path)
// 直传指令在目的地Hub上执行
s2sNode.Env().CopyFrom(toPub.Env())
s2sNode.Env().CopyFrom(toBase.Env())
// 先获取文件路径，送到S2S节点
brNode := ctx.DAG.NewBypassFromBaseStore(fromPub.UserSpace, fromPub.Path)
brNode.Env().CopyFrom(toPub.Env())
brNode.FilePathVar().ToSlot(s2sNode.SrcPathSlot())
// 传输结果通知目的节点
bwNode := ctx.DAG.NewBypassToBaseStore(toPub.UserSpace, toPub.Path)
bwNode.Env().CopyFrom(toPub.Env())
s2sNode.BypassFileInfoVar().ToSlot(bwNode.BypassFileInfoSlot())
bwNode.BypassCallbackVar().ToSlot(s2sNode.BypassCallbackSlot())
// 原本BaseWriteNode的FileInfoVar被替换成S2SNode的FileInfoVar
for _, dstSlot := range toBase.FileInfoVar().ListDstSlots() {
s2sNode.FileInfoVar().ToSlot(dstSlot)
}
// 从计划中删除目标节点
ctx.DAG.RemoveNode(toPub)
delete(ctx.ToNodes, toPub.To)
ctx.DAG.RemoveNode(toBase)
delete(ctx.ToNodes, toBase.To)
}
// 从计划中删除源节点

2
common/pkgs/ioswitch2/parser/parser.go
View File

@ -79,7 +79,7 @@ func Parse(ft ioswitch2.FromTo, blder *exec.PlanBuilder) error {
opt.RemoveUnusedFromNode(state)
opt.UseECMultiplier(state)
opt.UseS2STransfer(state)
opt.UseMultipartUpcopyToShardStore(state)
opt.UseMultipartUploadToShardStore(state)
opt.DropUnused(state)
opt.StoreShardWriteResult(state)
opt.GenerateRange(state)

29
common/pkgs/ioswitch2/plans/complete_multipart.go
View File

@ -3,6 +3,7 @@ package plans
import (
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/plan"
"gitlink.org.cn/cloudream/common/utils/os2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
@ -20,24 +21,34 @@ func CompleteMultipart(blocks []clitypes.ObjectBlock, blockSpaces []clitypes.Use
joinNode.Env().Pinned = true
for i, blk := range blocks {
rd := da.NewShardRead(nil, blockSpaces[i], types.NewOpen(blk.FileHash))
rd.Env().ToEnvWorker(getWorkerInfo(blockSpaces[i].RecommendHub))
rd.Env().Pinned = true
gs := da.NewGetShardInfo(blockSpaces[i], blk.FileHash)
gs.Env().ToEnvDriver()
gs.Env().Pinned = true
rd.Output().ToSlot(joinNode.InputSlot(i))
br := da.NewBaseReadDyn(nil, blockSpaces[i], types.DefaultOpen())
br.Env().ToEnvWorker(getWorkerInfo(blockSpaces[i].RecommendHub))
br.Env().Pinned = true
gs.FileInfoVar().ToSlot(br.FileInfoSlot())
br.Output().ToSlot(joinNode.InputSlot(i))
}
// TODO 应该采取更合理的方式同时支持Parser和直接生成DAG
wr := da.NewShardWrite(nil, targetSpace, shardInfoKey)
wr.Env().ToEnvWorker(getWorkerInfo(targetSpace.RecommendHub))
wr.Env().Pinned = true
br := da.NewBaseWrite(nil, targetSpace, os2.GenerateRandomFileName(20))
br.Env().ToEnvWorker(getWorkerInfo(targetSpace.RecommendHub))
br.Env().Pinned = true
joinNode.Joined().ToSlot(wr.Input())
as := da.NewStoreShard(targetSpace, shardInfoKey)
as.Env().ToEnvDriver()
as.Env().Pinned = true
joinNode.Joined().ToSlot(br.Input())
if shardInfoKey != "" {
store := da.NewStore()
store.Env().ToEnvDriver()
store.Store(shardInfoKey, wr.FileHashVar())
store.Store(shardInfoKey, as.ShardInfoVar().Var())
}
err := plan.Compile(da.Graph, blder)

400
common/pkgs/ioswitchlrc/ops2/base_store.go Normal file
View File

@ -0,0 +1,400 @@
package ops2
import (
"fmt"
"io"
"gitlink.org.cn/cloudream/common/pkgs/future"
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/io2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
)
func init() {
exec.UseOp[*BaseWrite]()
exec.UseOp[*BaseRead]()
}
type BaseRead struct {
Output exec.VarID
UserSpace clitypes.UserSpaceDetail
Path string
Option types.OpenOption
}
func (o *BaseRead) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
logger.
WithField("Output", o.Output).
WithField("UserSpace", o.UserSpace).
WithField("Path", o.Path).
Debug("base read")
defer logger.Debug("base read end")
stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
if err != nil {
return fmt.Errorf("getting storage pool: %w", err)
}
store, err := stgPool.GetBaseStore(&o.UserSpace)
if err != nil {
return fmt.Errorf("getting base store of storage %v: %w", o.UserSpace, err)
}
stream, err := store.Read(o.Path, o.Option)
if err != nil {
return fmt.Errorf("reading object %v: %w", o.Path, err)
}
fut := future.NewSetVoid()
output := &exec.StreamValue{
Stream: io2.AfterReadClosed(stream, func(closer io.ReadCloser) {
fut.SetVoid()
}),
}
e.PutVar(o.Output, output)
return fut.Wait(ctx.Context)
}
func (o *BaseRead) String() string {
return fmt.Sprintf("PublicRead %v:%v -> %v", o.UserSpace, o.Path, o.Output)
}
type BaseReadPathVar struct {
UserSpace clitypes.UserSpaceDetail
Output exec.VarID
Path exec.VarID
Option types.OpenOption
}
func (o *BaseReadPathVar) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
logger.
WithField("Output", o.Output).
WithField("UserSpace", o.UserSpace).
WithField("Path", o.Path).
Debug("base read")
defer logger.Debug("base read end")
stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
if err != nil {
return fmt.Errorf("getting storage pool: %w", err)
}
path, err := exec.BindVar[*FileInfoValue](e, ctx.Context, o.Path)
if err != nil {
return err
}
store, err := stgPool.GetBaseStore(&o.UserSpace)
if err != nil {
return fmt.Errorf("getting base store of storage %v: %w", o.UserSpace, err)
}
stream, err := store.Read(path.Path, o.Option)
if err != nil {
return fmt.Errorf("reading object %v: %w", o.Path, err)
}
fut := future.NewSetVoid()
output := &exec.StreamValue{
Stream: io2.AfterReadClosed(stream, func(closer io.ReadCloser) {
fut.SetVoid()
}),
}
e.PutVar(o.Output, output)
return fut.Wait(ctx.Context)
}
func (o *BaseReadPathVar) String() string {
return fmt.Sprintf("BaseReadPathVar %v:%v -> %v", o.UserSpace, o.Path, o.Output)
}
type BaseWrite struct {
Input exec.VarID
UserSpace clitypes.UserSpaceDetail
Path string
WriteResult exec.VarID
}
func (o *BaseWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
logger.
WithField("Input", o.Input).
Debugf("write file to base store")
defer logger.Debugf("write file to base store finished")
stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
if err != nil {
return fmt.Errorf("getting storage pool: %w", err)
}
store, err := stgPool.GetBaseStore(&o.UserSpace)
if err != nil {
return fmt.Errorf("getting base store of storage %v: %w", o.UserSpace, err)
}
input, err := exec.BindVar[*exec.StreamValue](e, ctx.Context, o.Input)
if err != nil {
return err
}
defer input.Stream.Close()
ret, err := store.Write(o.Path, input.Stream)
if err != nil {
return err
}
e.PutVar(o.WriteResult, &FileInfoValue{
FileInfo: ret,
})
return nil
}
func (o *BaseWrite) String() string {
return fmt.Sprintf("PublicWrite %v -> %v:%v", o.Input, o.UserSpace, o.Path)
}
// type BaseWriteTemp struct {
// Input exec.VarID
// UserSpace clitypes.UserSpaceDetail
// Path string
// Signal exec.VarID
// WriteResult exec.VarID
// }
// func (o *BaseWriteTemp) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
// logger.
// WithField("Input", o.Input).
// Debugf("write file to base store")
// defer logger.Debugf("write file to base store finished")
// stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
// if err != nil {
// return fmt.Errorf("getting storage pool: %w", err)
// }
// _, err = exec.BindVar[*exec.SignalValue](e, ctx.Context, o.Signal)
// if err != nil {
// return err
// }
// store, err := stgPool.GetBaseStore(&o.UserSpace)
// if err != nil {
// return fmt.Errorf("getting base store of storage %v: %w", o.UserSpace, err)
// }
// input, err := exec.BindVar[*exec.StreamValue](e, ctx.Context, o.Input)
// if err != nil {
// return err
// }
// defer input.Stream.Close()
// ret, err := store.Write(o.Path, input.Stream)
// if err != nil {
// return err
// }
// e.PutVar(o.WriteResult, &WriteResultValue{
// WriteResult: ret,
// })
// return nil
// }
// func (o *BaseWriteTemp) String() string {
// return fmt.Sprintf("PublicWriteTemp[signal: %v] %v -> %v:%v", o.Signal, o.Input, o.UserSpace, o.Path)
// }
type BaseReadNode struct {
dag.NodeBase
From ioswitchlrc.From
UserSpace clitypes.UserSpaceDetail
Path string
Option types.OpenOption
}
func (b *GraphNodeBuilder) NewBaseRead(from ioswitchlrc.From, userSpace clitypes.UserSpaceDetail, path string, opt types.OpenOption) *BaseReadNode {
node := &BaseReadNode{
From: from,
UserSpace: userSpace,
Path: path,
Option: opt,
}
b.AddNode(node)
node.OutputStreams().Init(node, 1)
return node
}
func (t *BaseReadNode) GetFrom() ioswitchlrc.From {
return t.From
}
func (t *BaseReadNode) Output() dag.StreamOutputSlot {
return dag.StreamOutputSlot{
Node: t,
Index: 0,
}
}
func (t *BaseReadNode) GenerateOp() (exec.Op, error) {
return &BaseRead{
Output: t.Output().Var().VarID,
UserSpace: t.UserSpace,
Path: t.Path,
Option: t.Option,
}, nil
}
type BaseReadPathVarNode struct {
dag.NodeBase
From ioswitchlrc.From
UserSpace clitypes.UserSpaceDetail
Option types.OpenOption
}
func (b *GraphNodeBuilder) NewBaseReadPathVar(from ioswitchlrc.From, userSpace clitypes.UserSpaceDetail, opt types.OpenOption) *BaseReadPathVarNode {
node := &BaseReadPathVarNode{
From: from,
UserSpace: userSpace,
Option: opt,
}
b.AddNode(node)
node.OutputStreams().Init(node, 1)
node.InputValues().Init(1)
return node
}
func (t *BaseReadPathVarNode) GetFrom() ioswitchlrc.From {
return t.From
}
func (t *BaseReadPathVarNode) Output() dag.StreamOutputSlot {
return dag.StreamOutputSlot{
Node: t,
Index: 0,
}
}
func (t *BaseReadPathVarNode) PathVar() dag.ValueInputSlot {
return dag.ValueInputSlot{
Node: t,
Index: 0,
}
}
func (t *BaseReadPathVarNode) GenerateOp() (exec.Op, error) {
return &BaseReadPathVar{
UserSpace: t.UserSpace,
Output: t.Output().Var().VarID,
Path: t.PathVar().Var().VarID,
Option: t.Option,
}, nil
}
type BaseWriteNode struct {
dag.NodeBase
To ioswitchlrc.To
UserSpace clitypes.UserSpaceDetail
Path string
}
func (b *GraphNodeBuilder) NewBaseWrite(to ioswitchlrc.To, userSpace clitypes.UserSpaceDetail, path string) *BaseWriteNode {
node := &BaseWriteNode{
To: to,
UserSpace: userSpace,
Path: path,
}
b.AddNode(node)
node.InputStreams().Init(1)
node.OutputValues().Init(node, 1)
return node
}
func (t *BaseWriteNode) GetTo() ioswitchlrc.To {
return t.To
}
func (t *BaseWriteNode) Input() dag.StreamInputSlot {
return dag.StreamInputSlot{
Node: t,
Index: 0,
}
}
func (t *BaseWriteNode) FileInfoVar() dag.ValueOutputSlot {
return dag.ValueOutputSlot{
Node: t,
Index: 0,
}
}
func (t *BaseWriteNode) GenerateOp() (exec.Op, error) {
return &BaseWrite{
Input: t.InputStreams().Get(0).VarID,
UserSpace: t.UserSpace,
Path: t.Path,
WriteResult: t.FileInfoVar().Var().VarID,
}, nil
}
// type BaseWriteTempNode struct {
// dag.NodeBase
// To ioswitchlrc.To
// UserSpace clitypes.UserSpaceDetail
// Path string
// }
// func (b *GraphNodeBuilder) NewBaseWriteTemp(to ioswitchlrc.To, userSpace clitypes.UserSpaceDetail, path string) *BaseWriteTempNode {
// node := &BaseWriteTempNode{
// To: to,
// UserSpace: userSpace,
// Path: path,
// }
// b.AddNode(node)
// node.InputStreams().Init(1)
// node.InputValues().Init(1)
// return node
// }
// func (t *BaseWriteTempNode) GetTo() ioswitchlrc.To {
// return t.To
// }
// func (t *BaseWriteTempNode) Input() dag.StreamInputSlot {
// return dag.StreamInputSlot{
// Node: t,
// Index: 0,
// }
// }
// func (t *BaseWriteTempNode) TargetSignal() dag.ValueInputSlot {
// return dag.ValueInputSlot{
// Node: t,
// Index: 0,
// }
// }
// func (t *BaseWriteTempNode) FileInfoVar() dag.ValueOutputSlot {
// return dag.ValueOutputSlot{
// Node: t,
// Index: 0,
// }
// }
// func (t *BaseWriteTempNode) GenerateOp() (exec.Op, error) {
// return &BaseWriteTemp{
// Input: t.Input().Var().VarID,
// UserSpace: t.UserSpace,
// Path: t.Path,
// Signal: t.TargetSignal().Var().VarID,
// WriteResult: t.FileInfoVar().Var().VarID,
// }, nil
// }

75
common/pkgs/ioswitchlrc/ops2/ops.go
View File

@ -2,7 +2,9 @@ package ops2
import (
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/plan/ops"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
)
type GraphNodeBuilder struct {
@ -21,73 +23,12 @@ type FromNode interface {
type ToNode interface {
dag.Node
Input() dag.StreamInputSlot
SetInput(input *dag.StreamVar)
}
// func formatStreamIO(node *dag.Node) string {
// is := ""
// for i, in := range node.InputStreams {
// if i > 0 {
// is += ","
// }
type FileInfoValue struct {
types.FileInfo
}
// if in == nil {
// is += "."
// } else {
// is += fmt.Sprintf("%v", in.ID)
// }
// }
// os := ""
// for i, out := range node.OutputStreams {
// if i > ops
// os += ","
// }
// if out == nil {
// os += "."
// } else {
// os += fmt.Sprintf("%v", out.ID)
// }
// }
// if is == "" && os == "" {
// return ""
// }
// return fmt.Sprintf("S{%s>%s}", is, os)
// }
// func formatValueIO(node *dag.Node) string {
// is := ""
// for i, in := range node.InputValues {
// if i > 0 {
// is += ","
// }
// if in == nil {
// is += "."
// } else {
// is += fmt.Sprintf("%v", in.ID)
// }
// }
// os := ""
// for i, out := range node.OutputValues {
// if i > 0 {
// os += ","
// }
// if out == nil {
// os += "."
// } else {
// os += fmt.Sprintf("%v", out.ID)
// }
// }
// if is == "" && os == "" {
// return ""
// }
// return fmt.Sprintf("V{%s>%s}", is, os)
// }
func (v *FileInfoValue) Clone() exec.VarValue {
return &FileInfoValue{v.FileInfo}
}

233
common/pkgs/ioswitchlrc/ops2/shard_store.go
View File

@ -2,209 +2,154 @@ package ops2
import (
"fmt"
"io"
"gitlink.org.cn/cloudream/common/pkgs/future"
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/io2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
)
func init() {
exec.UseOp[*ShardRead]()
exec.UseOp[*ShardWrite]()
exec.UseVarValue[*ShardInfoValue]()
exec.UseOp[*GetShardInfo]()
exec.UseOp[*StoreShard]()
}
type ShardInfoValue struct {
Hash clitypes.FileHash `json:"hash"`
Size int64 `json:"size"`
}
func (v *ShardInfoValue) Clone() exec.VarValue {
return &ShardInfoValue{Hash: v.Hash, Size: v.Size}
}
type ShardRead struct {
Output exec.VarID
type GetShardInfo struct {
UserSpace clitypes.UserSpaceDetail
Open types.OpenOption
FileHash clitypes.FileHash
ShardInfo exec.VarID
}
func (o *ShardRead) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
logger.
WithField("Open", o.Open.String()).
Debugf("reading from shard store")
defer logger.Debugf("reading from shard store finished")
func (o *GetShardInfo) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
if err != nil {
return fmt.Errorf("getting storage pool: %w", err)
return fmt.Errorf("getting shard store: %w", err)
}
store, err := stgPool.GetShardStore(&o.UserSpace)
if err != nil {
return fmt.Errorf("getting shard store of user space %v: %w", o.UserSpace, err)
return fmt.Errorf("getting shard store: %w", err)
}
file, err := store.Open(o.Open)
info, err := store.Info(o.FileHash)
if err != nil {
return fmt.Errorf("opening shard store file: %w", err)
return fmt.Errorf("getting shard info: %w", err)
}
fut := future.NewSetVoid()
e.PutVar(o.Output, &exec.StreamValue{
Stream: io2.AfterReadClosedOnce(file, func(closer io.ReadCloser) {
fut.SetVoid()
}),
})
return fut.Wait(ctx.Context)
}
func (o *ShardRead) String() string {
return fmt.Sprintf("ShardRead %v -> %v", o.Open.String(), o.Output)
}
type ShardWrite struct {
Input exec.VarID
FileHashVar exec.VarID
UserSpace clitypes.UserSpaceDetail
}
func (o *ShardWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
logger.
WithField("Input", o.Input).
WithField("FileHash", o.FileHashVar).
Debugf("writting file to shard store")
defer logger.Debugf("write to shard store finished")
stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
if err != nil {
return fmt.Errorf("getting storage pool: %w", err)
}
store, err := stgPool.GetShardStore(&o.UserSpace)
if err != nil {
return fmt.Errorf("getting shard store of user space %v: %w", o.UserSpace, err)
}
input, err := exec.BindVar[*exec.StreamValue](e, ctx.Context, o.Input)
if err != nil {
return err
}
defer input.Stream.Close()
fileInfo, err := store.Create(input.Stream)
if err != nil {
return fmt.Errorf("writing file to shard store: %w", err)
}
e.PutVar(o.FileHashVar, &ShardInfoValue{
Hash: fileInfo.Hash,
Size: fileInfo.Size,
e.PutVar(o.ShardInfo, &FileInfoValue{
FileInfo: info,
})
return nil
}
func (o *ShardWrite) String() string {
return fmt.Sprintf("ShardWrite %v -> %v", o.Input, o.FileHashVar)
func (o *GetShardInfo) String() string {
return fmt.Sprintf("GetShardInfo(%v)", o.FileHash)
}
type ShardReadNode struct {
dag.NodeBase
From *ioswitchlrc.FromNode
type StoreShard struct {
UserSpace clitypes.UserSpaceDetail
Open types.OpenOption
FileInfo exec.VarID
ShardInfo exec.VarID
}
func (b *GraphNodeBuilder) NewShardRead(fr *ioswitchlrc.FromNode, userSpace clitypes.UserSpaceDetail, open types.OpenOption) *ShardReadNode {
node := &ShardReadNode{
From: fr,
UserSpace: userSpace,
Open: open,
func (o *StoreShard) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
if err != nil {
return fmt.Errorf("getting storage pool: %w", err)
}
b.AddNode(node)
node.OutputStreams().Init(node, 1)
return node
}
func (t *ShardReadNode) GetFrom() ioswitchlrc.From {
return t.From
}
func (t *ShardReadNode) Output() dag.StreamOutputSlot {
return dag.StreamOutputSlot{
Node: t,
Index: 0,
store, err := stgPool.GetShardStore(&o.UserSpace)
if err != nil {
return fmt.Errorf("getting shard store: %w", err)
}
info, err := exec.BindVar[*FileInfoValue](e, ctx.Context, o.FileInfo)
if err != nil {
return err
}
stored, err := store.Store(info.Path, info.Hash, info.Size)
if err != nil {
return fmt.Errorf("adding shard: %w", err)
}
e.PutVar(o.ShardInfo, &FileInfoValue{
FileInfo: stored,
})
return nil
}
func (t *ShardReadNode) GenerateOp() (exec.Op, error) {
return &ShardRead{
Output: t.OutputStreams().Get(0).VarID,
UserSpace: t.UserSpace,
Open: t.Open,
}, nil
func (o *StoreShard) String() string {
return fmt.Sprintf("StoreShard: addInfo=%v, shardInfo=%v", o.FileInfo, o.ShardInfo)
}
// func (t *IPFSReadType) String() string {
// return fmt.Sprintf("IPFSRead[%s,%v+%v]%v%v", t.FileHash, t.Option.Offset, t.Option.Length, formatStreamIO(node), formatValueIO(node))
// }
type ShardWriteNode struct {
type GetShardInfoNode struct {
dag.NodeBase
To *ioswitchlrc.ToNode
UserSpace clitypes.UserSpaceDetail
FileHashStoreKey string
UserSpace clitypes.UserSpaceDetail
FileHash clitypes.FileHash
}
func (b *GraphNodeBuilder) NewShardWrite(to *ioswitchlrc.ToNode, userSpace clitypes.UserSpaceDetail, fileHashStoreKey string) *ShardWriteNode {
node := &ShardWriteNode{
To: to,
UserSpace: userSpace,
FileHashStoreKey: fileHashStoreKey,
func (b *GraphNodeBuilder) NewGetShardInfo(userSpace clitypes.UserSpaceDetail, fileHash clitypes.FileHash) *GetShardInfoNode {
node := &GetShardInfoNode{
UserSpace: userSpace,
FileHash: fileHash,
}
b.AddNode(node)
node.InputStreams().Init(1)
node.OutputValues().Init(node, 1)
return node
}
func (t *ShardWriteNode) GetTo() ioswitchlrc.To {
return t.To
func (n *GetShardInfoNode) FileInfoVar() dag.ValueOutputSlot {
return dag.ValueOutputSlot{
Node: n,
Index: 0,
}
}
func (t *ShardWriteNode) SetInput(input *dag.StreamVar) {
input.To(t, 0)
func (n *GetShardInfoNode) GenerateOp() (exec.Op, error) {
return &GetShardInfo{
UserSpace: n.UserSpace,
FileHash: n.FileHash,
ShardInfo: n.FileInfoVar().Var().VarID,
}, nil
}
func (t *ShardWriteNode) Input() dag.StreamInputSlot {
return dag.StreamInputSlot{
type StoreShardNode struct {
dag.NodeBase
UserSpace clitypes.UserSpaceDetail
ShardInfoKey string
}
func (b *GraphNodeBuilder) NewStoreShard(userSpace clitypes.UserSpaceDetail, shardInfoKey string) *StoreShardNode {
node := &StoreShardNode{
UserSpace: userSpace,
ShardInfoKey: shardInfoKey,
}
b.AddNode(node)
node.InputValues().Init(1)
node.OutputValues().Init(node, 1)
return node
}
func (t *StoreShardNode) FileInfoSlot() dag.ValueInputSlot {
return dag.ValueInputSlot{
Node: t,
Index: 0,
}
}
func (t *ShardWriteNode) FileHashVar() *dag.ValueVar {
return t.OutputValues().Get(0)
func (t *StoreShardNode) ShardInfoVar() dag.ValueOutputSlot {
return dag.ValueOutputSlot{
Node: t,
Index: 0,
}
}
func (t *ShardWriteNode) GenerateOp() (exec.Op, error) {
return &ShardWrite{
Input: t.InputStreams().Get(0).VarID,
FileHashVar: t.OutputValues().Get(0).VarID,
UserSpace: t.UserSpace,
func (t *StoreShardNode) GenerateOp() (exec.Op, error) {
return &StoreShard{
UserSpace: t.UserSpace,
FileInfo: t.FileInfoSlot().Var().VarID,
ShardInfo: t.ShardInfoVar().Var().VarID,
}, nil
}
// func (t *IPFSWriteType) String() string {
// return fmt.Sprintf("IPFSWrite[%s,%v+%v]%v%v", t.FileHashStoreKey, t.Range.Offset, t.Range.Length, formatStreamIO(node), formatValueIO(node))
// }

14
common/pkgs/ioswitchlrc/parser/generator.go
View File

@ -71,7 +71,7 @@ func buildDAGEncode(ctx *GenerateContext, fr ioswitchlrc.From, toes []ioswitchlr
}
ctx.ToNodes[to] = toNode
toNode.SetInput(frNode.Output().Var())
frNode.Output().ToSlot(toNode.Input())
} else if idx < ctx.LRC.K {
dataToes = append(dataToes, to)
} else {
@ -94,7 +94,7 @@ func buildDAGEncode(ctx *GenerateContext, fr ioswitchlrc.From, toes []ioswitchlr
}
ctx.ToNodes[to] = toNode
toNode.SetInput(splitNode.SubStream(to.GetDataIndex()))
splitNode.SubStream(to.GetDataIndex()).ToSlot(toNode.Input())
}
if len(parityToes) == 0 {
@ -116,7 +116,7 @@ func buildDAGEncode(ctx *GenerateContext, fr ioswitchlrc.From, toes []ioswitchlr
}
ctx.ToNodes[to] = toNode
toNode.SetInput(conType.NewOutput(to.GetDataIndex()))
conType.NewOutput(to.GetDataIndex()).ToSlot(toNode.Input())
}
return nil
}
@ -174,7 +174,7 @@ func buildDAGReconstructAny(ctx *GenerateContext, frs []ioswitchlrc.From, toes [
}
ctx.ToNodes[to] = toNode
toNode.SetInput(fr.Output().Var())
fr.Output().ToSlot(toNode.Input())
continue
}
@ -203,7 +203,7 @@ func buildDAGReconstructAny(ctx *GenerateContext, frs []ioswitchlrc.From, toes [
}
ctx.ToNodes[to] = toNode
toNode.SetInput(conNode.NewOutput(to.GetDataIndex()))
conNode.NewOutput(to.GetDataIndex()).ToSlot(toNode.Input())
}
if len(completeToes) == 0 {
@ -230,7 +230,7 @@ func buildDAGReconstructAny(ctx *GenerateContext, frs []ioswitchlrc.From, toes [
}
ctx.ToNodes[to] = toNode
toNode.SetInput(joinNode.Joined())
joinNode.Joined().ToSlot(toNode.Input())
}
// 如果不需要Construct任何块，则删除这个节点
@ -293,7 +293,7 @@ func buildDAGReconstructGroup(ctx *GenerateContext, frs []ioswitchlrc.From, toes
}
ctx.ToNodes[to] = toNode
toNode.SetInput(missedBlk)
missedBlk.ToSlot(toNode.Input())
}
return nil

43
common/pkgs/ioswitchlrc/parser/passes.go
View File

@ -6,6 +6,7 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/common/utils/os2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
@ -63,19 +64,23 @@ func buildFromNode(ctx *GenerateContext, f ioswitchlrc.From) (ops2.FromNode, err
switch f := f.(type) {
case *ioswitchlrc.FromNode:
t := ctx.DAG.NewShardRead(f, f.Space, types.NewOpen(f.FileHash))
getShard := ctx.DAG.NewGetShardInfo(f.Space, f.FileHash)
getShard.Env().ToEnvDriver()
getShard.Env().Pinned = true
read := ctx.DAG.NewBaseReadPathVar(f, f.Space, types.DefaultOpen())
if f.DataIndex == -1 {
t.Open.WithNullableLength(repRange.Offset, repRange.Length)
read.Option.WithNullableLength(repRange.Offset, repRange.Length)
} else {
t.Open.WithNullableLength(blkRange.Offset, blkRange.Length)
read.Option.WithNullableLength(blkRange.Offset, blkRange.Length)
}
// TODO2 支持HTTP协议
t.Env().ToEnvWorker(&ioswitchlrc.HubWorker{Hub: f.Space.RecommendHub, Address: *f.Space.RecommendHub.Address.(*cortypes.GRPCAddressInfo)})
t.Env().Pinned = true
getShard.Env().ToEnvWorker(&ioswitchlrc.HubWorker{Hub: f.Space.RecommendHub, Address: *f.Space.RecommendHub.Address.(*cortypes.GRPCAddressInfo)})
getShard.Env().Pinned = true
return t, nil
return read, nil
case *ioswitchlrc.FromDriver:
n := ctx.DAG.NewFromDriver(f.Handle)
@ -100,21 +105,29 @@ func buildFromNode(ctx *GenerateContext, f ioswitchlrc.From) (ops2.FromNode, err
func buildToNode(ctx *GenerateContext, t ioswitchlrc.To) (ops2.ToNode, error) {
switch t := t.(type) {
case *ioswitchlrc.ToNode:
n := ctx.DAG.NewShardWrite(t, t.Space, t.FileHashStoreKey)
tempFileName := os2.GenerateRandomFileName(20)
write := ctx.DAG.NewBaseWrite(t, t.Space, tempFileName)
switch addr := t.Space.RecommendHub.Address.(type) {
// case *cdssdk.HttpAddressInfo:
// n.Env().ToEnvWorker(&ioswitchlrc.HttpHubWorker{Node: t.Hub})
// TODO2 支持HTTP协议
case *cortypes.GRPCAddressInfo:
n.Env().ToEnvWorker(&ioswitchlrc.HubWorker{Hub: t.Space.RecommendHub, Address: *addr})
write.Env().ToEnvWorker(&ioswitchlrc.HubWorker{Hub: t.Space.RecommendHub, Address: *addr})
default:
return nil, fmt.Errorf("unsupported node address type %T", addr)
}
n.Env().Pinned = true
write.Env().Pinned = true
return n, nil
add := ctx.DAG.NewStoreShard(t.Space, t.FileHashStoreKey)
add.Env().ToEnvDriver()
add.Env().Pinned = true
write.FileInfoVar().ToSlot(add.FileInfoSlot())
return write, nil
case *ioswitchlrc.ToDriver:
n := ctx.DAG.NewToDriver(t.Handle)
@ -207,15 +220,15 @@ func dropUnused(ctx *GenerateContext) {
// 为IPFS写入指令存储结果
func storeIPFSWriteResult(ctx *GenerateContext) {
dag.WalkOnlyType[*ops2.ShardWriteNode](ctx.DAG.Graph, func(n *ops2.ShardWriteNode) bool {
if n.FileHashStoreKey == "" {
dag.WalkOnlyType[*ops2.StoreShardNode](ctx.DAG.Graph, func(n *ops2.StoreShardNode) bool {
if n.ShardInfoKey == "" {
return true
}
storeNode := ctx.DAG.NewStore()
storeNode.Env().ToEnvDriver()
storeNode.Store(n.FileHashStoreKey, n.FileHashVar())
storeNode.Store(n.ShardInfoKey, n.ShardInfoVar().Var())
return true
})
}
@ -238,7 +251,7 @@ func generateRange(ctx *GenerateContext) {
Length: toRng.Length,
})
toInput.Var().NotTo(toNode)
toNode.SetInput(rnged)
rnged.ToSlot(toInput)
} else {
stripSize := int64(ctx.LRC.ChunkSize * ctx.LRC.K)
@ -254,7 +267,7 @@ func generateRange(ctx *GenerateContext) {
Length: toRng.Length,
})
toInput.Var().NotTo(toNode)
toNode.SetInput(rnged)
rnged.ToSlot(toInput)
}
}
}

2
common/pkgs/rpc/hub/user_space.go
View File

@ -19,7 +19,7 @@ type BaseStoreListAll struct {
Path string
}
type BaseStoreListAllResp struct {
Entries []stgtypes.BaseStoreEntry
Entries []stgtypes.ListEntry
}
func (c *Client) BaseStoreListAll(ctx context.Context, req *BaseStoreListAll) (*BaseStoreListAllResp, *rpc.CodeError) {

42
common/pkgs/storage/efile/ec_multiplier.go
View File

@ -15,16 +15,15 @@ import (
)
type ECMultiplier struct {
blder *builder
url string
feat *cortypes.ECMultiplierFeature
outputs []string
completed bool
blder *builder
url string
feat *cortypes.ECMultiplierFeature
outputs []string
}
// 进行EC运算，coef * inputs。coef为编码矩阵，inputs为待编码数据，chunkSize为分块大小。
// 输出为每一个块文件的路径，数组长度 = len(coef)
func (m *ECMultiplier) Multiply(coef [][]byte, inputs []types.HTTPRequest, chunkSize int) ([]types.BypassedFileInfo, error) {
func (m *ECMultiplier) Multiply(coef [][]byte, inputs []types.HTTPRequest, chunkSize int) ([]types.FileInfo, error) {
type Request struct {
Inputs []types.HTTPRequest `json:"inputs"`
Outputs []string `json:"outputs"`
@ -94,9 +93,9 @@ func (m *ECMultiplier) Multiply(coef [][]byte, inputs []types.HTTPRequest, chunk
return nil, fmt.Errorf("data length not match outputs length")
}
ret := make([]types.BypassedFileInfo, len(r.Data))
ret := make([]types.FileInfo, len(r.Data))
for i, data := range r.Data {
ret[i] = types.BypassedFileInfo{
ret[i] = types.FileInfo{
Path: m.outputs[i],
Size: data.Size,
Hash: clitypes.NewFullHashFromString(data.Sha256),
@ -106,29 +105,4 @@ func (m *ECMultiplier) Multiply(coef [][]byte, inputs []types.HTTPRequest, chunk
return ret, nil
}
// 完成计算
func (m *ECMultiplier) Complete() {
m.completed = true
}
// 取消计算。如果已经调用了Complete，则应该无任何影响
func (m *ECMultiplier) Abort() {
if !m.completed {
u, err := url.JoinPath(m.url, "efile/openapi/v2/file/remove")
if err != nil {
return
}
token, err := m.blder.getToken()
if err != nil {
return
}
for _, output := range m.outputs {
http2.PostJSON(u, http2.RequestParam{
Header: map[string]string{"token": token},
Query: map[string]string{"paths": output},
})
}
}
}
func (m *ECMultiplier) Close() {}

61
common/pkgs/storage/local/base_store.go
View File

@ -48,19 +48,34 @@ func (s *BaseStore) Write(objPath string, stream io.Reader) (types.FileInfo, err
}
return types.FileInfo{
Hash: clitypes.NewFullHash(hasher.Sum()),
Path: objPath,
Size: counter.Count(),
Hash: clitypes.NewFullHash(hasher.Sum()),
}, nil
}
func (s *BaseStore) Read(objPath string) (io.ReadCloser, error) {
func (s *BaseStore) Read(objPath string, opt types.OpenOption) (io.ReadCloser, error) {
absObjPath := filepath.Join(s.root, objPath)
f, err := os.Open(absObjPath)
file, err := os.Open(absObjPath)
if err != nil {
return nil, err
}
return f, nil
if opt.Offset > 0 {
_, err = file.Seek(opt.Offset, io.SeekStart)
if err != nil {
file.Close()
return nil, err
}
}
var ret io.ReadCloser = file
if opt.Length >= 0 {
ret = io2.Length(ret, opt.Length)
}
return ret, nil
}
func (s *BaseStore) Mkdir(path string) error {
@ -73,10 +88,10 @@ func (s *BaseStore) Mkdir(path string) error {
return nil
}
func (s *BaseStore) ListAll(path string) ([]types.BaseStoreEntry, error) {
func (s *BaseStore) ListAll(path string) ([]types.ListEntry, error) {
absObjPath := filepath.Join(s.root, path)
var es []types.BaseStoreEntry
var es []types.ListEntry
err := filepath.WalkDir(absObjPath, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
@ -88,7 +103,7 @@ func (s *BaseStore) ListAll(path string) ([]types.BaseStoreEntry, error) {
}
if d.IsDir() {
es = append(es, types.BaseStoreEntry{
es = append(es, types.ListEntry{
Path: filepath.ToSlash(relaPath),
Size: 0,
IsDir: true,
@ -100,7 +115,7 @@ func (s *BaseStore) ListAll(path string) ([]types.BaseStoreEntry, error) {
return err
}
es = append(es, types.BaseStoreEntry{
es = append(es, types.ListEntry{
Path: filepath.ToSlash(relaPath),
Size: info.Size(),
IsDir: false,
@ -117,6 +132,36 @@ func (s *BaseStore) ListAll(path string) ([]types.BaseStoreEntry, error) {
return es, nil
}
func (s *BaseStore) CleanTemps() {
log := s.getLogger()
entries, err := os.ReadDir(filepath.Join(s.root, TempDir))
if err != nil {
log.Warnf("read temp dir: %v", err)
return
}
for _, entry := range entries {
if entry.IsDir() {
continue
}
info, err := entry.Info()
if err != nil {
log.Warnf("get temp file %v info: %v", entry.Name(), err)
continue
}
path := filepath.Join(s.root, TempDir, entry.Name())
err = os.Remove(path)
if err != nil {
log.Warnf("remove temp file %v: %v", path, err)
} else {
log.Infof("remove unused temp file %v, size: %v, last mod time: %v", path, info.Size(), info.ModTime())
}
}
}
func (s *BaseStore) getLogger() logger.Logger {
return logger.WithField("BaseStore", "Local").WithField("UserSpace", s.detail.UserSpace)
}

15
common/pkgs/storage/local/multipart_upload.go
View File

@ -86,14 +86,14 @@ func (i *MultipartTask) InitState() types.MultipartInitState {
}
}
func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPartInfo) (types.BypassedFileInfo, error) {
func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPartInfo) (types.FileInfo, error) {
parts = sort2.Sort(parts, func(l, r types.UploadedPartInfo) int {
return l.PartNumber - r.PartNumber
})
joined, err := os.Create(i.joinedFilePath)
if err != nil {
return types.BypassedFileInfo{}, err
return types.FileInfo{}, err
}
defer joined.Close()
@ -103,14 +103,14 @@ func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPar
for _, part := range parts {
partSize, err := i.writePart(part, joined, hasher)
if err != nil {
return types.BypassedFileInfo{}, err
return types.FileInfo{}, err
}
size += partSize
}
h := hasher.Sum(nil)
return types.BypassedFileInfo{
return types.FileInfo{
Path: joined.Name(),
Size: size,
Hash: clitypes.NewFullHash(h),
@ -147,11 +147,6 @@ func (i *MultipartTask) writePart(partInfo types.UploadedPartInfo, joined *os.Fi
return size, nil
}
func (i *MultipartTask) Complete() {
i.Abort()
}
func (i *MultipartTask) Abort() {
os.Remove(i.joinedFilePath)
func (i *MultipartTask) Close() {
os.RemoveAll(i.tempPartsDir)
}

37
common/pkgs/storage/local/s2s.go
View File

@ -2,12 +2,9 @@ package local
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"gitlink.org.cn/cloudream/common/utils/os2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
@ -34,43 +31,33 @@ func (s *S2STransfer) CanTransfer(src *clitypes.UserSpaceDetail) bool {
}
// 执行数据直传
func (s *S2STransfer) Transfer(ctx context.Context, src *clitypes.UserSpaceDetail, srcPath string, opt types.S2SOption) (string, error) {
absTempDir, err := filepath.Abs(s.feat.TempDir)
if err != nil {
return "", fmt.Errorf("get abs temp dir %v: %v", s.feat.TempDir, err)
}
s.dstPath = opt.DestPathHint
if s.dstPath == "" {
s.dstPath = filepath.Join(absTempDir, os2.GenerateRandomFileName(10))
}
func (s *S2STransfer) Transfer(ctx context.Context, src *clitypes.UserSpaceDetail, srcPath string, dstPath string) (types.FileInfo, error) {
s.dstPath = dstPath
copy, err := os.OpenFile(s.dstPath, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return "", err
return types.FileInfo{}, err
}
defer copy.Close()
srcFile, err := os.Open(srcPath)
if err != nil {
return "", err
return types.FileInfo{}, err
}
defer srcFile.Close()
_, err = io.Copy(copy, srcFile)
n, err := io.Copy(copy, srcFile)
if err != nil {
return "", err
return types.FileInfo{}, err
}
return s.dstPath, nil
return types.FileInfo{
Path: dstPath,
Size: n,
Hash: "",
}, nil
}
func (s *S2STransfer) Complete() {
func (s *S2STransfer) Close() {
}
func (s *S2STransfer) Abort() {
if s.dstPath != "" {
os.Remove(s.dstPath)
}
}

286
common/pkgs/storage/local/shard_store.go
View File

@ -1,18 +1,14 @@
package local
import (
"crypto/sha256"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"sync"
"time"
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/io2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
)
@ -23,11 +19,10 @@ const (
)
type ShardStore struct {
detail *clitypes.UserSpaceDetail
absRoot string
lock sync.Mutex
workingTempFiles map[string]bool
done chan any
detail *clitypes.UserSpaceDetail
absRoot string
lock sync.Mutex
done chan any
}
func NewShardStore(root string, detail *clitypes.UserSpaceDetail) (*ShardStore, error) {
@ -37,163 +32,31 @@ func NewShardStore(root string, detail *clitypes.UserSpaceDetail) (*ShardStore,
}
return &ShardStore{
detail: detail,
absRoot: absRoot,
workingTempFiles: make(map[string]bool),
done: make(chan any, 1),
detail: detail,
absRoot: absRoot,
done: make(chan any, 1),
}, nil
}
func (s *ShardStore) Start(ch *types.StorageEventChan) {
s.getLogger().Infof("component start, root: %v, max size: %v", s.absRoot, s.detail.UserSpace.ShardStore.MaxSize)
go func() {
removeTempTicker := time.NewTicker(time.Minute * 10)
defer removeTempTicker.Stop()
for {
select {
case <-removeTempTicker.C:
s.removeUnusedTempFiles()
case <-s.done:
return
}
}
}()
}
func (s *ShardStore) removeUnusedTempFiles() {
s.lock.Lock()
defer s.lock.Unlock()
log := s.getLogger()
entries, err := os.ReadDir(filepath.Join(s.absRoot, TempDir))
if err != nil {
log.Warnf("read temp dir: %v", err)
return
}
for _, entry := range entries {
if entry.IsDir() {
continue
}
if s.workingTempFiles[entry.Name()] {
continue
}
info, err := entry.Info()
if err != nil {
log.Warnf("get temp file %v info: %v", entry.Name(), err)
continue
}
path := filepath.Join(s.absRoot, TempDir, entry.Name())
err = os.Remove(path)
if err != nil {
log.Warnf("remove temp file %v: %v", path, err)
} else {
log.Infof("remove unused temp file %v, size: %v, last mod time: %v", path, info.Size(), info.ModTime())
}
}
}
func (s *ShardStore) Stop() {
s.getLogger().Infof("component stop")
select {
case s.done <- nil:
default:
}
}
func (s *ShardStore) Create(stream io.Reader) (types.FileInfo, error) {
file, err := s.createTempFile()
if err != nil {
return types.FileInfo{}, err
}
counter := io2.Counter(stream)
size, hash, err := s.writeTempFile(file, counter)
// TODO2
// if stgglb.Stats.HubStorageTransfer != nil {
// stgglb.Stats.HubStorageTransfer.RecordUpload(s.detail.Storage.StorageID, counter.Count(), err == nil)
// }
if err != nil {
// Name是文件完整路径
s.onCreateFailed(file.Name())
return types.FileInfo{}, err
}
return s.onCreateFinished(file.Name(), size, hash)
}
func (s *ShardStore) createTempFile() (*os.File, error) {
func (s *ShardStore) Store(path string, hash clitypes.FileHash, size int64) (types.FileInfo, error) {
s.lock.Lock()
defer s.lock.Unlock()
tmpDir := filepath.Join(s.absRoot, TempDir)
err := os.MkdirAll(tmpDir, 0755)
if err != nil {
s.lock.Unlock()
return nil, err
}
file, err := os.CreateTemp(tmpDir, "tmp-*")
if err != nil {
s.lock.Unlock()
return nil, err
}
s.workingTempFiles[filepath.Base(file.Name())] = true
return file, nil
}
func (s *ShardStore) writeTempFile(file *os.File, stream io.Reader) (int64, clitypes.FileHash, error) {
defer file.Close()
buf := make([]byte, 32*1024)
size := int64(0)
hasher := sha256.New()
for {
n, err := stream.Read(buf)
if n > 0 {
size += int64(n)
io2.WriteAll(hasher, buf[:n])
err := io2.WriteAll(file, buf[:n])
if err != nil {
return 0, "", err
}
}
if err == io.EOF {
break
}
if err != nil {
return 0, "", err
}
}
h := hasher.Sum(nil)
return size, clitypes.NewFullHash(h), nil
}
func (s *ShardStore) onCreateFinished(tempFilePath string, size int64, hash clitypes.FileHash) (types.FileInfo, error) {
s.lock.Lock()
defer s.lock.Unlock()
defer delete(s.workingTempFiles, filepath.Base(tempFilePath))
log := s.getLogger()
log.Debugf("write file %v finished, size: %v, hash: %v", tempFilePath, size, hash)
log.Debugf("%v bypass uploaded, size: %v, hash: %v", path, size, hash)
blockDir := s.getFileDirFromHash(hash)
err := os.MkdirAll(blockDir, 0755)
if err != nil {
s.removeTempFile(tempFilePath)
log.Warnf("make block dir %v: %v", blockDir, err)
return types.FileInfo{}, fmt.Errorf("making block dir: %w", err)
}
@ -201,77 +64,24 @@ func (s *ShardStore) onCreateFinished(tempFilePath string, size int64, hash clit
newPath := filepath.Join(blockDir, string(hash))
_, err = os.Stat(newPath)
if os.IsNotExist(err) {
err = os.Rename(tempFilePath, newPath)
err = os.Rename(path, newPath)
if err != nil {
s.removeTempFile(tempFilePath)
log.Warnf("rename %v to %v: %v", tempFilePath, newPath, err)
log.Warnf("rename %v to %v: %v", path, newPath, err)
return types.FileInfo{}, fmt.Errorf("rename file: %w", err)
}
} else if err != nil {
s.removeTempFile(tempFilePath)
log.Warnf("get file %v stat: %v", newPath, err)
return types.FileInfo{}, fmt.Errorf("get file stat: %w", err)
} else {
s.removeTempFile(tempFilePath)
}
return types.FileInfo{
Hash: hash,
Size: size,
Description: newPath,
Hash: hash,
Size: size,
Path: newPath,
}, nil
}
func (s *ShardStore) onCreateFailed(tempFilePath string) {
s.lock.Lock()
defer s.lock.Unlock()
s.getLogger().Debugf("writting file %v aborted", tempFilePath)
s.removeTempFile(tempFilePath)
delete(s.workingTempFiles, filepath.Base(tempFilePath))
}
func (s *ShardStore) removeTempFile(path string) {
err := os.Remove(path)
if err != nil {
s.getLogger().Warnf("removing temp file %v: %v", path, err)
}
}
// 使用NewOpen函数创建Option对象
func (s *ShardStore) Open(opt types.OpenOption) (io.ReadCloser, error) {
s.lock.Lock()
defer s.lock.Unlock()
filePath := s.getFilePathFromHash(opt.FileHash)
file, err := os.Open(filePath)
if err != nil {
return nil, err
}
if opt.Offset > 0 {
_, err = file.Seek(opt.Offset, io.SeekStart)
if err != nil {
file.Close()
return nil, err
}
}
var ret io.ReadCloser = file
if opt.Length >= 0 {
ret = io2.Length(ret, opt.Length)
}
return io2.CounterCloser(ret, func(cnt int64, err error) {
// TODO2
// if stgglb.Stats.HubStorageTransfer != nil {
// stgglb.Stats.HubStorageTransfer.RecordDownload(s.detail.Storage.StorageID, cnt, err == nil || err == io.EOF)
// }
}), nil
}
func (s *ShardStore) Info(hash clitypes.FileHash) (types.FileInfo, error) {
s.lock.Lock()
defer s.lock.Unlock()
@ -283,9 +93,9 @@ func (s *ShardStore) Info(hash clitypes.FileHash) (types.FileInfo, error) {
}
return types.FileInfo{
Hash: hash,
Size: info.Size(),
Description: filePath,
Hash: hash,
Size: info.Size(),
Path: filePath,
}, nil
}
@ -316,9 +126,9 @@ func (s *ShardStore) ListAll() ([]types.FileInfo, error) {
}
infos = append(infos, types.FileInfo{
Hash: fileHash,
Size: info.Size(),
Description: filepath.Join(blockDir, path),
Hash: fileHash,
Size: info.Size(),
Path: filepath.Join(blockDir, path),
})
return nil
})
@ -399,59 +209,3 @@ func (s *ShardStore) getFileDirFromHash(hash clitypes.FileHash) string {
func (s *ShardStore) getFilePathFromHash(hash clitypes.FileHash) string {
return filepath.Join(s.absRoot, BlocksDir, hash.GetHashPrefix(2), string(hash))
}
var _ types.BypassShardWrite = (*ShardStore)(nil)
func (s *ShardStore) BypassedShard(info types.BypassedFileInfo) error {
s.lock.Lock()
defer s.lock.Unlock()
log := s.getLogger()
log.Debugf("%v bypass uploaded, size: %v, hash: %v", info.Path, info.Size, info.Hash)
blockDir := s.getFileDirFromHash(info.Hash)
err := os.MkdirAll(blockDir, 0755)
if err != nil {
log.Warnf("make block dir %v: %v", blockDir, err)
return fmt.Errorf("making block dir: %w", err)
}
newPath := filepath.Join(blockDir, string(info.Hash))
_, err = os.Stat(newPath)
if os.IsNotExist(err) {
err = os.Rename(info.Path, newPath)
if err != nil {
log.Warnf("rename %v to %v: %v", info.Path, newPath, err)
return fmt.Errorf("rename file: %w", err)
}
} else if err != nil {
log.Warnf("get file %v stat: %v", newPath, err)
return fmt.Errorf("get file stat: %w", err)
}
return nil
}
var _ types.BypassShardRead = (*ShardStore)(nil)
func (s *ShardStore) BypassShardRead(fileHash clitypes.FileHash) (types.BypassFilePath, error) {
s.lock.Lock()
defer s.lock.Unlock()
filePath := s.getFilePathFromHash(fileHash)
stat, err := os.Stat(filePath)
if err != nil {
return types.BypassFilePath{}, err
}
return types.BypassFilePath{
Path: filePath,
Info: types.FileInfo{
Hash: fileHash,
Size: stat.Size(),
Description: filePath,
},
}, nil
}

10
common/pkgs/storage/obs/obs.go
View File

@ -31,11 +31,11 @@ func newBuilder(detail *clitypes.UserSpaceDetail) types.StorageBuilder {
func (b *builder) FeatureDesc() types.FeatureDesc {
return types.FeatureDesc{
HasBypassShardWrite: true,
HasBypassPublicWrite: true,
HasBypassShardRead: true,
HasBypassPublicRead: true,
HasBypassHTTPRead: true,
HasBypassShardWrite: true,
HasBypassBaseWrite: true,
HasBypassShardRead: true,
HasBypassBaseRead: true,
HasBypassHTTPRead: true,
}
}

8
common/pkgs/storage/obs/obs_test.go
View File

@ -6,7 +6,6 @@ import (
. "github.com/smartystreets/goconvey/convey"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
)
@ -28,7 +27,7 @@ func Test_S2S(t *testing.T) {
},
}
newPath, err := s2s.Transfer(context.TODO(), &clitypes.UserSpaceDetail{
_, err := s2s.Transfer(context.TODO(), &clitypes.UserSpaceDetail{
UserSpace: clitypes.UserSpace{
Storage: &cortypes.OBSType{
Region: "cn-north-4",
@ -41,10 +40,9 @@ func Test_S2S(t *testing.T) {
SK: "",
},
},
}, "test_data/test03.txt", types.S2SOption{})
defer s2s.Abort()
}, "test_data/test03.txt", "atest.txt")
defer s2s.Close()
So(err, ShouldEqual, nil)
t.Logf("newPath: %s", newPath)
})
}

63
common/pkgs/storage/obs/s2s.go
View File

@ -6,6 +6,8 @@ import (
"fmt"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/huaweicloud/huaweicloud-sdk-go-v3/core/auth/basic"
oms "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/oms/v2"
"github.com/huaweicloud/huaweicloud-sdk-go-v3/services/oms/v2/model"
@ -40,10 +42,10 @@ func (s *S2STransfer) CanTransfer(src *clitypes.UserSpaceDetail) bool {
}
// 执行数据直传。返回传输后的文件路径
func (s *S2STransfer) Transfer(ctx context.Context, src *clitypes.UserSpaceDetail, srcPath string, opt types.S2SOption) (string, error) {
func (s *S2STransfer) Transfer(ctx context.Context, src *clitypes.UserSpaceDetail, srcPath string, dstPath string) (types.FileInfo, error) {
req := s.makeRequest(src, srcPath)
if req == nil {
return "", fmt.Errorf("unsupported source storage type: %T", src.UserSpace.Storage)
return types.FileInfo{}, fmt.Errorf("unsupported source storage type: %T", src.UserSpace.Storage)
}
auth, err := basic.NewCredentialsBuilder().
@ -52,12 +54,12 @@ func (s *S2STransfer) Transfer(ctx context.Context, src *clitypes.UserSpaceDetai
WithProjectId(s.stgType.ProjectID).
SafeBuild()
if err != nil {
return "", err
return types.FileInfo{}, err
}
region, err := omsregion.SafeValueOf(s.stgType.Region)
if err != nil {
return "", err
return types.FileInfo{}, err
}
cli, err := oms.OmsClientBuilder().
@ -65,9 +67,10 @@ func (s *S2STransfer) Transfer(ctx context.Context, src *clitypes.UserSpaceDetai
WithCredential(auth).
SafeBuild()
if err != nil {
return "", err
return types.FileInfo{}, err
}
// 先上传成一个临时文件
tempPrefix := s3.JoinKey(s.feat.TempDir, os2.GenerateRandomFileName(10)) + "/"
taskType := model.GetCreateTaskReqTaskTypeEnum().OBJECT
@ -86,17 +89,37 @@ func (s *S2STransfer) Transfer(ctx context.Context, src *clitypes.UserSpaceDetai
},
})
if err != nil {
return "", fmt.Errorf("create task: %w", err)
return types.FileInfo{}, fmt.Errorf("create task: %w", err)
}
s.taskID = resp.Id
err = s.waitTask(ctx, *resp.Id)
// 轮询任务状态，直到完成
size, err := s.waitTask(ctx, *resp.Id)
if err != nil {
return "", fmt.Errorf("wait task: %w", err)
return types.FileInfo{}, fmt.Errorf("wait task: %w", err)
}
return s3.JoinKey(tempPrefix, srcPath), nil
// 传输完成后，将文件名改成目标路径
obsCli, bkt, err := createClient(s.stgType, s.cred)
if err != nil {
return types.FileInfo{}, err
}
_, err = obsCli.CopyObject(ctx, &awss3.CopyObjectInput{
Bucket: aws.String(bkt),
CopySource: aws.String(s3.JoinKey(bkt, tempPrefix, srcPath)),
Key: aws.String(dstPath),
})
if err != nil {
return types.FileInfo{}, fmt.Errorf("copy object: %w", err)
}
return types.FileInfo{
Path: dstPath,
Size: size,
Hash: "",
}, nil
}
func (s *S2STransfer) makeRequest(srcStg *clitypes.UserSpaceDetail, srcPath string) *model.SrcNodeReq {
@ -123,7 +146,7 @@ func (s *S2STransfer) makeRequest(srcStg *clitypes.UserSpaceDetail, srcPath stri
}
}
func (s *S2STransfer) waitTask(ctx context.Context, taskId int64) error {
func (s *S2STransfer) waitTask(ctx context.Context, taskId int64) (int64, error) {
ticker := time.NewTicker(time.Second * 5)
defer ticker.Stop()
@ -139,38 +162,32 @@ func (s *S2STransfer) waitTask(ctx context.Context, taskId int64) error {
continue
}
return fmt.Errorf("show task failed too many times: %w", err)
return 0, fmt.Errorf("show task failed too many times: %w", err)
}
failures = 0
if *resp.Status == 3 {
return fmt.Errorf("task stopped")
return 0, fmt.Errorf("task stopped")
}
if *resp.Status == 4 {
return errors.New(resp.ErrorReason.String())
return 0, errors.New(resp.ErrorReason.String())
}
if *resp.Status == 5 {
return nil
return *resp.CompleteSize, nil
}
select {
case <-ticker.C:
continue
case <-ctx.Done():
return ctx.Err()
return 0, ctx.Err()
}
}
}
// 完成传输
func (s *S2STransfer) Complete() {
}
// 取消传输。如果已经调用了Complete，则这个方法应该无效果
func (s *S2STransfer) Abort() {
func (s *S2STransfer) Close() {
if s.taskID != nil {
s.omsCli.StopTask(&model.StopTaskRequest{
TaskId: fmt.Sprintf("%v", *s.taskID),
@ -179,7 +196,5 @@ func (s *S2STransfer) Abort() {
s.omsCli.DeleteTask(&model.DeleteTaskRequest{
TaskId: fmt.Sprintf("%v", *s.taskID),
})
// TODO 清理临时文件
}
}

2
common/pkgs/storage/obs/shard_store.go
View File

@ -32,7 +32,7 @@ func NewShardStore(detail *clitypes.UserSpaceDetail, stgType *cortypes.OBSType,
return &sd, nil
}
func (s *ShardStore) HTTPBypassRead(fileHash clitypes.FileHash) (types.HTTPRequest, error) {
func (s *ShardStore) MakeHTTPReadRequest(fileHash clitypes.FileHash) (types.HTTPRequest, error) {
cli, err := obs.New(s.cred.AK, s.cred.SK, s.stgType.Endpoint)
if err != nil {
return types.HTTPRequest{}, err

106
common/pkgs/storage/s3/base_store.go
View File

@ -62,6 +62,7 @@ func (s *BaseStore) Write(objPath string, stream io.Reader) (types.FileInfo, err
}
return types.FileInfo{
Path: key,
Hash: clitypes.NewFullHash(hash),
Size: counter.Count(),
}, nil
@ -78,17 +79,24 @@ func (s *BaseStore) Write(objPath string, stream io.Reader) (types.FileInfo, err
}
return types.FileInfo{
Path: key,
Hash: clitypes.NewFullHash(hashStr.Sum()),
Size: counter.Count(),
}, nil
}
func (s *BaseStore) Read(objPath string) (io.ReadCloser, error) {
func (s *BaseStore) Read(objPath string, opt types.OpenOption) (io.ReadCloser, error) {
key := objPath
rngStr := fmt.Sprintf("bytes=%d-", opt.Offset)
if opt.Length >= 0 {
rngStr += fmt.Sprintf("%d", opt.Offset+opt.Length-1)
}
resp, err := s.cli.GetObject(context.TODO(), &s3.GetObjectInput{
Bucket: aws.String(s.Bucket),
Key: aws.String(key),
Range: aws.String(rngStr),
})
if err != nil {
@ -107,7 +115,7 @@ func (s *BaseStore) Mkdir(path string) error {
return err
}
func (s *BaseStore) ListAll(path string) ([]types.BaseStoreEntry, error) {
func (s *BaseStore) ListAll(path string) ([]types.ListEntry, error) {
key := path
// TODO 待测试
@ -117,7 +125,7 @@ func (s *BaseStore) ListAll(path string) ([]types.BaseStoreEntry, error) {
Delimiter: aws.String("/"),
}
var objs []types.BaseStoreEntry
var objs []types.ListEntry
var marker *string
for {
@ -128,7 +136,7 @@ func (s *BaseStore) ListAll(path string) ([]types.BaseStoreEntry, error) {
}
for _, obj := range resp.Contents {
objs = append(objs, types.BaseStoreEntry{
objs = append(objs, types.ListEntry{
Path: *obj.Key,
Size: *obj.Size,
IsDir: false,
@ -145,43 +153,59 @@ func (s *BaseStore) ListAll(path string) ([]types.BaseStoreEntry, error) {
return objs, nil
}
func (s *BaseStore) CleanTemps() {
log := s.getLogger()
var deletes []s3types.ObjectIdentifier
deleteObjs := make(map[string]s3types.Object)
var marker *string
for {
resp, err := s.cli.ListObjects(context.Background(), &s3.ListObjectsInput{
Bucket: aws.String(s.Bucket),
Prefix: aws.String(JoinKey(s.Detail.UserSpace.ShardStore.BaseDir, TempDir, "/")),
Marker: marker,
})
if err != nil {
log.Warnf("read temp dir: %v", err)
return
}
for _, obj := range resp.Contents {
deletes = append(deletes, s3types.ObjectIdentifier{
Key: obj.Key,
})
deleteObjs[*obj.Key] = obj
}
if !*resp.IsTruncated {
break
}
marker = resp.NextMarker
}
if len(deletes) == 0 {
return
}
resp, err := s.cli.DeleteObjects(context.Background(), &s3.DeleteObjectsInput{
Bucket: aws.String(s.Bucket),
Delete: &s3types.Delete{
Objects: deletes,
},
})
if err != nil {
log.Warnf("delete temp files: %v", err)
return
}
for _, del := range resp.Deleted {
obj := deleteObjs[*del.Key]
log.Infof("remove unused temp file %v, size: %v, last mod time: %v", *obj.Key, *obj.Size, *obj.LastModified)
}
}
func (s *BaseStore) getLogger() logger.Logger {
return logger.WithField("BaseStore", "S3").WithField("Storage", s.Detail.UserSpace.Storage.String())
}
var _ types.BypassPublicRead = (*BaseStore)(nil)
func (s *BaseStore) BypassPublicRead(pa string) (types.BypassFilePath, error) {
info, err := s.cli.HeadObject(context.TODO(), &s3.HeadObjectInput{
Bucket: aws.String(s.Bucket),
Key: aws.String(pa),
})
if err != nil {
s.getLogger().Warnf("get file %v: %v", pa, err)
return types.BypassFilePath{}, err
}
return types.BypassFilePath{
Path: pa,
Info: types.FileInfo{
Size: *info.ContentLength,
Description: pa,
},
}, nil
}
var _ types.BypassPublicWrite = (*BaseStore)(nil)
func (s *BaseStore) BypassedPublic(info types.BypassedFileInfo, dstPath string) error {
_, err := s.cli.CopyObject(context.TODO(), &s3.CopyObjectInput{
Bucket: aws.String(s.Bucket),
CopySource: aws.String(s.Bucket + "/" + info.Path),
Key: aws.String(dstPath),
})
if err != nil {
s.getLogger().Warnf("bypass public write %v to %v: %v", info.Path, dstPath, err)
return err
}
return nil
}

20
common/pkgs/storage/s3/multipart_upload.go
View File

@ -101,7 +101,7 @@ func (i *MultipartTask) InitState() types.MultipartInitState {
}
}
func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPartInfo) (types.BypassedFileInfo, error) {
func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPartInfo) (types.FileInfo, error) {
parts = sort2.Sort(parts, func(l, r types.UploadedPartInfo) int {
return l.PartNumber - r.PartNumber
})
@ -127,7 +127,7 @@ func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPar
},
})
if err != nil {
return types.BypassedFileInfo{}, err
return types.FileInfo{}, err
}
headResp, err := i.cli.HeadObject(ctx, &s3.HeadObjectInput{
@ -135,12 +135,12 @@ func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPar
Key: aws.String(i.tempFilePath),
})
if err != nil {
return types.BypassedFileInfo{}, err
return types.FileInfo{}, err
}
hash := clitypes.CalculateCompositeHash(partHashes)
return types.BypassedFileInfo{
return types.FileInfo{
Path: i.tempFilePath,
Size: *headResp.ContentLength,
Hash: hash,
@ -148,20 +148,10 @@ func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPar
}
func (i *MultipartTask) Complete() {
}
func (i *MultipartTask) Abort() {
// TODO2 根据注释描述，Abort不能停止正在上传的分片，需要等待其上传完成才能彻底删除，
// 考虑增加定时任务去定时清理
func (i *MultipartTask) Close() {
i.cli.AbortMultipartUpload(context.Background(), &s3.AbortMultipartUploadInput{
Bucket: aws.String(i.bucket),
Key: aws.String(i.tempFilePath),
UploadId: aws.String(i.uploadID),
})
i.cli.DeleteObject(context.Background(), &s3.DeleteObjectInput{
Bucket: aws.String(i.bucket),
Key: aws.String(i.tempFilePath),
})
}

10
common/pkgs/storage/s3/s3.go
View File

@ -30,11 +30,11 @@ func newBuilder(detail *clitypes.UserSpaceDetail) types.StorageBuilder {
func (b *builder) FeatureDesc() types.FeatureDesc {
return types.FeatureDesc{
HasBypassShardWrite: true,
HasBypassPublicWrite: true,
HasBypassShardRead: true,
HasBypassPublicRead: true,
HasBypassHTTPRead: false,
HasBypassShardWrite: true,
HasBypassBaseWrite: true,
HasBypassShardRead: true,
HasBypassBaseRead: true,
HasBypassHTTPRead: false,
}
}

343
common/pkgs/storage/s3/shard_store.go
View File

@ -2,20 +2,12 @@ package s3
import (
"context"
"crypto/sha256"
"errors"
"fmt"
"io"
"path/filepath"
"sync"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/io2"
"gitlink.org.cn/cloudream/common/utils/os2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
)
@ -30,286 +22,58 @@ type ShardStoreOption struct {
}
type ShardStore struct {
Detail *clitypes.UserSpaceDetail
Bucket string
cli *s3.Client
opt ShardStoreOption
lock sync.Mutex
workingTempFiles map[string]bool
done chan any
Detail *clitypes.UserSpaceDetail
Bucket string
cli *s3.Client
opt ShardStoreOption
lock sync.Mutex
}
func NewShardStore(detail *clitypes.UserSpaceDetail, cli *s3.Client, bkt string, opt ShardStoreOption) (*ShardStore, error) {
return &ShardStore{
Detail: detail,
Bucket: bkt,
cli: cli,
opt: opt,
workingTempFiles: make(map[string]bool),
done: make(chan any, 1),
Detail: detail,
Bucket: bkt,
cli: cli,
opt: opt,
}, nil
}
func (s *ShardStore) Start(ch *types.StorageEventChan) {
s.getLogger().Infof("start, root: %v", s.Detail.UserSpace.ShardStore.BaseDir)
go func() {
removeTempTicker := time.NewTicker(time.Minute * 10)
defer removeTempTicker.Stop()
for {
select {
case <-removeTempTicker.C:
s.removeUnusedTempFiles()
case <-s.done:
return
}
}
}()
}
func (s *ShardStore) removeUnusedTempFiles() {
s.lock.Lock()
defer s.lock.Unlock()
log := s.getLogger()
var deletes []s3types.ObjectIdentifier
deleteObjs := make(map[string]s3types.Object)
var marker *string
for {
resp, err := s.cli.ListObjects(context.Background(), &s3.ListObjectsInput{
Bucket: aws.String(s.Bucket),
Prefix: aws.String(JoinKey(s.Detail.UserSpace.ShardStore.BaseDir, TempDir, "/")),
Marker: marker,
})
if err != nil {
log.Warnf("read temp dir: %v", err)
return
}
for _, obj := range resp.Contents {
objName := BaseKey(*obj.Key)
if s.workingTempFiles[objName] {
continue
}
deletes = append(deletes, s3types.ObjectIdentifier{
Key: obj.Key,
})
deleteObjs[*obj.Key] = obj
}
if !*resp.IsTruncated {
break
}
marker = resp.NextMarker
}
if len(deletes) == 0 {
return
}
resp, err := s.cli.DeleteObjects(context.Background(), &s3.DeleteObjectsInput{
Bucket: aws.String(s.Bucket),
Delete: &s3types.Delete{
Objects: deletes,
},
})
if err != nil {
log.Warnf("delete temp files: %v", err)
return
}
for _, del := range resp.Deleted {
obj := deleteObjs[*del.Key]
log.Infof("remove unused temp file %v, size: %v, last mod time: %v", *obj.Key, *obj.Size, *obj.LastModified)
}
}
func (s *ShardStore) Stop() {
s.getLogger().Infof("component stop")
select {
case s.done <- nil:
default:
}
}
func (s *ShardStore) Create(stream io.Reader) (types.FileInfo, error) {
if s.opt.UseAWSSha256 {
return s.createWithAwsSha256(stream)
} else {
return s.createWithCalcSha256(stream)
}
}
func (s *ShardStore) createWithAwsSha256(stream io.Reader) (types.FileInfo, error) {
log := s.getLogger()
key, fileName := s.createTempFile()
counter := io2.Counter(stream)
resp, err := s.cli.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(s.Bucket),
Key: aws.String(key),
Body: counter,
ChecksumAlgorithm: s3types.ChecksumAlgorithmSha256,
})
// TODO2
// if stgglb.Stats.HubStorageTransfer != nil {
// stgglb.Stats.HubStorageTransfer.RecordUpload(s.Detail.Storage.StorageID, counter.Count(), err == nil)
// }
if err != nil {
log.Warnf("uploading file %v: %v", key, err)
s.lock.Lock()
defer s.lock.Unlock()
delete(s.workingTempFiles, fileName)
return types.FileInfo{}, err
}
if resp.ChecksumSHA256 == nil {
log.Warnf("SHA256 checksum not found in response of uploaded file %v", key)
s.onCreateFailed(key, fileName)
return types.FileInfo{}, errors.New("SHA256 checksum not found in response")
}
hash, err := DecodeBase64Hash(*resp.ChecksumSHA256)
if err != nil {
log.Warnf("decode SHA256 checksum %v: %v", *resp.ChecksumSHA256, err)
s.onCreateFailed(key, fileName)
return types.FileInfo{}, fmt.Errorf("decode SHA256 checksum: %v", err)
}
return s.onCreateFinished(key, counter.Count(), clitypes.NewFullHash(hash))
}
func (s *ShardStore) createWithCalcSha256(stream io.Reader) (types.FileInfo, error) {
log := s.getLogger()
key, fileName := s.createTempFile()
hashStr := io2.NewReadHasher(sha256.New(), stream)
counter := io2.Counter(hashStr)
_, err := s.cli.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(s.Bucket),
Key: aws.String(key),
Body: counter,
})
// TODO2
// if stgglb.Stats.HubStorageTransfer != nil {
// stgglb.Stats.HubStorageTransfer.RecordUpload(s.Detail.Storage.StorageID, counter.Count(), err == nil)
// }
if err != nil {
log.Warnf("uploading file %v: %v", key, err)
s.lock.Lock()
defer s.lock.Unlock()
delete(s.workingTempFiles, fileName)
return types.FileInfo{}, err
}
return s.onCreateFinished(key, counter.Count(), clitypes.NewFullHash(hashStr.Sum()))
}
func (s *ShardStore) createTempFile() (string, string) {
func (s *ShardStore) Store(path string, hash clitypes.FileHash, size int64) (types.FileInfo, error) {
s.lock.Lock()
defer s.lock.Unlock()
tmpDir := JoinKey(s.Detail.UserSpace.ShardStore.BaseDir, TempDir)
tmpName := os2.GenerateRandomFileName(20)
s.workingTempFiles[tmpName] = true
return JoinKey(tmpDir, tmpName), tmpName
}
func (s *ShardStore) onCreateFinished(tempFilePath string, size int64, hash clitypes.FileHash) (types.FileInfo, error) {
s.lock.Lock()
defer s.lock.Unlock()
defer delete(s.workingTempFiles, filepath.Base(tempFilePath))
defer func() {
// 不管是否成功。即使失败了也有定时清理机制去兜底
s.cli.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(s.Bucket),
Key: aws.String(tempFilePath),
})
}()
log := s.getLogger()
log.Debugf("write file %v finished, size: %v, hash: %v", tempFilePath, size, hash)
log.Debugf("write file %v finished, size: %v, hash: %v", path, size, hash)
blockDir := s.GetFileDirFromHash(hash)
newPath := JoinKey(blockDir, string(hash))
_, err := s.cli.CopyObject(context.Background(), &s3.CopyObjectInput{
Bucket: aws.String(s.Bucket),
CopySource: aws.String(JoinKey(s.Bucket, tempFilePath)),
CopySource: aws.String(JoinKey(s.Bucket, path)),
Key: aws.String(newPath),
})
if err != nil {
log.Warnf("copy file %v to %v: %v", tempFilePath, newPath, err)
log.Warnf("copy file %v to %v: %v", path, newPath, err)
return types.FileInfo{}, err
}
return types.FileInfo{
Hash: hash,
Size: size,
Description: newPath,
Hash: hash,
Size: size,
Path: newPath,
}, nil
}
func (s *ShardStore) onCreateFailed(key string, fileName string) {
// 不管是否成功。即使失败了也有定时清理机制去兜底
s.cli.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(s.Bucket),
Key: aws.String(key),
})
s.lock.Lock()
defer s.lock.Unlock()
delete(s.workingTempFiles, fileName)
}
// 使用NewOpen函数创建Option对象
func (s *ShardStore) Open(opt types.OpenOption) (io.ReadCloser, error) {
s.lock.Lock()
defer s.lock.Unlock()
filePath := s.GetFilePathFromHash(opt.FileHash)
rngStr := fmt.Sprintf("bytes=%d-", opt.Offset)
if opt.Length >= 0 {
rngStr += fmt.Sprintf("%d", opt.Offset+opt.Length-1)
}
resp, err := s.cli.GetObject(context.TODO(), &s3.GetObjectInput{
Bucket: aws.String(s.Bucket),
Key: aws.String(filePath),
Range: aws.String(rngStr),
})
if err != nil {
s.getLogger().Warnf("get file %v: %v", filePath, err)
return nil, err
}
return io2.CounterCloser(resp.Body, func(cnt int64, err error) {
// TODO2
// if stgglb.Stats.HubStorageTransfer != nil {
// stgglb.Stats.HubStorageTransfer.RecordDownload(s.Detail.Storage.StorageID, cnt, err == nil || err == io.EOF)
// }
}), nil
}
func (s *ShardStore) Info(hash clitypes.FileHash) (types.FileInfo, error) {
s.lock.Lock()
defer s.lock.Unlock()
@ -325,9 +89,9 @@ func (s *ShardStore) Info(hash clitypes.FileHash) (types.FileInfo, error) {
}
return types.FileInfo{
Hash: hash,
Size: *info.ContentLength,
Description: filePath,
Hash: hash,
Size: *info.ContentLength,
Path: filePath,
}, nil
}
@ -361,9 +125,9 @@ func (s *ShardStore) ListAll() ([]types.FileInfo, error) {
}
infos = append(infos, types.FileInfo{
Hash: fileHash,
Size: *obj.Size,
Description: *obj.Key,
Hash: fileHash,
Size: *obj.Size,
Path: *obj.Key,
})
}
@ -462,66 +226,3 @@ func (s *ShardStore) GetFileDirFromHash(hash clitypes.FileHash) string {
func (s *ShardStore) GetFilePathFromHash(hash clitypes.FileHash) string {
return JoinKey(s.Detail.UserSpace.ShardStore.BaseDir, BlocksDir, hash.GetHashPrefix(2), string(hash))
}
var _ types.BypassShardWrite = (*ShardStore)(nil)
func (s *ShardStore) BypassedShard(info types.BypassedFileInfo) error {
if info.Hash == "" {
return fmt.Errorf("empty file hash is not allowed by this shard store")
}
s.lock.Lock()
defer s.lock.Unlock()
defer func() {
// 不管是否成功。即使失败了也有定时清理机制去兜底
s.cli.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(s.Bucket),
Key: aws.String(info.Path),
})
}()
log := s.getLogger()
log.Debugf("%v bypass uploaded, size: %v, hash: %v", info.Path, info.Size, info.Hash)
blockDir := s.GetFileDirFromHash(info.Hash)
newPath := JoinKey(blockDir, string(info.Hash))
_, err := s.cli.CopyObject(context.Background(), &s3.CopyObjectInput{
CopySource: aws.String(JoinKey(s.Bucket, info.Path)),
Bucket: aws.String(s.Bucket),
Key: aws.String(newPath),
})
if err != nil {
log.Warnf("copy file %v to %v: %v", info.Path, newPath, err)
return fmt.Errorf("copy file: %w", err)
}
return nil
}
var _ types.BypassShardRead = (*ShardStore)(nil)
func (s *ShardStore) BypassShardRead(fileHash clitypes.FileHash) (types.BypassFilePath, error) {
s.lock.Lock()
defer s.lock.Unlock()
filePath := s.GetFilePathFromHash(fileHash)
info, err := s.cli.HeadObject(context.TODO(), &s3.HeadObjectInput{
Bucket: aws.String(s.Bucket),
Key: aws.String(filePath),
})
if err != nil {
s.getLogger().Warnf("get file %v: %v", filePath, err)
return types.BypassFilePath{}, err
}
return types.BypassFilePath{
Path: filePath,
Info: types.FileInfo{
Hash: fileHash,
Size: *info.ContentLength,
Description: filePath,
},
}, nil
}

51
common/pkgs/storage/types/base_store.go
View File

@ -1,10 +1,11 @@
package types
import (
"fmt"
"io"
)
type BaseStoreEntry struct {
type ListEntry struct {
Path string
Size int64
IsDir bool
@ -12,11 +13,55 @@ type BaseStoreEntry struct {
type BaseStore interface {
Write(path string, stream io.Reader) (FileInfo, error)
Read(path string) (io.ReadCloser, error)
Read(path string, opt OpenOption) (io.ReadCloser, error)
// 创建指定路径的文件夹。对于不支持空文件夹的存储系统来说，可以采用创建以/结尾的对象的方式来模拟文件夹。
Mkdir(path string) error
// 返回指定路径下的所有文件，文件路径是包含path在内的完整路径。返回结果的第一条一定是路径本身，可能是文件，也可能是目录。
// 如果路径不存在，那么不会返回错误，而是返回一个空列表。
// 返回的内容严格按照存储系统的原始结果来，比如当存储系统是一个对象存储时，那么就可能不会包含目录，或者包含用于模拟的以“/”结尾的对象。
ListAll(path string) ([]BaseStoreEntry, error)
ListAll(path string) ([]ListEntry, error)
// 清空临时目录。只应该在此存储服务未被使用时调用
CleanTemps()
}
type OpenOption struct {
Offset int64
Length int64
}
func DefaultOpen() OpenOption {
return OpenOption{
Offset: 0,
Length: -1,
}
}
func (o *OpenOption) WithLength(len int64) OpenOption {
o.Length = len
return *o
}
// [start, end)，不包含end
func (o *OpenOption) WithRange(start int64, end int64) OpenOption {
o.Offset = start
o.Length = end - start
return *o
}
func (o *OpenOption) WithNullableLength(offset int64, length *int64) {
o.Offset = offset
if length != nil {
o.Length = *length
}
}
func (o *OpenOption) String() string {
rangeStart := fmt.Sprintf("%d", o.Offset)
rangeEnd := ""
if o.Length >= 0 {
rangeEnd = fmt.Sprintf("%d", o.Offset+o.Length)
}
return fmt.Sprintf("%s:%s", rangeStart, rangeEnd)
}

38
common/pkgs/storage/types/bypass.go
View File

@ -4,44 +4,10 @@ import (
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
)
// 通过旁路上传后的文件的信息
type BypassedFileInfo struct {
Path string
Hash clitypes.FileHash
Size int64
}
// 不通过ShardStore上传文件，但上传完成后需要通知ShardStore。
type BypassShardWrite interface {
BypassedShard(info BypassedFileInfo) error
}
// 不通过BaseStore上传文件。
type BypassPublicWrite interface {
BypassedPublic(info BypassedFileInfo, dstPath string) error
}
// 描述指定文件在分片存储中的路径。可以考虑设计成interface。
type BypassFilePath struct {
Path string
// 文件信息。此结构体中的字段不一定都会存在，比如如果是从BypassPublicRead接口获取的，那么Hash字段就为空。
Info FileInfo
}
// 不通过ShardStore读取文件，但需要它返回文件的路径。
type BypassShardRead interface {
BypassShardRead(fileHash clitypes.FileHash) (BypassFilePath, error)
}
// 不通过BaseStore读取文件。虽然仅使用path就已经可以读取到文件，但还是增加了此接口用于获取更详细的文件信息。
type BypassPublicRead interface {
BypassPublicRead(path string) (BypassFilePath, error)
}
// 能通过一个Http请求直接访问文件
// 仅用于分片存储。
type HTTPBypassShardRead interface {
HTTPBypassRead(fileHash clitypes.FileHash) (HTTPRequest, error)
type HTTPShardRead interface {
MakeHTTPReadRequest(fileHash clitypes.FileHash) (HTTPRequest, error)
}
type HTTPRequest struct {

7
common/pkgs/storage/types/ec_multiplier.go
View File

@ -3,9 +3,6 @@ package types
type ECMultiplier interface {
// 进行EC运算，coef * inputs。coef为编码矩阵，inputs为待编码数据，chunkSize为分块大小。
// 输出为每一个块文件的路径，数组长度 = len(coef)
Multiply(coef [][]byte, inputs []HTTPRequest, chunkSize int) ([]BypassedFileInfo, error)
// 完成计算
Complete()
// 取消计算。如果已经调用了Complete，则应该无任何影响
Abort()
Multiply(coef [][]byte, inputs []HTTPRequest, chunkSize int) ([]FileInfo, error)
Close()
}

1
common/pkgs/storage/types/empty_builder.go
View File

@ -13,6 +13,7 @@ type EmptyBuilder struct {
func (b *EmptyBuilder) FeatureDesc() FeatureDesc {
return FeatureDesc{}
}
func (b *EmptyBuilder) CreateShardStore() (ShardStore, error) {
return nil, fmt.Errorf("create shard store for %T: %w", b.Detail.UserSpace.Storage, ErrUnsupported)
}

15
common/pkgs/storage/types/s2s.go
View File

@ -6,19 +6,10 @@ import (
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
)
type S2SOption struct {
// 传输目的地路径。如果为空，则自动生成一个临时路径。
// 只是一个建议，非强制，如果存储服务不支持直接指定目的路径，则可以忽略这个参数。
DestPathHint string
}
type S2STransfer interface {
// 判断是否能从指定的源存储中直传到当前存储的目的路径。仅在生成计划时使用
CanTransfer(src *clitypes.UserSpaceDetail) bool
// 执行数据直传。返回传输后的文件路径
Transfer(ctx context.Context, src *clitypes.UserSpaceDetail, srcPath string, opt S2SOption) (string, error)
// 完成传输
Complete()
// 取消传输。如果已经调用了Complete，则这个方法应该无效果
Abort()
// 从远端获取文件并保存到本地路径
Transfer(ctx context.Context, src *clitypes.UserSpaceDetail, srcPath string, dstPath string) (FileInfo, error)
Close()
}

7
common/pkgs/storage/types/s3_client.go
View File

@ -17,11 +17,8 @@ type Multiparter interface {
type MultipartTask interface {
InitState() MultipartInitState
// 所有分片上传完成后，合并分片
JoinParts(ctx context.Context, parts []UploadedPartInfo) (BypassedFileInfo, error)
// 合成之后的文件已被使用
Complete()
// 取消上传。如果在调用Complete之前调用，则应该删除合并后的文件。如果已经调用Complete，则应该不做任何事情。
Abort()
JoinParts(ctx context.Context, parts []UploadedPartInfo) (FileInfo, error)
Close()
}
// TODO 可以考虑重构成一个接口，支持不同的类型的分片有不同内容的实现

73
common/pkgs/storage/types/shard_store.go
View File

@ -1,9 +1,6 @@
package types
import (
"fmt"
"io"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
)
@ -11,7 +8,9 @@ type Status interface {
String() string
}
type OKStatus struct{}
type OKStatus struct {
StoreEvent
}
func (s *OKStatus) String() string {
return "OK"
@ -20,15 +19,14 @@ func (s *OKStatus) String() string {
var StatusOK = &OKStatus{}
type StoreEvent interface {
IsStoreEvent()
}
type ShardStore interface {
Start(ch *StorageEventChan)
Stop()
// 写入一个新文件，写入后获得FileHash
Create(stream io.Reader) (FileInfo, error)
// 使用F函数创建Option对象
Open(opt OpenOption) (io.ReadCloser, error)
// 将存储系统中已有的文件作为分片纳入管理范围
Store(path string, hash clitypes.FileHash, size int64) (FileInfo, error)
// 获得指定文件信息
Info(fileHash clitypes.FileHash) (FileInfo, error)
// 获取所有文件信息，尽量保证操作是原子的
@ -39,14 +37,6 @@ type ShardStore interface {
Stats() Stats
}
type FileInfo struct {
// 文件的SHA256哈希值，全大写的16进制字符串格式
Hash clitypes.FileHash
Size int64
// 文件描述信息，比如文件名，用于调试
Description string
}
type Stats struct {
// 存储服务状态，如果状态正常，此值应该是StatusOK
Status Status
@ -59,54 +49,3 @@ type Stats struct {
// 描述信息，用于调试
Description string
}
type OpenOption struct {
FileHash clitypes.FileHash
Offset int64
Length int64
}
func NewOpen(fileHash clitypes.FileHash) OpenOption {
return OpenOption{
FileHash: fileHash,
Offset: 0,
Length: -1,
}
}
func (o *OpenOption) WithLength(len int64) OpenOption {
o.Length = len
return *o
}
// [start, end)，不包含end
func (o *OpenOption) WithRange(start int64, end int64) OpenOption {
o.Offset = start
o.Length = end - start
return *o
}
func (o *OpenOption) WithNullableLength(offset int64, length *int64) {
o.Offset = offset
if length != nil {
o.Length = *length
}
}
func (o *OpenOption) String() string {
rangeStart := ""
if o.Offset > 0 {
rangeStart = fmt.Sprintf("%d", o.Offset)
}
rangeEnd := ""
if o.Length >= 0 {
rangeEnd = fmt.Sprintf("%d", o.Offset+o.Length)
}
if rangeStart == "" && rangeEnd == "" {
return string(o.FileHash)
}
return fmt.Sprintf("%s[%s:%s]", string(o.FileHash), rangeStart, rangeEnd)
}

14
common/pkgs/storage/types/types.go
View File

@ -4,6 +4,7 @@ import (
"errors"
"gitlink.org.cn/cloudream/common/pkgs/async"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
)
var ErrStorageNotFound = errors.New("storage not found")
@ -35,11 +36,20 @@ type FeatureDesc struct {
// 是否能旁路上传分片
HasBypassShardWrite bool
// 是否能旁路上传公共存储
HasBypassPublicWrite bool
HasBypassBaseWrite bool
// 是否能旁路读取分片
HasBypassShardRead bool
// 公共存储是否支持旁路读取
HasBypassPublicRead bool
HasBypassBaseRead bool
// 是否能通过HTTP读取
HasBypassHTTPRead bool
}
type FileInfo struct {
// 分片在存储系统中的路径，可以通过BaseStore读取的
Path string
// 文件大小
Size int64
// 分片的哈希值，不一定有值，根据来源不同，可能为空
Hash clitypes.FileHash
}