JointCloud
/
JCS-pub
24
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

types类型的包名前增加唯一的缩写

This commit is contained in:
Sydonian 2025-07-21 10:53:02 +08:00
parent 4c69f2ca5b
commit 2638f72fba
44 changed files with 260 additions and 260 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
common/pkgs/ioswitch2/fromto.go
View File

@ -3,7 +3,7 @@ package ioswitch2
import (
"gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -211,7 +211,7 @@ func (t *ToShardStore) GetRange() math2.Range {
type ToBaseStore struct {
UserSpace jcstypes.UserSpaceDetail
ObjectPath jcstypes.JPath
Option types.WriteOption
Option stgtypes.WriteOption
}
func NewToBaseStore(space jcstypes.UserSpaceDetail, objectPath jcstypes.JPath) *ToBaseStore {

20
common/pkgs/ioswitch2/ops2/base_store.go
View File

@ -12,7 +12,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -45,7 +45,7 @@ type BaseRead struct {
Output exec.VarID
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.OpenOption
Option stgtypes.OpenOption
}
func (o *BaseRead) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
@ -104,7 +104,7 @@ type BaseReadDyn struct {
UserSpace jcstypes.UserSpaceDetail
Output exec.VarID
FileInfo exec.VarID
Option types.OpenOption
Option stgtypes.OpenOption
}
func (o *BaseReadDyn) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
@ -170,7 +170,7 @@ type BaseWrite struct {
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
FileInfo exec.VarID
Option types.WriteOption
Option stgtypes.WriteOption
}
func (o *BaseWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
@ -215,10 +215,10 @@ type BaseReadNode struct {
From ioswitch2.From
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.OpenOption
Option stgtypes.OpenOption
}
func (b *GraphNodeBuilder) NewBaseRead(from ioswitch2.From, userSpace jcstypes.UserSpaceDetail, path jcstypes.JPath, opt types.OpenOption) *BaseReadNode {
func (b *GraphNodeBuilder) NewBaseRead(from ioswitch2.From, userSpace jcstypes.UserSpaceDetail, path jcstypes.JPath, opt stgtypes.OpenOption) *BaseReadNode {
node := &BaseReadNode{
From: from,
UserSpace: userSpace,
@ -255,10 +255,10 @@ type BaseReadDynNode struct {
dag.NodeBase
From ioswitch2.From
UserSpace jcstypes.UserSpaceDetail
Option types.OpenOption
Option stgtypes.OpenOption
}
func (b *GraphNodeBuilder) NewBaseReadDyn(from ioswitch2.From, userSpace jcstypes.UserSpaceDetail, opt types.OpenOption) *BaseReadDynNode {
func (b *GraphNodeBuilder) NewBaseReadDyn(from ioswitch2.From, userSpace jcstypes.UserSpaceDetail, opt stgtypes.OpenOption) *BaseReadDynNode {
node := &BaseReadDynNode{
From: from,
UserSpace: userSpace,
@ -303,10 +303,10 @@ type BaseWriteNode struct {
To ioswitch2.To
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.WriteOption
Option stgtypes.WriteOption
}
func (b *GraphNodeBuilder) NewBaseWrite(to ioswitch2.To, userSpace jcstypes.UserSpaceDetail, path jcstypes.JPath, opt types.WriteOption) *BaseWriteNode {
func (b *GraphNodeBuilder) NewBaseWrite(to ioswitch2.To, userSpace jcstypes.UserSpaceDetail, path jcstypes.JPath, opt stgtypes.WriteOption) *BaseWriteNode {
node := &BaseWriteNode{
To: to,
UserSpace: userSpace,

6
common/pkgs/ioswitch2/ops2/bypass.go
View File

@ -6,7 +6,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -23,7 +23,7 @@ type GetShardHTTPRequest struct {
}
type HTTPRequestValue struct {
types.HTTPRequest
stgtypes.HTTPRequest
}
func (v *HTTPRequestValue) Clone() exec.VarValue {
@ -43,7 +43,7 @@ func (o *GetShardHTTPRequest) Execute(ctx *exec.ExecContext, e *exec.Executor) e
return err
}
br, ok := shardStore.(types.HTTPShardRead)
br, ok := shardStore.(stgtypes.HTTPShardRead)
if !ok {
return fmt.Errorf("shard store %v not support bypass read", o.UserSpace)
}

4
common/pkgs/ioswitch2/ops2/ec.go
View File

@ -12,7 +12,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/utils"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -170,7 +170,7 @@ func (o *CallECMultiplier) Execute(ctx *exec.ExecContext, e *exec.Executor) erro
return err
}
reqs := make([]types.HTTPRequest, 0, len(inputs))
reqs := make([]stgtypes.HTTPRequest, 0, len(inputs))
for _, input := range inputs {
reqs = append(reqs, input.HTTPRequest)
}

8
common/pkgs/ioswitch2/ops2/multipart.go
View File

@ -8,7 +8,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -20,7 +20,7 @@ func init() {
}
type MultipartUploadArgsValue struct {
InitState types.MultipartInitState
InitState stgtypes.MultipartInitState
}
func (v *MultipartUploadArgsValue) Clone() exec.VarValue {
@ -30,7 +30,7 @@ func (v *MultipartUploadArgsValue) Clone() exec.VarValue {
}
type UploadedPartInfoValue struct {
types.UploadedPartInfo
stgtypes.UploadedPartInfo
}
func (v *UploadedPartInfoValue) Clone() exec.VarValue {
@ -75,7 +75,7 @@ func (o *MultipartInitiator) Execute(ctx *exec.ExecContext, e *exec.Executor) er
return fmt.Errorf("getting uploaded parts: %v", err)
}
partInfos := make([]types.UploadedPartInfo, len(partInfoValues))
partInfos := make([]stgtypes.UploadedPartInfo, len(partInfoValues))
for i, v := range partInfoValues {
partInfos[i] = v.UploadedPartInfo
}

4
common/pkgs/ioswitch2/ops2/ops.go
View File

@ -5,7 +5,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/plan/ops"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
)
type GraphNodeBuilder struct {
@ -29,7 +29,7 @@ type ToNode interface {
}
type FileInfoValue struct {
types.FileInfo
stgtypes.FileInfo
}
func (v *FileInfoValue) Clone() exec.VarValue {

10
common/pkgs/ioswitch2/parser/gen/generator.go
View File

@ -11,7 +11,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser/state"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -261,7 +261,7 @@ func buildFromNode(ctx *state.GenerateState, f ioswitch2.From) (ops2.FromNode, e
getShard := ctx.DAG.NewGetShardInfo(f.UserSpace, f.FileHash)
getShard.Env().ToEnvDriver(true)
read := ctx.DAG.NewBaseReadDyn(f, f.UserSpace, types.DefaultOpen())
read := ctx.DAG.NewBaseReadDyn(f, f.UserSpace, stgtypes.DefaultOpen())
getShard.FileInfoVar().ToSlot(read.FileInfoSlot())
@ -332,7 +332,7 @@ func buildFromNode(ctx *state.GenerateState, f ioswitch2.From) (ops2.FromNode, e
case *ioswitch2.FromBaseStore:
// TODO 可以考虑支持设置读取范围
n := ctx.DAG.NewBaseRead(f, f.UserSpace, f.Path, types.DefaultOpen())
n := ctx.DAG.NewBaseRead(f, f.UserSpace, f.Path, stgtypes.DefaultOpen())
if err := setEnvBySpace(n, &f.UserSpace); err != nil {
return nil, fmt.Errorf("set node env by user space: %w", err)
}
@ -347,9 +347,9 @@ func buildFromNode(ctx *state.GenerateState, f ioswitch2.From) (ops2.FromNode, e
func buildToNode(ctx *state.GenerateState, t ioswitch2.To) (ops2.ToNode, error) {
switch t := t.(type) {
case *ioswitch2.ToShardStore:
tempFileName := types.MakeTempDirPath(&t.UserSpace, os2.GenerateRandomFileName(20))
tempFileName := stgtypes.MakeTempDirPath(&t.UserSpace, os2.GenerateRandomFileName(20))
write := ctx.DAG.NewBaseWrite(t, t.UserSpace, tempFileName, types.WriteOption{})
write := ctx.DAG.NewBaseWrite(t, t.UserSpace, tempFileName, stgtypes.WriteOption{})
if err := setEnvBySpace(write, &t.UserSpace); err != nil {
return nil, fmt.Errorf("set node env by user space: %w", err)
}

4
common/pkgs/ioswitch2/parser/opt/ec.go
View File

@ -7,7 +7,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser/state"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
)
// 减少未使用的Multiply指令的输出流。如果减少到0，则删除该指令
@ -102,7 +102,7 @@ func UseECMultiplier(ctx *state.GenerateState) {
if err != nil {
return true
}
_, ok = store.(types.HTTPShardRead)
_, ok = store.(stgtypes.HTTPShardRead)
if !ok {
return true
}

6
common/pkgs/ioswitch2/plans/complete_multipart.go
View File

@ -7,7 +7,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/plan"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -27,7 +27,7 @@ func CompleteMultipart(blocks []jcstypes.ObjectBlock, blockSpaces []jcstypes.Use
gs := da.NewGetShardInfo(blockSpaces[i], blk.FileHash)
gs.Env().ToEnvDriver(true)
br := da.NewBaseReadDyn(nil, blockSpaces[i], types.DefaultOpen())
br := da.NewBaseReadDyn(nil, blockSpaces[i], stgtypes.DefaultOpen())
if err := setEnvBySpace(br, &blockSpaces[i]); err != nil {
return fmt.Errorf("set node env by user space: %w", err)
}
@ -38,7 +38,7 @@ func CompleteMultipart(blocks []jcstypes.ObjectBlock, blockSpaces []jcstypes.Use
}
// TODO 应该采取更合理的方式同时支持Parser和直接生成DAG
br := da.NewBaseWrite(nil, targetSpace, types.MakeTempDirPath(&targetSpace, os2.GenerateRandomFileName(20)), types.WriteOption{})
br := da.NewBaseWrite(nil, targetSpace, stgtypes.MakeTempDirPath(&targetSpace, os2.GenerateRandomFileName(20)), stgtypes.WriteOption{})
if err := setEnvBySpace(br, &targetSpace); err != nil {
return fmt.Errorf("set node env by user space: %w", err)
}

4
common/pkgs/ioswitchlrc/fromto.go
View File

@ -3,7 +3,7 @@ package ioswitchlrc
import (
"gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -92,7 +92,7 @@ type ToNode struct {
DataIndex int
Range math2.Range
FileHashStoreKey string
Option types.WriteOption
Option stgtypes.WriteOption
}
func NewToStorage(space jcstypes.UserSpaceDetail, dataIndex int, fileHashStoreKey string) *ToNode {

20
common/pkgs/ioswitchlrc/ops2/base_store.go
View File

@ -11,7 +11,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -25,7 +25,7 @@ type BaseRead struct {
Output exec.VarID
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.OpenOption
Option stgtypes.OpenOption
}
func (o *BaseRead) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
@ -70,7 +70,7 @@ type BaseReadDyn struct {
UserSpace jcstypes.UserSpaceDetail
Output exec.VarID
Path exec.VarID
Option types.OpenOption
Option stgtypes.OpenOption
}
func (o *BaseReadDyn) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
@ -121,7 +121,7 @@ type BaseWrite struct {
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
WriteResult exec.VarID
Option types.WriteOption
Option stgtypes.WriteOption
}
func (o *BaseWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
@ -166,10 +166,10 @@ type BaseReadNode struct {
From ioswitchlrc.From
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.OpenOption
Option stgtypes.OpenOption
}
func (b *GraphNodeBuilder) NewBaseRead(from ioswitchlrc.From, userSpace jcstypes.UserSpaceDetail, path jcstypes.JPath, opt types.OpenOption) *BaseReadNode {
func (b *GraphNodeBuilder) NewBaseRead(from ioswitchlrc.From, userSpace jcstypes.UserSpaceDetail, path jcstypes.JPath, opt stgtypes.OpenOption) *BaseReadNode {
node := &BaseReadNode{
From: from,
UserSpace: userSpace,
@ -206,10 +206,10 @@ type BaseReadDynNode struct {
dag.NodeBase
From ioswitchlrc.From
UserSpace jcstypes.UserSpaceDetail
Option types.OpenOption
Option stgtypes.OpenOption
}
func (b *GraphNodeBuilder) NewBaseReadDyn(from ioswitchlrc.From, userSpace jcstypes.UserSpaceDetail, opt types.OpenOption) *BaseReadDynNode {
func (b *GraphNodeBuilder) NewBaseReadDyn(from ioswitchlrc.From, userSpace jcstypes.UserSpaceDetail, opt stgtypes.OpenOption) *BaseReadDynNode {
node := &BaseReadDynNode{
From: from,
UserSpace: userSpace,
@ -254,10 +254,10 @@ type BaseWriteNode struct {
To ioswitchlrc.To
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.WriteOption
Option stgtypes.WriteOption
}
func (b *GraphNodeBuilder) NewBaseWrite(to ioswitchlrc.To, userSpace jcstypes.UserSpaceDetail, path jcstypes.JPath, opt types.WriteOption) *BaseWriteNode {
func (b *GraphNodeBuilder) NewBaseWrite(to ioswitchlrc.To, userSpace jcstypes.UserSpaceDetail, path jcstypes.JPath, opt stgtypes.WriteOption) *BaseWriteNode {
node := &BaseWriteNode{
To: to,
UserSpace: userSpace,

4
common/pkgs/ioswitchlrc/ops2/ops.go
View File

@ -4,7 +4,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/plan/ops"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
)
type GraphNodeBuilder struct {
@ -26,7 +26,7 @@ type ToNode interface {
}
type FileInfoValue struct {
types.FileInfo
stgtypes.FileInfo
}
func (v *FileInfoValue) Clone() exec.VarValue {

6
common/pkgs/ioswitchlrc/parser/passes.go
View File

@ -9,7 +9,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
)
// 计算输入流的打开范围。会把流的范围按条带大小取整
@ -66,7 +66,7 @@ func buildFromNode(ctx *GenerateContext, f ioswitchlrc.From) (ops2.FromNode, err
getShard := ctx.DAG.NewGetShardInfo(f.UserSpace, f.FileHash)
getShard.Env().ToEnvDriver(true)
read := ctx.DAG.NewBaseReadDyn(f, f.UserSpace, types.DefaultOpen())
read := ctx.DAG.NewBaseReadDyn(f, f.UserSpace, stgtypes.DefaultOpen())
if f.DataIndex == -1 {
read.Option.WithNullableLength(repRange.Offset, repRange.Length)
@ -102,7 +102,7 @@ func buildFromNode(ctx *GenerateContext, f ioswitchlrc.From) (ops2.FromNode, err
func buildToNode(ctx *GenerateContext, t ioswitchlrc.To) (ops2.ToNode, error) {
switch t := t.(type) {
case *ioswitchlrc.ToNode:
tempFileName := types.MakeTempDirPath(&t.UserSpace, os2.GenerateRandomFileName(20))
tempFileName := stgtypes.MakeTempDirPath(&t.UserSpace, os2.GenerateRandomFileName(20))
write := ctx.DAG.NewBaseWrite(t, t.UserSpace, tempFileName, t.Option)
if err := setEnvBySpace(write, &t.UserSpace); err != nil {

18
common/pkgs/storage/efile/ec_multiplier.go
View File

@ -9,7 +9,7 @@ import (
"gitlink.org.cn/cloudream/common/utils/http2"
"gitlink.org.cn/cloudream/common/utils/os2"
"gitlink.org.cn/cloudream/common/utils/serder"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -22,12 +22,12 @@ type ECMultiplier struct {
// 进行EC运算，coef * inputs。coef为编码矩阵，inputs为待编码数据，chunkSize为分块大小。
// 输出为每一个块文件的路径，数组长度 = len(coef)
func (m *ECMultiplier) Multiply(coef [][]byte, inputs []types.HTTPRequest, chunkSize int) ([]types.FileInfo, error) {
func (m *ECMultiplier) Multiply(coef [][]byte, inputs []stgtypes.HTTPRequest, chunkSize int) ([]stgtypes.FileInfo, error) {
type Request struct {
Inputs []types.HTTPRequest `json:"inputs"`
Outputs []string `json:"outputs"`
Coefs [][]int `json:"coefs"` // 用int防止被base64编码
ChunkSize int `json:"chunkSize"`
Inputs []stgtypes.HTTPRequest `json:"inputs"`
Outputs []string `json:"outputs"`
Coefs [][]int `json:"coefs"` // 用int防止被base64编码
ChunkSize int `json:"chunkSize"`
}
type Response struct {
Code string `json:"code"`
@ -47,7 +47,7 @@ func (m *ECMultiplier) Multiply(coef [][]byte, inputs []types.HTTPRequest, chunk
}
fileName := os2.GenerateRandomFileName(10)
tempDir := path.Join(m.blder.detail.UserSpace.WorkingDir.String(), types.TempWorkingDir)
tempDir := path.Join(m.blder.detail.UserSpace.WorkingDir.String(), stgtypes.TempWorkingDir)
m.outputs = make([]string, len(coef))
for i := range m.outputs {
m.outputs[i] = path.Join(tempDir, fmt.Sprintf("%s_%d", fileName, i))
@ -93,9 +93,9 @@ func (m *ECMultiplier) Multiply(coef [][]byte, inputs []types.HTTPRequest, chunk
return nil, fmt.Errorf("data length not match outputs length")
}
ret := make([]types.FileInfo, len(r.Data))
ret := make([]stgtypes.FileInfo, len(r.Data))
for i, data := range r.Data {
ret[i] = types.FileInfo{
ret[i] = stgtypes.FileInfo{
// TODO 要确认一下output的格式
Path: jcstypes.PathFromJcsPathString(m.outputs[i]),
Size: data.Size,

10
common/pkgs/storage/efile/efile.go
View File

@ -9,12 +9,12 @@ import (
"gitlink.org.cn/cloudream/common/utils/http2"
"gitlink.org.cn/cloudream/common/utils/serder"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func init() {
reg.RegisterBuilder[*jcstypes.EFileType](func(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
reg.RegisterBuilder[*jcstypes.EFileType](func(detail *jcstypes.UserSpaceDetail) stgtypes.StorageBuilder {
return &builder{
detail: detail,
}
@ -22,7 +22,7 @@ func init() {
}
type builder struct {
types.EmptyBuilder
stgtypes.EmptyBuilder
detail *jcstypes.UserSpaceDetail
token string
tokenLock sync.Mutex
@ -89,8 +89,8 @@ func (b *builder) getToken() (string, error) {
return "", fmt.Errorf("clusterID %s not found", stgType.ClusterID)
}
func (b *builder) CreateECMultiplier(typeOnly bool) (types.ECMultiplier, error) {
feat := types.FindFeature[*jcstypes.ECMultiplierFeature](b.detail)
func (b *builder) CreateECMultiplier(typeOnly bool) (stgtypes.ECMultiplier, error) {
feat := stgtypes.FindFeature[*jcstypes.ECMultiplierFeature](b.detail)
if feat == nil {
return nil, fmt.Errorf("feature ECMultiplier not found")
}

6
common/pkgs/storage/factory/factory.go
View File

@ -5,18 +5,18 @@ import (
_ "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
// 此函数永远不会返回nil。如果找不到对应的Builder，则会返回EmptyBuilder，
// 此Builder的所有函数都会返回否定值或者封装后的ErrUnsupported错误（需要使用errors.Is检查）
func GetBuilder(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
func GetBuilder(detail *jcstypes.UserSpaceDetail) stgtypes.StorageBuilder {
typ := reflect.TypeOf(detail.UserSpace.Storage)
ctor, ok := reg.StorageBuilders[typ]
if !ok {
return &types.EmptyBuilder{}
return &stgtypes.EmptyBuilder{}
}
return ctor(detail)

8
common/pkgs/storage/factory/reg/reg.go
View File

@ -4,11 +4,11 @@ import (
"reflect"
"gitlink.org.cn/cloudream/common/utils/reflect2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type BuilderCtor func(detail *jcstypes.UserSpaceDetail) types.StorageBuilder
type BuilderCtor func(detail *jcstypes.UserSpaceDetail) stgtypes.StorageBuilder
var StorageBuilders = make(map[reflect.Type]BuilderCtor)
@ -20,12 +20,12 @@ func RegisterBuilder[T jcstypes.StorageType](ctor BuilderCtor) {
// 注：此函数只给storage包内部使用，外部包请使用外层的factory.GetBuilder
// 此函数永远不会返回nil。如果找不到对应的Builder，则会返回EmptyBuilder，
// 此Builder的所有函数都会返回否定值或者封装后的ErrUnsupported错误（需要使用errors.Is检查）
func GetBuilderInternal(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
func GetBuilderInternal(detail *jcstypes.UserSpaceDetail) stgtypes.StorageBuilder {
typ := reflect.TypeOf(detail.UserSpace.Storage)
ctor, ok := StorageBuilders[typ]
if !ok {
return &types.EmptyBuilder{}
return &stgtypes.EmptyBuilder{}
}
return ctor(detail)

18
common/pkgs/storage/local/base_store.go
View File

@ -8,7 +8,7 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/io2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -24,19 +24,19 @@ func NewBaseStore(root string, detail *jcstypes.UserSpaceDetail) (*BaseStore, er
}, nil
}
func (s *BaseStore) Write(pat jcstypes.JPath, stream io.Reader, opt types.WriteOption) (types.FileInfo, error) {
func (s *BaseStore) Write(pat jcstypes.JPath, stream io.Reader, opt stgtypes.WriteOption) (stgtypes.FileInfo, error) {
log := s.getLogger()
absObjPath := filepath.Join(s.root, pat.String())
err := os.MkdirAll(filepath.Dir(absObjPath), 0755)
if err != nil {
return types.FileInfo{}, err
return stgtypes.FileInfo{}, err
}
f, err := os.Create(absObjPath)
if err != nil {
return types.FileInfo{}, err
return stgtypes.FileInfo{}, err
}
defer f.Close()
@ -45,7 +45,7 @@ func (s *BaseStore) Write(pat jcstypes.JPath, stream io.Reader, opt types.WriteO
_, err = io.Copy(f, hasher)
if err != nil {
return types.FileInfo{}, err
return stgtypes.FileInfo{}, err
}
if !opt.ModTime.IsZero() {
@ -55,14 +55,14 @@ func (s *BaseStore) Write(pat jcstypes.JPath, stream io.Reader, opt types.WriteO
}
}
return types.FileInfo{
return stgtypes.FileInfo{
Path: pat,
Size: counter.Count(),
Hash: jcstypes.NewFullHash(hasher.Sum()),
}, nil
}
func (s *BaseStore) Read(objPath jcstypes.JPath, opt types.OpenOption) (io.ReadCloser, error) {
func (s *BaseStore) Read(objPath jcstypes.JPath, opt stgtypes.OpenOption) (io.ReadCloser, error) {
absObjPath := filepath.Join(s.root, objPath.JoinOSPath())
file, err := os.Open(absObjPath)
if err != nil {
@ -96,7 +96,7 @@ func (s *BaseStore) Mkdir(path jcstypes.JPath) error {
return nil
}
func (s *BaseStore) ReadDir(pat jcstypes.JPath) types.DirReader {
func (s *BaseStore) ReadDir(pat jcstypes.JPath) stgtypes.DirReader {
return &DirReader{
absRootPath: filepath.Join(s.root, pat.JoinOSPath()),
rootJPath: pat.Clone(),
@ -106,7 +106,7 @@ func (s *BaseStore) ReadDir(pat jcstypes.JPath) types.DirReader {
func (s *BaseStore) CleanTemps() {
log := s.getLogger()
tempDir := filepath.Join(s.root, s.detail.UserSpace.WorkingDir.JoinOSPath(), types.TempWorkingDir)
tempDir := filepath.Join(s.root, s.detail.UserSpace.WorkingDir.JoinOSPath(), stgtypes.TempWorkingDir)
entries, err := os.ReadDir(tempDir)
if err != nil {
log.Warnf("read temp dir: %v", err)

20
common/pkgs/storage/local/dir_reader.go
View File

@ -5,7 +5,7 @@ import (
"os"
"path/filepath"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -18,16 +18,16 @@ type DirReader struct {
curEntries []dirEntry
}
func (r *DirReader) Next() (types.DirEntry, error) {
func (r *DirReader) Next() (stgtypes.DirEntry, error) {
if !r.init {
info, err := os.Stat(r.absRootPath)
if err != nil {
return types.DirEntry{}, err
return stgtypes.DirEntry{}, err
}
if !info.IsDir() {
r.init = true
return types.DirEntry{
return stgtypes.DirEntry{
Path: r.rootJPath,
Size: info.Size(),
ModTime: info.ModTime(),
@ -37,7 +37,7 @@ func (r *DirReader) Next() (types.DirEntry, error) {
es, err := os.ReadDir(r.absRootPath)
if err != nil {
return types.DirEntry{}, err
return stgtypes.DirEntry{}, err
}
for _, e := range es {
@ -50,7 +50,7 @@ func (r *DirReader) Next() (types.DirEntry, error) {
r.init = true
}
if len(r.curEntries) == 0 {
return types.DirEntry{}, io.EOF
return stgtypes.DirEntry{}, io.EOF
}
entry := r.curEntries[0]
@ -59,7 +59,7 @@ func (r *DirReader) Next() (types.DirEntry, error) {
if entry.entry.IsDir() {
es, err := os.ReadDir(filepath.Join(r.absRootPath, entry.dir.JoinOSPath(), entry.entry.Name()))
if err != nil {
return types.DirEntry{}, err
return stgtypes.DirEntry{}, err
}
// 多个entry对象共享同一个JPath对象，但因为不会修改JPath，所以没问题
@ -75,14 +75,14 @@ func (r *DirReader) Next() (types.DirEntry, error) {
info, err := entry.entry.Info()
if err != nil {
return types.DirEntry{}, err
return stgtypes.DirEntry{}, err
}
p := r.rootJPath.ConcatNew(entry.dir)
p.Push(entry.entry.Name())
if entry.entry.IsDir() {
return types.DirEntry{
return stgtypes.DirEntry{
Path: p,
Size: 0,
ModTime: info.ModTime(),
@ -90,7 +90,7 @@ func (r *DirReader) Next() (types.DirEntry, error) {
}, nil
}
return types.DirEntry{
return stgtypes.DirEntry{
Path: p,
Size: info.Size(),
ModTime: info.ModTime(),

22
common/pkgs/storage/local/local.go
View File

@ -4,12 +4,12 @@ import (
"fmt"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
func init() {
reg.RegisterBuilder[*jcstypes.LocalType](func(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
reg.RegisterBuilder[*jcstypes.LocalType](func(detail *jcstypes.UserSpaceDetail) stgtypes.StorageBuilder {
return &builder{
detail: detail,
}
@ -17,15 +17,15 @@ func init() {
}
type builder struct {
types.EmptyBuilder
stgtypes.EmptyBuilder
detail *jcstypes.UserSpaceDetail
}
func (b *builder) FeatureDesc() types.FeatureDesc {
return types.FeatureDesc{}
func (b *builder) FeatureDesc() stgtypes.FeatureDesc {
return stgtypes.FeatureDesc{}
}
func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
func (b *builder) CreateShardStore(typeOnly bool) (stgtypes.ShardStore, error) {
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.LocalCred)
if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for local storage", b.detail.UserSpace.Credential)
@ -38,7 +38,7 @@ func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
return NewShardStore(cred.RootDir, b.detail)
}
func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) {
func (b *builder) CreateBaseStore(typeOnly bool) (stgtypes.BaseStore, error) {
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.LocalCred)
if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for local storage", b.detail.UserSpace.Credential)
@ -51,8 +51,8 @@ func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) {
return NewBaseStore(cred.RootDir, b.detail)
}
func (b *builder) CreateMultiparter(typeOnly bool) (types.Multiparter, error) {
feat := types.FindFeature[*jcstypes.MultipartUploadFeature](b.detail)
func (b *builder) CreateMultiparter(typeOnly bool) (stgtypes.Multiparter, error) {
feat := stgtypes.FindFeature[*jcstypes.MultipartUploadFeature](b.detail)
if feat == nil {
return nil, fmt.Errorf("feature %T not found", jcstypes.MultipartUploadFeature{})
}
@ -66,8 +66,8 @@ func (b *builder) CreateMultiparter(typeOnly bool) (types.Multiparter, error) {
}, nil
}
func (b *builder) CreateS2STransfer(typeOnly bool) (types.S2STransfer, error) {
feat := types.FindFeature[*jcstypes.S2STransferFeature](b.detail)
func (b *builder) CreateS2STransfer(typeOnly bool) (stgtypes.S2STransfer, error) {
feat := stgtypes.FindFeature[*jcstypes.S2STransferFeature](b.detail)
if feat == nil {
return nil, fmt.Errorf("feature %T not found", jcstypes.S2STransferFeature{})
}

32
common/pkgs/storage/local/multipart_upload.go
View File

@ -12,7 +12,7 @@ import (
"gitlink.org.cn/cloudream/common/utils/io2"
"gitlink.org.cn/cloudream/common/utils/os2"
"gitlink.org.cn/cloudream/common/utils/sort2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -30,8 +30,8 @@ func (*Multiparter) MaxPartSize() int64 {
return 5 * 1024 * 1024 * 1024 // 5GB
}
func (m *Multiparter) Initiate(ctx context.Context) (types.MultipartTask, error) {
tempDir := filepath.Join(m.localStg.RootDir, m.detail.UserSpace.WorkingDir.JoinOSPath(), types.TempWorkingDir)
func (m *Multiparter) Initiate(ctx context.Context) (stgtypes.MultipartTask, error) {
tempDir := filepath.Join(m.localStg.RootDir, m.detail.UserSpace.WorkingDir.JoinOSPath(), stgtypes.TempWorkingDir)
absTempDir, err := filepath.Abs(tempDir)
if err != nil {
return nil, fmt.Errorf("get abs temp dir %v: %v", tempDir, err)
@ -51,25 +51,25 @@ func (m *Multiparter) Initiate(ctx context.Context) (types.MultipartTask, error)
absTempDir: absTempDir,
tempFileName: tempFileName,
tempPartsDir: tempPartsDir,
joinedFileJPath: m.detail.UserSpace.WorkingDir.ConcatCompsNew(types.TempWorkingDir, tempFileName+".joined"),
joinedFileJPath: m.detail.UserSpace.WorkingDir.ConcatCompsNew(stgtypes.TempWorkingDir, tempFileName+".joined"),
absJoinedFilePath: absJoinedFilePath,
uploadID: tempPartsDir,
}, nil
}
func (m *Multiparter) UploadPart(ctx context.Context, init types.MultipartInitState, partSize int64, partNumber int, stream io.Reader) (types.UploadedPartInfo, error) {
func (m *Multiparter) UploadPart(ctx context.Context, init stgtypes.MultipartInitState, partSize int64, partNumber int, stream io.Reader) (stgtypes.UploadedPartInfo, error) {
partFilePath := filepath.Join(init.UploadID, fmt.Sprintf("%v", partNumber))
partFile, err := os.Create(partFilePath)
if err != nil {
return types.UploadedPartInfo{}, err
return stgtypes.UploadedPartInfo{}, err
}
defer partFile.Close()
_, err = io.Copy(partFile, stream)
if err != nil {
return types.UploadedPartInfo{}, err
return stgtypes.UploadedPartInfo{}, err
}
return types.UploadedPartInfo{
return stgtypes.UploadedPartInfo{
ETag: partFilePath,
PartNumber: partNumber,
}, nil
@ -84,20 +84,20 @@ type MultipartTask struct {
uploadID string
}
func (i *MultipartTask) InitState() types.MultipartInitState {
return types.MultipartInitState{
func (i *MultipartTask) InitState() stgtypes.MultipartInitState {
return stgtypes.MultipartInitState{
UploadID: i.uploadID,
}
}
func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPartInfo) (types.FileInfo, error) {
parts = sort2.Sort(parts, func(l, r types.UploadedPartInfo) int {
func (i *MultipartTask) JoinParts(ctx context.Context, parts []stgtypes.UploadedPartInfo) (stgtypes.FileInfo, error) {
parts = sort2.Sort(parts, func(l, r stgtypes.UploadedPartInfo) int {
return l.PartNumber - r.PartNumber
})
joined, err := os.Create(i.absJoinedFilePath)
if err != nil {
return types.FileInfo{}, err
return stgtypes.FileInfo{}, err
}
defer joined.Close()
@ -107,21 +107,21 @@ func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPar
for _, part := range parts {
partSize, err := i.writePart(part, joined, hasher)
if err != nil {
return types.FileInfo{}, err
return stgtypes.FileInfo{}, err
}
size += partSize
}
h := hasher.Sum(nil)
return types.FileInfo{
return stgtypes.FileInfo{
Path: i.joinedFileJPath,
Size: size,
Hash: jcstypes.NewFullHash(h),
}, nil
}
func (i *MultipartTask) writePart(partInfo types.UploadedPartInfo, joined *os.File, hasher hash.Hash) (int64, error) {
func (i *MultipartTask) writePart(partInfo stgtypes.UploadedPartInfo, joined *os.File, hasher hash.Hash) (int64, error) {
part, err := os.Open(partInfo.ETag)
if err != nil {
return 0, err

14
common/pkgs/storage/local/s2s.go
View File

@ -6,7 +6,7 @@ import (
"os"
"path/filepath"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -19,7 +19,7 @@ type S2STransfer struct {
// 只有同一个机器的存储之间才可以进行数据直传
func (*S2STransfer) CanTransfer(src, dst *jcstypes.UserSpaceDetail) bool {
if types.FindFeature[*jcstypes.S2STransferFeature](dst) == nil {
if stgtypes.FindFeature[*jcstypes.S2STransferFeature](dst) == nil {
return false
}
@ -36,27 +36,27 @@ func (*S2STransfer) CanTransfer(src, dst *jcstypes.UserSpaceDetail) bool {
}
// 执行数据直传
func (s *S2STransfer) Transfer(ctx context.Context, src *jcstypes.UserSpaceDetail, srcPath jcstypes.JPath, dstPath jcstypes.JPath) (types.FileInfo, error) {
func (s *S2STransfer) Transfer(ctx context.Context, src *jcstypes.UserSpaceDetail, srcPath jcstypes.JPath, dstPath jcstypes.JPath) (stgtypes.FileInfo, error) {
s.dstPath = dstPath
copy, err := os.OpenFile(filepath.Join(s.localStg.RootDir, s.dstPath.JoinOSPath()), os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return types.FileInfo{}, err
return stgtypes.FileInfo{}, err
}
defer copy.Close()
srcFile, err := os.Open(filepath.Join(s.localStg.RootDir, srcPath.JoinOSPath()))
if err != nil {
return types.FileInfo{}, err
return stgtypes.FileInfo{}, err
}
defer srcFile.Close()
n, err := io.Copy(copy, srcFile)
if err != nil {
return types.FileInfo{}, err
return stgtypes.FileInfo{}, err
}
return types.FileInfo{
return stgtypes.FileInfo{
Path: dstPath,
Size: n,
Hash: "",

36
common/pkgs/storage/local/shard_store.go
View File

@ -9,7 +9,7 @@ import (
"sync"
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -22,7 +22,7 @@ type ShardStore struct {
}
func NewShardStore(root string, detail *jcstypes.UserSpaceDetail) (*ShardStore, error) {
storeAbsRoot, err := filepath.Abs(filepath.Join(root, detail.UserSpace.WorkingDir.JoinOSPath(), types.ShardStoreWorkingDir))
storeAbsRoot, err := filepath.Abs(filepath.Join(root, detail.UserSpace.WorkingDir.JoinOSPath(), stgtypes.ShardStoreWorkingDir))
if err != nil {
return nil, fmt.Errorf("get abs root: %w", err)
}
@ -35,7 +35,7 @@ func NewShardStore(root string, detail *jcstypes.UserSpaceDetail) (*ShardStore,
}, nil
}
func (s *ShardStore) Start(ch *types.StorageEventChan) {
func (s *ShardStore) Start(ch *stgtypes.StorageEventChan) {
s.getLogger().Infof("component start, root: %v, max size: %v", s.storeAbsRoot, s.detail.UserSpace.ShardStore.MaxSize)
}
@ -43,7 +43,7 @@ func (s *ShardStore) Stop() {
s.getLogger().Infof("component stop")
}
func (s *ShardStore) Store(path jcstypes.JPath, hash jcstypes.FileHash, size int64) (types.FileInfo, error) {
func (s *ShardStore) Store(path jcstypes.JPath, hash jcstypes.FileHash, size int64) (stgtypes.FileInfo, error) {
fullTempPath := filepath.Join(s.stgRoot, path.JoinOSPath())
s.lock.Lock()
@ -57,7 +57,7 @@ func (s *ShardStore) Store(path jcstypes.JPath, hash jcstypes.FileHash, size int
err := os.MkdirAll(blockDir, 0755)
if err != nil {
log.Warnf("make block dir %v: %v", blockDir, err)
return types.FileInfo{}, fmt.Errorf("making block dir: %w", err)
return stgtypes.FileInfo{}, fmt.Errorf("making block dir: %w", err)
}
newPath := filepath.Join(blockDir, string(hash))
@ -66,43 +66,43 @@ func (s *ShardStore) Store(path jcstypes.JPath, hash jcstypes.FileHash, size int
err = os.Rename(fullTempPath, newPath)
if err != nil {
log.Warnf("rename %v to %v: %v", fullTempPath, newPath, err)
return types.FileInfo{}, fmt.Errorf("rename file: %w", err)
return stgtypes.FileInfo{}, fmt.Errorf("rename file: %w", err)
}
} else if err != nil {
log.Warnf("get file %v stat: %v", newPath, err)
return types.FileInfo{}, fmt.Errorf("get file stat: %w", err)
return stgtypes.FileInfo{}, fmt.Errorf("get file stat: %w", err)
}
return types.FileInfo{
return stgtypes.FileInfo{
Hash: hash,
Size: size,
Path: s.getJPathFromHash(hash),
}, nil
}
func (s *ShardStore) Info(hash jcstypes.FileHash) (types.FileInfo, error) {
func (s *ShardStore) Info(hash jcstypes.FileHash) (stgtypes.FileInfo, error) {
s.lock.Lock()
defer s.lock.Unlock()
filePath := s.getFilePathFromHash(hash)
info, err := os.Stat(filePath)
if err != nil {
return types.FileInfo{}, err
return stgtypes.FileInfo{}, err
}
return types.FileInfo{
return stgtypes.FileInfo{
Hash: hash,
Size: info.Size(),
Path: s.getJPathFromHash(hash),
}, nil
}
func (s *ShardStore) ListAll() ([]types.FileInfo, error) {
func (s *ShardStore) ListAll() ([]stgtypes.FileInfo, error) {
s.lock.Lock()
defer s.lock.Unlock()
var infos []types.FileInfo
var infos []stgtypes.FileInfo
err := filepath.WalkDir(s.storeAbsRoot, func(path string, d fs.DirEntry, err error) error {
if err != nil {
@ -123,7 +123,7 @@ func (s *ShardStore) ListAll() ([]types.FileInfo, error) {
return nil
}
infos = append(infos, types.FileInfo{
infos = append(infos, stgtypes.FileInfo{
Hash: fileHash,
Size: info.Size(),
Path: s.getJPathFromHash(fileHash),
@ -188,10 +188,10 @@ func (s *ShardStore) GC(avaiables []jcstypes.FileHash) error {
return nil
}
func (s *ShardStore) Stats() types.Stats {
func (s *ShardStore) Stats() stgtypes.Stats {
// TODO 统计本地存储的相关信息
return types.Stats{
Status: types.StatusOK,
return stgtypes.Stats{
Status: stgtypes.StatusOK,
}
}
@ -208,5 +208,5 @@ func (s *ShardStore) getFilePathFromHash(hash jcstypes.FileHash) string {
}
func (s *ShardStore) getJPathFromHash(hash jcstypes.FileHash) jcstypes.JPath {
return s.detail.UserSpace.WorkingDir.ConcatCompsNew(types.ShardStoreWorkingDir, hash.GetHashPrefix(2), string(hash))
return s.detail.UserSpace.WorkingDir.ConcatCompsNew(stgtypes.ShardStoreWorkingDir, hash.GetHashPrefix(2), string(hash))
}

22
common/pkgs/storage/obs/obs.go
View File

@ -8,7 +8,7 @@ import (
"github.com/aws/aws-sdk-go-v2/service/s3"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
s3stg "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/s3"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -17,21 +17,21 @@ func init() {
}
type builder struct {
types.EmptyBuilder
stgtypes.EmptyBuilder
detail *jcstypes.UserSpaceDetail
}
func newBuilder(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
func newBuilder(detail *jcstypes.UserSpaceDetail) stgtypes.StorageBuilder {
return &builder{
detail: detail,
}
}
func (b *builder) FeatureDesc() types.FeatureDesc {
return types.FeatureDesc{}
func (b *builder) FeatureDesc() stgtypes.FeatureDesc {
return stgtypes.FeatureDesc{}
}
func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
func (b *builder) CreateShardStore(typeOnly bool) (stgtypes.ShardStore, error) {
stgType := b.detail.UserSpace.Storage.(*jcstypes.OBSType)
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.OBSCred)
if !ok {
@ -50,7 +50,7 @@ func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
return NewShardStore(b.detail, stgType, cred, cli, bucket)
}
func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) {
func (b *builder) CreateBaseStore(typeOnly bool) (stgtypes.BaseStore, error) {
stgType := b.detail.UserSpace.Storage.(*jcstypes.OBSType)
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.OBSCred)
if !ok {
@ -89,9 +89,9 @@ func createClient(stgType *jcstypes.OBSType, cred *jcstypes.OBSCred) (*s3.Client
return cli, stgType.Bucket, nil
}
func (b *builder) CreateMultiparter(typeOnly bool) (types.Multiparter, error) {
func (b *builder) CreateMultiparter(typeOnly bool) (stgtypes.Multiparter, error) {
stgType := b.detail.UserSpace.Storage.(*jcstypes.OBSType)
feat := types.FindFeature[*jcstypes.MultipartUploadFeature](b.detail)
feat := stgtypes.FindFeature[*jcstypes.MultipartUploadFeature](b.detail)
if feat == nil {
return nil, fmt.Errorf("feature %T not found", jcstypes.MultipartUploadFeature{})
}
@ -118,9 +118,9 @@ func (b *builder) CreateMultiparter(typeOnly bool) (types.Multiparter, error) {
), nil
}
func (b *builder) CreateS2STransfer(typeOnly bool) (types.S2STransfer, error) {
func (b *builder) CreateS2STransfer(typeOnly bool) (stgtypes.S2STransfer, error) {
stgType := b.detail.UserSpace.Storage.(*jcstypes.OBSType)
feat := types.FindFeature[*jcstypes.S2STransferFeature](b.detail)
feat := stgtypes.FindFeature[*jcstypes.S2STransferFeature](b.detail)
if feat == nil {
return nil, fmt.Errorf("feature %T not found", jcstypes.S2STransferFeature{})
}

24
common/pkgs/storage/obs/s2s.go
View File

@ -14,7 +14,7 @@ import (
omsregion "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/oms/v2/region"
"gitlink.org.cn/cloudream/common/utils/os2"
stgs3 "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/s3"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -43,10 +43,10 @@ func (*S2STransfer) CanTransfer(src, dst *jcstypes.UserSpaceDetail) bool {
}
// 执行数据直传。返回传输后的文件路径
func (s *S2STransfer) Transfer(ctx context.Context, src *jcstypes.UserSpaceDetail, srcPath jcstypes.JPath, dstPath jcstypes.JPath) (types.FileInfo, error) {
func (s *S2STransfer) Transfer(ctx context.Context, src *jcstypes.UserSpaceDetail, srcPath jcstypes.JPath, dstPath jcstypes.JPath) (stgtypes.FileInfo, error) {
req := makeRequest(src, srcPath)
if req == nil {
return types.FileInfo{}, fmt.Errorf("unsupported source storage type: %T", src.UserSpace.Storage)
return stgtypes.FileInfo{}, fmt.Errorf("unsupported source storage type: %T", src.UserSpace.Storage)
}
auth, err := basic.NewCredentialsBuilder().
@ -55,12 +55,12 @@ func (s *S2STransfer) Transfer(ctx context.Context, src *jcstypes.UserSpaceDetai
WithProjectId(s.stgType.ProjectID).
SafeBuild()
if err != nil {
return types.FileInfo{}, err
return stgtypes.FileInfo{}, err
}
region, err := omsregion.SafeValueOf(s.stgType.Region)
if err != nil {
return types.FileInfo{}, err
return stgtypes.FileInfo{}, err
}
cli, err := oms.OmsClientBuilder().
@ -68,11 +68,11 @@ func (s *S2STransfer) Transfer(ctx context.Context, src *jcstypes.UserSpaceDetai
WithCredential(auth).
SafeBuild()
if err != nil {
return types.FileInfo{}, err
return stgtypes.FileInfo{}, err
}
// 先上传成一个临时文件
tempDir := stgs3.JoinKey(s.detail.UserSpace.WorkingDir.String(), types.TempWorkingDir)
tempDir := stgs3.JoinKey(s.detail.UserSpace.WorkingDir.String(), stgtypes.TempWorkingDir)
tempPrefix := stgs3.JoinKey(tempDir, os2.GenerateRandomFileName(10)) + "/"
taskType := model.GetCreateTaskReqTaskTypeEnum().OBJECT
@ -91,7 +91,7 @@ func (s *S2STransfer) Transfer(ctx context.Context, src *jcstypes.UserSpaceDetai
},
})
if err != nil {
return types.FileInfo{}, fmt.Errorf("create task: %w", err)
return stgtypes.FileInfo{}, fmt.Errorf("create task: %w", err)
}
s.taskID = resp.Id
@ -99,13 +99,13 @@ func (s *S2STransfer) Transfer(ctx context.Context, src *jcstypes.UserSpaceDetai
// 轮询任务状态，直到完成
size, err := s.waitTask(ctx, *resp.Id)
if err != nil {
return types.FileInfo{}, fmt.Errorf("wait task: %w", err)
return stgtypes.FileInfo{}, fmt.Errorf("wait task: %w", err)
}
// 传输完成后，将文件名改成目标路径
obsCli, bkt, err := createClient(s.stgType, s.cred)
if err != nil {
return types.FileInfo{}, err
return stgtypes.FileInfo{}, err
}
_, err = obsCli.CopyObject(ctx, &awss3.CopyObjectInput{
@ -114,10 +114,10 @@ func (s *S2STransfer) Transfer(ctx context.Context, src *jcstypes.UserSpaceDetai
Key: aws.String(dstPath.String()),
})
if err != nil {
return types.FileInfo{}, fmt.Errorf("copy object: %w", err)
return stgtypes.FileInfo{}, fmt.Errorf("copy object: %w", err)
}
return types.FileInfo{
return stgtypes.FileInfo{
Path: dstPath,
Size: size,
Hash: "",

10
common/pkgs/storage/obs/shard_store.go
View File

@ -4,7 +4,7 @@ import (
awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/huaweicloud/huaweicloud-sdk-go-obs/obs"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/s3"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -31,10 +31,10 @@ func NewShardStore(detail *jcstypes.UserSpaceDetail, stgType *jcstypes.OBSType,
return &sd, nil
}
func (s *ShardStore) MakeHTTPReadRequest(fileHash jcstypes.FileHash) (types.HTTPRequest, error) {
func (s *ShardStore) MakeHTTPReadRequest(fileHash jcstypes.FileHash) (stgtypes.HTTPRequest, error) {
cli, err := obs.New(s.cred.AK, s.cred.SK, s.stgType.Endpoint)
if err != nil {
return types.HTTPRequest{}, err
return stgtypes.HTTPRequest{}, err
}
filePath := s.GetFilePathFromHash(fileHash)
@ -45,10 +45,10 @@ func (s *ShardStore) MakeHTTPReadRequest(fileHash jcstypes.FileHash) (types.HTTP
Expires: 3600,
})
if err != nil {
return types.HTTPRequest{}, err
return stgtypes.HTTPRequest{}, err
}
return types.HTTPRequest{
return stgtypes.HTTPRequest{
URL: getSigned.SignedUrl,
Method: "GET",
}, nil

18
common/pkgs/storage/pool/pool.go
View File

@ -5,13 +5,13 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/async"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
type userSpace struct {
detail *jcstypes.UserSpaceDetail
store types.ShardStore
store stgtypes.ShardStore
}
func (u *userSpace) Drop() {
@ -29,13 +29,13 @@ type userSpaceKey struct {
type Pool struct {
spaces map[userSpaceKey]*userSpace
lock sync.Mutex
eventChan *types.StorageEventChan
eventChan *stgtypes.StorageEventChan
}
func NewPool() *Pool {
return &Pool{
spaces: make(map[userSpaceKey]*userSpace),
eventChan: async.NewUnboundChannel[types.StorageEvent](),
eventChan: async.NewUnboundChannel[stgtypes.StorageEvent](),
}
}
@ -55,7 +55,7 @@ func (p *Pool) Drop(userID jcstypes.UserID, spaceID jcstypes.UserSpaceID) {
delete(p.spaces, key)
}
func (p *Pool) GetShardStore(spaceDetail *jcstypes.UserSpaceDetail) (types.ShardStore, error) {
func (p *Pool) GetShardStore(spaceDetail *jcstypes.UserSpaceDetail) (stgtypes.ShardStore, error) {
p.lock.Lock()
defer p.lock.Unlock()
@ -90,18 +90,18 @@ func (p *Pool) GetShardStore(spaceDetail *jcstypes.UserSpaceDetail) (types.Shard
return space.store, nil
}
func (p *Pool) GetBaseStore(spaceDetail *jcstypes.UserSpaceDetail) (types.BaseStore, error) {
func (p *Pool) GetBaseStore(spaceDetail *jcstypes.UserSpaceDetail) (stgtypes.BaseStore, error) {
return factory.GetBuilder(spaceDetail).CreateBaseStore(false)
}
func (p *Pool) GetMultiparter(spaceDetail *jcstypes.UserSpaceDetail) (types.Multiparter, error) {
func (p *Pool) GetMultiparter(spaceDetail *jcstypes.UserSpaceDetail) (stgtypes.Multiparter, error) {
return factory.GetBuilder(spaceDetail).CreateMultiparter(false)
}
func (p *Pool) GetS2STransfer(spaceDetail *jcstypes.UserSpaceDetail) (types.S2STransfer, error) {
func (p *Pool) GetS2STransfer(spaceDetail *jcstypes.UserSpaceDetail) (stgtypes.S2STransfer, error) {
return factory.GetBuilder(spaceDetail).CreateS2STransfer(false)
}
func (p *Pool) GetECMultiplier(spaceDetail *jcstypes.UserSpaceDetail) (types.ECMultiplier, error) {
func (p *Pool) GetECMultiplier(spaceDetail *jcstypes.UserSpaceDetail) (stgtypes.ECMultiplier, error) {
return factory.GetBuilder(spaceDetail).CreateECMultiplier(false)
}

14
common/pkgs/storage/pubshards/pubshards.go
View File

@ -2,7 +2,7 @@ package pubshards
import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -11,21 +11,21 @@ func init() {
}
type builder struct {
types.EmptyBuilder
stgtypes.EmptyBuilder
detail *jcstypes.UserSpaceDetail
}
func newBuilder(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
func newBuilder(detail *jcstypes.UserSpaceDetail) stgtypes.StorageBuilder {
return &builder{
detail: detail,
}
}
func (b *builder) FeatureDesc() types.FeatureDesc {
return types.FeatureDesc{}
func (b *builder) FeatureDesc() stgtypes.FeatureDesc {
return stgtypes.FeatureDesc{}
}
func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
func (b *builder) CreateShardStore(typeOnly bool) (stgtypes.ShardStore, error) {
stgType := b.detail.UserSpace.Storage.(*jcstypes.PubShardsType)
if typeOnly {
@ -35,7 +35,7 @@ func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
return NewShardStore(b.detail, stgType)
}
func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) {
func (b *builder) CreateBaseStore(typeOnly bool) (stgtypes.BaseStore, error) {
stgType := b.detail.UserSpace.Storage.(*jcstypes.PubShardsType)
newDeatil := jcstypes.UserSpaceDetail{
UserID: b.detail.UserID,

18
common/pkgs/storage/pubshards/shard_store.go
View File

@ -6,7 +6,7 @@ import (
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
hubrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/hub"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -27,7 +27,7 @@ func NewShardStore(detail *jcstypes.UserSpaceDetail, stgType *jcstypes.PubShards
}, nil
}
func (s *ShardStore) Start(ch *types.StorageEventChan) {
func (s *ShardStore) Start(ch *stgtypes.StorageEventChan) {
s.hubCli = stgglb.HubRPCPool.GetByID(s.stgType.MasterHub)
}
@ -38,7 +38,7 @@ func (s *ShardStore) Stop() {
}
}
func (s *ShardStore) Store(path jcstypes.JPath, hash jcstypes.FileHash, size int64) (types.FileInfo, error) {
func (s *ShardStore) Store(path jcstypes.JPath, hash jcstypes.FileHash, size int64) (stgtypes.FileInfo, error) {
resp, cerr := s.hubCli.PubShardsStore(context.Background(), &hubrpc.PubShardsStore{
PubShardsID: s.stgType.PubShardsID,
Password: s.stgType.Password,
@ -47,25 +47,25 @@ func (s *ShardStore) Store(path jcstypes.JPath, hash jcstypes.FileHash, size int
Size: size,
})
if cerr != nil {
return types.FileInfo{}, cerr.ToError()
return stgtypes.FileInfo{}, cerr.ToError()
}
return resp.Info, nil
}
func (s *ShardStore) Info(hash jcstypes.FileHash) (types.FileInfo, error) {
func (s *ShardStore) Info(hash jcstypes.FileHash) (stgtypes.FileInfo, error) {
resp, cerr := s.hubCli.PubShardsInfo(context.Background(), &hubrpc.PubShardsInfo{
PubShardsID: s.stgType.PubShardsID,
Password: s.stgType.Password,
FileHash: hash,
})
if cerr != nil {
return types.FileInfo{}, cerr.ToError()
return stgtypes.FileInfo{}, cerr.ToError()
}
return resp.Info, nil
}
func (s *ShardStore) ListAll() ([]types.FileInfo, error) {
func (s *ShardStore) ListAll() ([]stgtypes.FileInfo, error) {
resp, cerr := s.hubCli.PubShardsListAll(context.Background(), &hubrpc.PubShardsListAll{
PubShardsID: s.stgType.PubShardsID,
Password: s.stgType.Password,
@ -88,13 +88,13 @@ func (s *ShardStore) GC(avaiables []jcstypes.FileHash) error {
return nil
}
func (s *ShardStore) Stats() types.Stats {
func (s *ShardStore) Stats() stgtypes.Stats {
resp, cerr := s.hubCli.PubShardsStats(context.Background(), &hubrpc.PubShardsStats{
PubShardsID: s.stgType.PubShardsID,
Password: s.stgType.Password,
})
if cerr != nil {
return types.Stats{}
return stgtypes.Stats{}
}
return resp.Stats
}

24
common/pkgs/storage/s3/base_store.go
View File

@ -15,7 +15,7 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/io2"
"gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -43,7 +43,7 @@ func NewBaseStore(detail *jcstypes.UserSpaceDetail, cli *s3.Client, bkt string,
}, nil
}
func (s *BaseStore) Write(pat jcstypes.JPath, stream io.Reader, opt types.WriteOption) (types.FileInfo, error) {
func (s *BaseStore) Write(pat jcstypes.JPath, stream io.Reader, opt stgtypes.WriteOption) (stgtypes.FileInfo, error) {
key := pat
meta := make(map[string]string)
if opt.ModTime.IsZero() {
@ -52,7 +52,7 @@ func (s *BaseStore) Write(pat jcstypes.JPath, stream io.Reader, opt types.WriteO
} else {
mt, err := opt.ModTime.MarshalText()
if err != nil {
return types.FileInfo{}, err
return stgtypes.FileInfo{}, err
}
meta[ModTimeHeader] = string(mt)
}
@ -68,18 +68,18 @@ func (s *BaseStore) Write(pat jcstypes.JPath, stream io.Reader, opt types.WriteO
Metadata: meta,
})
if err != nil {
return types.FileInfo{}, err
return stgtypes.FileInfo{}, err
}
if resp.ChecksumSHA256 == nil {
return types.FileInfo{}, errors.New("SHA256 checksum not found in response")
return stgtypes.FileInfo{}, errors.New("SHA256 checksum not found in response")
}
hash, err := DecodeBase64Hash(*resp.ChecksumSHA256)
if err != nil {
return types.FileInfo{}, fmt.Errorf("decode SHA256 checksum: %v", err)
return stgtypes.FileInfo{}, fmt.Errorf("decode SHA256 checksum: %v", err)
}
return types.FileInfo{
return stgtypes.FileInfo{
Path: key,
Hash: jcstypes.NewFullHash(hash),
Size: counter.Count(),
@ -94,17 +94,17 @@ func (s *BaseStore) Write(pat jcstypes.JPath, stream io.Reader, opt types.WriteO
Metadata: meta,
})
if err != nil {
return types.FileInfo{}, err
return stgtypes.FileInfo{}, err
}
return types.FileInfo{
return stgtypes.FileInfo{
Path: key,
Hash: jcstypes.NewFullHash(hashStr.Sum()),
Size: counter.Count(),
}, nil
}
func (s *BaseStore) Read(pat jcstypes.JPath, opt types.OpenOption) (io.ReadCloser, error) {
func (s *BaseStore) Read(pat jcstypes.JPath, opt stgtypes.OpenOption) (io.ReadCloser, error) {
key := pat
input := &s3.GetObjectInput{
@ -137,7 +137,7 @@ func (s *BaseStore) Mkdir(path jcstypes.JPath) error {
return err
}
func (s *BaseStore) ReadDir(path jcstypes.JPath) types.DirReader {
func (s *BaseStore) ReadDir(path jcstypes.JPath) stgtypes.DirReader {
return &DirReader{
cli: s.cli,
bucket: s.Bucket,
@ -154,7 +154,7 @@ func (s *BaseStore) CleanTemps() {
for {
resp, err := s.cli.ListObjects(context.Background(), &s3.ListObjectsInput{
Bucket: aws.String(s.Bucket),
Prefix: aws.String(JoinKey(s.Detail.UserSpace.WorkingDir.String(), types.TempWorkingDir, "/")),
Prefix: aws.String(JoinKey(s.Detail.UserSpace.WorkingDir.String(), stgtypes.TempWorkingDir, "/")),
Marker: marker,
})

12
common/pkgs/storage/s3/dir_reader.go
View File

@ -6,7 +6,7 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -15,18 +15,18 @@ type DirReader struct {
bucket string
rootPath jcstypes.JPath
marker *string
curInfos []types.DirEntry
curInfos []stgtypes.DirEntry
eof bool
}
func (r *DirReader) Next() (types.DirEntry, error) {
func (r *DirReader) Next() (stgtypes.DirEntry, error) {
if len(r.curInfos) > 0 {
e := r.curInfos[0]
r.curInfos = r.curInfos[1:]
return e, nil
}
if r.eof {
return types.DirEntry{}, io.EOF
return stgtypes.DirEntry{}, io.EOF
}
resp, err := r.cli.ListObjects(context.Background(), &s3.ListObjectsInput{
@ -35,13 +35,13 @@ func (r *DirReader) Next() (types.DirEntry, error) {
Marker: r.marker,
})
if err != nil {
return types.DirEntry{}, err
return stgtypes.DirEntry{}, err
}
for _, obj := range resp.Contents {
key := jcstypes.PathFromJcsPathString(*obj.Key)
r.curInfos = append(r.curInfos, types.DirEntry{
r.curInfos = append(r.curInfos, stgtypes.DirEntry{
Path: key,
Size: *obj.Size,
ModTime: *obj.LastModified,

26
common/pkgs/storage/s3/multipart_upload.go
View File

@ -11,7 +11,7 @@ import (
"gitlink.org.cn/cloudream/common/utils/io2"
"gitlink.org.cn/cloudream/common/utils/os2"
"gitlink.org.cn/cloudream/common/utils/sort2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -39,10 +39,10 @@ func (*Multiparter) MaxPartSize() int64 {
return 5 * 1024 * 1024 * 1024 // 5GB
}
func (m *Multiparter) Initiate(ctx context.Context) (types.MultipartTask, error) {
func (m *Multiparter) Initiate(ctx context.Context) (stgtypes.MultipartTask, error) {
tempFileName := os2.GenerateRandomFileName(10)
tempDir := m.detail.UserSpace.WorkingDir.Clone()
tempDir.Push(types.TempWorkingDir)
tempDir.Push(stgtypes.TempWorkingDir)
tempFilePath := tempDir.Clone()
tempFilePath.Push(tempFileName)
@ -65,7 +65,7 @@ func (m *Multiparter) Initiate(ctx context.Context) (types.MultipartTask, error)
}, nil
}
func (m *Multiparter) UploadPart(ctx context.Context, init types.MultipartInitState, partSize int64, partNumber int, stream io.Reader) (types.UploadedPartInfo, error) {
func (m *Multiparter) UploadPart(ctx context.Context, init stgtypes.MultipartInitState, partSize int64, partNumber int, stream io.Reader) (stgtypes.UploadedPartInfo, error) {
hashStr := io2.NewReadHasher(sha256.New(), stream)
resp, err := m.cli.UploadPart(ctx, &s3.UploadPartInput{
Bucket: aws.String(init.Bucket),
@ -75,10 +75,10 @@ func (m *Multiparter) UploadPart(ctx context.Context, init types.MultipartInitSt
Body: hashStr,
})
if err != nil {
return types.UploadedPartInfo{}, err
return stgtypes.UploadedPartInfo{}, err
}
return types.UploadedPartInfo{
return stgtypes.UploadedPartInfo{
ETag: *resp.ETag,
PartNumber: partNumber,
PartHash: hashStr.Sum(),
@ -94,16 +94,16 @@ type MultipartTask struct {
uploadID string
}
func (i *MultipartTask) InitState() types.MultipartInitState {
return types.MultipartInitState{
func (i *MultipartTask) InitState() stgtypes.MultipartInitState {
return stgtypes.MultipartInitState{
UploadID: i.uploadID,
Bucket: i.bucket,
Key: i.tempFilePath.String(),
}
}
func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPartInfo) (types.FileInfo, error) {
parts = sort2.Sort(parts, func(l, r types.UploadedPartInfo) int {
func (i *MultipartTask) JoinParts(ctx context.Context, parts []stgtypes.UploadedPartInfo) (stgtypes.FileInfo, error) {
parts = sort2.Sort(parts, func(l, r stgtypes.UploadedPartInfo) int {
return l.PartNumber - r.PartNumber
})
@ -128,7 +128,7 @@ func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPar
},
})
if err != nil {
return types.FileInfo{}, err
return stgtypes.FileInfo{}, err
}
headResp, err := i.cli.HeadObject(ctx, &s3.HeadObjectInput{
@ -136,12 +136,12 @@ func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPar
Key: aws.String(i.tempFilePath.String()),
})
if err != nil {
return types.FileInfo{}, err
return stgtypes.FileInfo{}, err
}
hash := jcstypes.CalculateCompositeHash(partHashes)
return types.FileInfo{
return stgtypes.FileInfo{
Path: i.tempFilePath,
Size: *headResp.ContentLength,
Hash: hash,

18
common/pkgs/storage/s3/s3.go
View File

@ -7,7 +7,7 @@ import (
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -16,21 +16,21 @@ func init() {
}
type builder struct {
types.EmptyBuilder
stgtypes.EmptyBuilder
detail *jcstypes.UserSpaceDetail
}
func newBuilder(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
func newBuilder(detail *jcstypes.UserSpaceDetail) stgtypes.StorageBuilder {
return &builder{
detail: detail,
}
}
func (b *builder) FeatureDesc() types.FeatureDesc {
return types.FeatureDesc{}
func (b *builder) FeatureDesc() stgtypes.FeatureDesc {
return stgtypes.FeatureDesc{}
}
func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
func (b *builder) CreateShardStore(typeOnly bool) (stgtypes.ShardStore, error) {
stgType := b.detail.UserSpace.Storage.(*jcstypes.S3Type)
s3Cred, ok := b.detail.UserSpace.Credential.(*jcstypes.S3Cred)
if !ok {
@ -49,7 +49,7 @@ func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
return NewShardStore(b.detail, cli, bkt, ShardStoreOption{UseAWSSha256: true})
}
func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) {
func (b *builder) CreateBaseStore(typeOnly bool) (stgtypes.BaseStore, error) {
stgType := b.detail.UserSpace.Storage.(*jcstypes.S3Type)
s3Cred, ok := b.detail.UserSpace.Credential.(*jcstypes.S3Cred)
if !ok {
@ -90,9 +90,9 @@ func createClient(stgType *jcstypes.S3Type, cred *jcstypes.S3Cred) (*s3.Client,
return cli, stgType.Bucket, nil
}
func (b *builder) CreateMultiparter(typeOnly bool) (types.Multiparter, error) {
func (b *builder) CreateMultiparter(typeOnly bool) (stgtypes.Multiparter, error) {
stgType := b.detail.UserSpace.Storage.(*jcstypes.S3Type)
feat := types.FindFeature[*jcstypes.MultipartUploadFeature](b.detail)
feat := stgtypes.FindFeature[*jcstypes.MultipartUploadFeature](b.detail)
if feat == nil {
return nil, fmt.Errorf("feature %T not found", jcstypes.MultipartUploadFeature{})
}

30
common/pkgs/storage/s3/shard_store.go
View File

@ -9,7 +9,7 @@ import (
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)
@ -28,7 +28,7 @@ type ShardStore struct {
func NewShardStore(detail *jcstypes.UserSpaceDetail, cli *s3.Client, bkt string, opt ShardStoreOption) (*ShardStore, error) {
wd := detail.UserSpace.WorkingDir.Clone()
wd.Push(types.ShardStoreWorkingDir)
wd.Push(stgtypes.ShardStoreWorkingDir)
return &ShardStore{
Detail: detail,
Bucket: bkt,
@ -38,7 +38,7 @@ func NewShardStore(detail *jcstypes.UserSpaceDetail, cli *s3.Client, bkt string,
}, nil
}
func (s *ShardStore) Start(ch *types.StorageEventChan) {
func (s *ShardStore) Start(ch *stgtypes.StorageEventChan) {
s.getLogger().Infof("start, root: %v", s.workingDir)
}
@ -46,7 +46,7 @@ func (s *ShardStore) Stop() {
s.getLogger().Infof("component stop")
}
func (s *ShardStore) Store(path jcstypes.JPath, hash jcstypes.FileHash, size int64) (types.FileInfo, error) {
func (s *ShardStore) Store(path jcstypes.JPath, hash jcstypes.FileHash, size int64) (stgtypes.FileInfo, error) {
s.lock.Lock()
defer s.lock.Unlock()
@ -63,17 +63,17 @@ func (s *ShardStore) Store(path jcstypes.JPath, hash jcstypes.FileHash, size int
})
if err != nil {
log.Warnf("copy file %v to %v: %v", path, newPath, err)
return types.FileInfo{}, err
return stgtypes.FileInfo{}, err
}
return types.FileInfo{
return stgtypes.FileInfo{
Hash: hash,
Size: size,
Path: newPath,
}, nil
}
func (s *ShardStore) Info(hash jcstypes.FileHash) (types.FileInfo, error) {
func (s *ShardStore) Info(hash jcstypes.FileHash) (stgtypes.FileInfo, error) {
s.lock.Lock()
defer s.lock.Unlock()
@ -84,21 +84,21 @@ func (s *ShardStore) Info(hash jcstypes.FileHash) (types.FileInfo, error) {
})
if err != nil {
s.getLogger().Warnf("get file %v: %v", filePath, err)
return types.FileInfo{}, err
return stgtypes.FileInfo{}, err
}
return types.FileInfo{
return stgtypes.FileInfo{
Hash: hash,
Size: *info.ContentLength,
Path: filePath,
}, nil
}
func (s *ShardStore) ListAll() ([]types.FileInfo, error) {
func (s *ShardStore) ListAll() ([]stgtypes.FileInfo, error) {
s.lock.Lock()
defer s.lock.Unlock()
var infos []types.FileInfo
var infos []stgtypes.FileInfo
var marker *string
for {
@ -121,7 +121,7 @@ func (s *ShardStore) ListAll() ([]types.FileInfo, error) {
continue
}
infos = append(infos, types.FileInfo{
infos = append(infos, stgtypes.FileInfo{
Hash: fileHash,
Size: *obj.Size,
Path: jcstypes.PathFromJcsPathString(*obj.Key),
@ -205,10 +205,10 @@ func (s *ShardStore) GC(avaiables []jcstypes.FileHash) error {
return nil
}
func (s *ShardStore) Stats() types.Stats {
func (s *ShardStore) Stats() stgtypes.Stats {
// TODO 统计本地存储的相关信息
return types.Stats{
Status: types.StatusOK,
return stgtypes.Stats{
Status: stgtypes.StatusOK,
}
}

2
common/pkgs/storage/types/base_store.go
View File

@ -1,4 +1,4 @@
package types
package stgtypes
import (
"fmt"

2
common/pkgs/storage/types/bypass.go
View File

@ -1,4 +1,4 @@
package types
package stgtypes
import (
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"

2
common/pkgs/storage/types/ec_multiplier.go
View File

@ -1,4 +1,4 @@
package types
package stgtypes
type ECMultiplier interface {
// 进行EC运算，coef * inputs。coef为编码矩阵，inputs为待编码数据，chunkSize为分块大小。

2
common/pkgs/storage/types/empty_builder.go
View File

@ -1,4 +1,4 @@
package types
package stgtypes
import (
"fmt"

2
common/pkgs/storage/types/faas.go
View File

@ -1,4 +1,4 @@
package types
package stgtypes
import "context"

2
common/pkgs/storage/types/multiparter.go
View File

@ -1,4 +1,4 @@
package types
package stgtypes
import (
"context"

2
common/pkgs/storage/types/s2s.go
View File

@ -1,4 +1,4 @@
package types
package stgtypes
import (
"context"

2
common/pkgs/storage/types/shard_store.go
View File

@ -1,4 +1,4 @@
package types
package stgtypes
import (
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"

2
common/pkgs/storage/types/types.go
View File

@ -1,4 +1,4 @@
package types
package stgtypes
import (
"errors"

2
common/pkgs/storage/types/utils.go
View File

@ -1,4 +1,4 @@
package types
package stgtypes
import (
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"