JointCloud
/
JCS-pub
24
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

增加上传下载文件的命令

This commit is contained in:
Sydonian 2025-06-30 16:53:51 +08:00
parent 72eecbe356
commit c86826a79b
37 changed files with 1455 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
client/internal/db/db.go
View File

@ -44,6 +44,17 @@ func DoTx01[R any](db *DB, do func(tx SQLContext) (R, error)) (R, error) {
return ret, err
}
func DoTx02[R1, R2 any](db *DB, do func(tx SQLContext) (R1, R2, error)) (R1, R2, error) {
var ret1 R1
var ret2 R2
err := db.db.Transaction(func(tx *gorm.DB) error {
var err error
ret1, ret2, err = do(SQLContext{tx})
return err
})
return ret1, ret2, err
}
func DoTx11[T any, R any](db *DB, do func(tx SQLContext, t T) (R, error), t T) (R, error) {
var ret R
err := db.db.Transaction(func(tx *gorm.DB) error {

72
client/internal/downloader/downloader.go
View File

@ -8,7 +8,7 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/iterator"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader/strategy"
"gitlink.org.cn/cloudream/jcs-pub/client/types"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/connectivity"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
)
@ -20,18 +20,18 @@ const (
type DownloadIterator = iterator.Iterator[*Downloading]
type DownloadReqeust struct {
ObjectID types.ObjectID
ObjectID clitypes.ObjectID
Offset int64
Length int64
}
type downloadReqeust2 struct {
Detail *types.ObjectDetail
Detail *clitypes.ObjectDetail
Raw DownloadReqeust
}
type Downloading struct {
Object *types.Object
Object *clitypes.Object
File io.ReadCloser // 文件流，如果文件不存在，那么为nil
Request DownloadReqeust
}
@ -62,7 +62,7 @@ func NewDownloader(cfg Config, conn *connectivity.Collector, stgPool *pool.Pool,
}
func (d *Downloader) DownloadObjects(reqs []DownloadReqeust) DownloadIterator {
objIDs := make([]types.ObjectID, len(reqs))
objIDs := make([]clitypes.ObjectID, len(reqs))
for i, req := range reqs {
objIDs[i] = req.ObjectID
}
@ -76,7 +76,7 @@ func (d *Downloader) DownloadObjects(reqs []DownloadReqeust) DownloadIterator {
return iterator.FuseError[*Downloading](fmt.Errorf("request to db: %w", err))
}
detailsMap := make(map[types.ObjectID]*types.ObjectDetail)
detailsMap := make(map[clitypes.ObjectID]*clitypes.ObjectDetail)
for _, detail := range objDetails {
d := detail
detailsMap[detail.Object.ObjectID] = &d
@ -93,7 +93,7 @@ func (d *Downloader) DownloadObjects(reqs []DownloadReqeust) DownloadIterator {
return NewDownloadObjectIterator(d, req2s)
}
func (d *Downloader) DownloadObjectByDetail(detail types.ObjectDetail, off int64, length int64) (*Downloading, error) {
func (d *Downloader) DownloadObjectByDetail(detail clitypes.ObjectDetail, off int64, length int64) (*Downloading, error) {
req2s := []downloadReqeust2{{
Detail: &detail,
Raw: DownloadReqeust{
@ -107,10 +107,56 @@ func (d *Downloader) DownloadObjectByDetail(detail types.ObjectDetail, off int64
return iter.MoveNext()
}
func (d *Downloader) DownloadPackage(pkgID types.PackageID) DownloadIterator {
details, err := db.DoTx11(d.db, d.db.Object().GetPackageObjectDetails, pkgID)
func (d *Downloader) DownloadPackage(pkgID clitypes.PackageID, prefix string) (clitypes.Package, DownloadIterator, error) {
pkg, details, err := db.DoTx02(d.db, func(tx db.SQLContext) (clitypes.Package, []clitypes.ObjectDetail, error) {
pkg, err := d.db.Package().GetByID(tx, pkgID)
if err != nil {
return clitypes.Package{}, nil, err
}
var details []clitypes.ObjectDetail
if prefix != "" {
objs, err := d.db.Object().GetWithPathPrefix(tx, pkgID, prefix)
if err != nil {
return clitypes.Package{}, nil, err
}
objIDs := make([]clitypes.ObjectID, len(objs))
for i, obj := range objs {
objIDs[i] = obj.ObjectID
}
allBlocks, err := d.db.ObjectBlock().BatchGetByObjectID(tx, objIDs)
if err != nil {
return clitypes.Package{}, nil, err
}
allPinnedObjs, err := d.db.PinnedObject().BatchGetByObjectID(tx, objIDs)
if err != nil {
return clitypes.Package{}, nil, err
}
details = make([]clitypes.ObjectDetail, 0, len(objs))
for _, obj := range objs {
detail := clitypes.ObjectDetail{
Object: obj,
}
details = append(details, detail)
}
clitypes.DetailsFillObjectBlocks(details, allBlocks)
clitypes.DetailsFillPinnedAt(details, allPinnedObjs)
} else {
details, err = d.db.Object().GetPackageObjectDetails(tx, pkgID)
if err != nil {
return clitypes.Package{}, nil, err
}
}
return pkg, details, nil
})
if err != nil {
return iterator.FuseError[*Downloading](fmt.Errorf("get package object details: %w", err))
return clitypes.Package{}, nil, err
}
req2s := make([]downloadReqeust2, len(details))
@ -126,16 +172,16 @@ func (d *Downloader) DownloadPackage(pkgID types.PackageID) DownloadIterator {
}
}
return NewDownloadObjectIterator(d, req2s)
return pkg, NewDownloadObjectIterator(d, req2s), nil
}
type ObjectECStrip struct {
Data []byte
ObjectFileHash types.FileHash // 添加这条缓存时，Object的FileHash
ObjectFileHash clitypes.FileHash // 添加这条缓存时，Object的FileHash
}
type ECStripKey struct {
ObjectID types.ObjectID
ObjectID clitypes.ObjectID
StripIndex int64
}

22
client/internal/http/v1/bucket.go
View File

@ -22,6 +22,28 @@ func (s *Server) Bucket() *BucketService {
}
}
func (s *BucketService) Get(ctx *gin.Context) {
log := logger.WithField("HTTP", "Bucket.Get")
var req cliapi.BucketGet
if err := ctx.ShouldBindQuery(&req); err != nil {
log.Warnf("binding query: %s", err.Error())
ctx.JSON(http.StatusBadRequest, types.Failed(ecode.BadArgument, "missing argument or invalid argument"))
return
}
bucket, err := s.svc.DB.Bucket().GetByID(s.svc.DB.DefCtx(), req.BucketID)
if err != nil {
log.Warnf("getting bucket by name: %s", err.Error())
ctx.JSON(http.StatusOK, types.FailedError(err))
return
}
ctx.JSON(http.StatusOK, types.OK(cliapi.BucketGetResp{
Bucket: bucket,
}))
}
func (s *BucketService) GetByName(ctx *gin.Context) {
log := logger.WithField("HTTP", "Bucket.GetByName")

4
client/internal/http/v1/object.go
View File

@ -69,7 +69,7 @@ func (s *ObjectService) ListByIDs(ctx *gin.Context) {
ctx.JSON(http.StatusOK, types.OK(cliapi.ObjectListByIDsResp{Objects: objs}))
}
type ObjectUploadReq struct {
type ObjectUpload struct {
Info cliapi.ObjectUploadInfo `form:"info" binding:"required"`
Files []*multipart.FileHeader `form:"files"`
}
@ -77,7 +77,7 @@ type ObjectUploadReq struct {
func (s *ObjectService) Upload(ctx *gin.Context) {
log := logger.WithField("HTTP", "Object.Upload")
var req ObjectUploadReq
var req ObjectUpload
if err := ctx.ShouldBind(&req); err != nil {
log.Warnf("binding body: %s", err.Error())
ctx.JSON(http.StatusBadRequest, types.Failed(ecode.BadArgument, "missing argument or invalid argument"))

124
client/internal/http/v1/package.go
View File

@ -1,18 +1,24 @@
package http
import (
"archive/tar"
"archive/zip"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/url"
"path/filepath"
"github.com/gin-gonic/gin"
"gitlink.org.cn/cloudream/common/consts/errorcode"
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/ecode"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/iterator"
)
// PackageService 包服务，负责处理包相关的HTTP请求。
@ -30,7 +36,7 @@ func (s *Server) Package() *PackageService {
func (s *PackageService) Get(ctx *gin.Context) {
log := logger.WithField("HTTP", "Package.Get")
var req cliapi.PackageGetReq
var req cliapi.PackageGet
if err := ctx.ShouldBindQuery(&req); err != nil {
log.Warnf("binding body: %s", err.Error())
ctx.JSON(http.StatusBadRequest, types.Failed(ecode.BadArgument, "missing argument or invalid argument"))
@ -164,6 +170,122 @@ func (s *PackageService) CreateLoad(ctx *gin.Context) {
ctx.JSON(http.StatusOK, types.OK(cliapi.PackageCreateUploadResp{Package: ret.Package, Objects: objs}))
}
func (s *PackageService) Download(ctx *gin.Context) {
log := logger.WithField("HTTP", "Package.Download")
var req cliapi.PackageDownload
if err := ctx.ShouldBindQuery(&req); err != nil {
log.Warnf("binding query: %s", err.Error())
ctx.JSON(http.StatusBadRequest, types.Failed(errorcode.BadArgument, "missing argument or invalid argument"))
return
}
pkg, iter, err := s.svc.Downloader.DownloadPackage(req.PackageID, req.Prefix)
if err != nil {
log.Warnf("downloading package: %s", err.Error())
ctx.JSON(http.StatusOK, types.Failed(errorcode.OperationFailed, err.Error()))
return
}
defer iter.Close()
if req.Zip {
s.downloadZip(ctx, req, pkg, iter)
} else {
s.downloadTar(ctx, req, pkg, iter)
}
}
func (s *PackageService) downloadZip(ctx *gin.Context, req cliapi.PackageDownload, pkg clitypes.Package, iter downloader.DownloadIterator) {
log := logger.WithField("HTTP", "Package.Download")
ctx.Header("Content-Disposition", "attachment; filename="+url.PathEscape(pkg.Name)+".zip")
ctx.Header("Content-Type", "application/zip")
ctx.Header("Content-Transfer-Encoding", "binary")
zipFile := zip.NewWriter(ctx.Writer)
defer zipFile.Close()
for {
item, err := iter.MoveNext()
if err == iterator.ErrNoMoreItem {
return
}
if err != nil {
log.Warnf("iterating next object: %v", err)
return
}
filePath := item.Object.Path
if req.Prefix != "" && req.NewPrefix != nil {
filePath = *req.NewPrefix + filePath[len(req.Prefix):]
}
zf, err := zipFile.Create(filePath)
if err != nil {
log.Warnf("creating zip file: %v", err)
item.File.Close()
return
}
_, err = io.Copy(zf, item.File)
if err != nil {
log.Warnf("copying file to zip: %v", err)
item.File.Close()
return
}
item.File.Close()
}
}
func (s *PackageService) downloadTar(ctx *gin.Context, req cliapi.PackageDownload, pkg clitypes.Package, iter downloader.DownloadIterator) {
log := logger.WithField("HTTP", "Package.Download")
ctx.Header("Content-Disposition", "attachment; filename="+url.PathEscape(pkg.Name)+".tar")
ctx.Header("Content-Type", "application/x-tar")
ctx.Header("Content-Transfer-Encoding", "binary")
tarFile := tar.NewWriter(ctx.Writer)
defer tarFile.Close()
for {
item, err := iter.MoveNext()
if err == iterator.ErrNoMoreItem {
return
}
if err != nil {
log.Warnf("iterating next object: %v", err)
return
}
filePath := item.Object.Path
if req.Prefix != "" && req.NewPrefix != nil {
filePath = *req.NewPrefix + filePath[len(req.Prefix):]
}
err = tarFile.WriteHeader(&tar.Header{
Typeflag: tar.TypeReg,
Name: filePath,
Size: item.Object.Size,
ModTime: item.Object.CreateTime,
})
if err != nil {
log.Warnf("creating tar header: %v", err)
item.File.Close()
return
}
_, err = io.Copy(tarFile, item.File)
if err != nil {
log.Warnf("copying file to tar: %v", err)
item.File.Close()
return
}
item.File.Close()
}
}
func (s *PackageService) Delete(ctx *gin.Context) {
log := logger.WithField("HTTP", "Package.Delete")

2
client/internal/http/v1/server.go
View File

@ -43,6 +43,7 @@ func (s *Server) InitRouters(rt gin.IRoutes, ah *auth.Auth) {
rt.POST(cliapi.PackageCreateUploadPath, certAuth, s.Package().CreateLoad)
rt.POST(cliapi.PackageDeletePath, certAuth, s.Package().Delete)
rt.POST(cliapi.PackageClonePath, certAuth, s.Package().Clone)
rt.GET(cliapi.PackageDownloadPath, certAuth, s.Package().Download)
rt.GET(cliapi.PackageListBucketPackagesPath, certAuth, s.Package().ListBucketPackages)
rt.POST(cliapi.UserSpaceDownloadPackagePath, certAuth, s.UserSpace().DownloadPackage)
@ -55,6 +56,7 @@ func (s *Server) InitRouters(rt gin.IRoutes, ah *auth.Auth) {
rt.POST(cliapi.UserSpaceDeletePath, certAuth, s.UserSpace().Delete)
rt.POST(cliapi.UserSpaceTestPath, certAuth, s.UserSpace().Test)
rt.GET(cliapi.BucketGetPath, certAuth, s.Bucket().Get)
rt.GET(cliapi.BucketGetByNamePath, certAuth, s.Bucket().GetByName)
rt.POST(cliapi.BucketCreatePath, certAuth, s.Bucket().Create)
rt.POST(cliapi.BucketDeletePath, certAuth, s.Bucket().Delete)

5
client/internal/services/package.go
View File

@ -7,7 +7,6 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader"
"gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/models/datamap"
)
@ -47,10 +46,6 @@ func (svc *PackageService) Create(bucketID types.BucketID, name string) (types.P
return pkg, nil
}
func (svc *PackageService) DownloadPackage(packageID types.PackageID) (downloader.DownloadIterator, error) {
return svc.Downloader.DownloadPackage(packageID), nil
}
// DeletePackage 删除指定的包
func (svc *PackageService) DeletePackage(packageID types.PackageID) error {
err := svc.DB.Package().DeleteComplete(svc.DB.DefCtx(), packageID)

4
client/internal/ticktock/redundancy_recover.go
View File

@ -37,14 +37,14 @@ func (t *ChangeRedundancy) chooseRedundancy(ctx *changeRedundancyContext, obj cl
return &clitypes.DefaultECRedundancy, newStgs
}
return clitypes.DefaultRepRedundancy, t.rechooseUserSpacesForRep(ctx, &clitypes.DefaultRepRedundancy)
return &clitypes.DefaultRepRedundancy, t.rechooseUserSpacesForRep(ctx, &clitypes.DefaultRepRedundancy)
case *clitypes.ECRedundancy:
if obj.Object.Size < ctx.ticktock.cfg.ECFileSizeThreshold {
return &clitypes.DefaultRepRedundancy, t.chooseNewUserSpacesForRep(ctx, &clitypes.DefaultRepRedundancy)
}
return clitypes.DefaultECRedundancy, t.rechooseUserSpacesForEC(ctx, obj, &clitypes.DefaultECRedundancy)
return &clitypes.DefaultECRedundancy, t.rechooseUserSpacesForEC(ctx, obj, &clitypes.DefaultECRedundancy)
case *clitypes.LRCRedundancy:
newLRCStgs := t.rechooseUserSpacesForLRC(ctx, obj, &clitypes.DefaultLRCRedundancy)

22
client/sdk/api/v1/bucket.go
View File

@ -15,6 +15,28 @@ func (c *Client) Bucket() *BucketService {
return &BucketService{c}
}
const BucketGetPath = "/bucket/get"
type BucketGet struct {
BucketID clitypes.BucketID `json:"bucketID" binding:"required"`
}
func (r *BucketGet) MakeParam() *sdks.RequestParam {
return sdks.MakeJSONParam(http.MethodGet, BucketGetPath, r)
}
type BucketGetResp struct {
Bucket clitypes.Bucket `json:"bucket"`
}
func (r *BucketGetResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}
func (c *BucketService) Get(req BucketGet) (*BucketGetResp, error) {
return JSONAPI(&c.cfg, c.httpCli, &req, &BucketGetResp{})
}
const BucketGetByNamePath = "/bucket/getByName"
type BucketGetByName struct {

14
client/sdk/api/v1/object.go
View File

@ -119,7 +119,7 @@ func (c *ObjectService) Upload(req ObjectUpload) (*ObjectUploadResp, error) {
return nil, fmt.Errorf("upload info to json: %w", err)
}
resp, err := PostMultiPart(&c.cfg, url,
resp, err := PostMultiPart(&c.cfg, c.httpCli, url,
uploadInfo{Info: string(infoJSON)},
iterator.Map(req.Files, func(src *UploadingObject) (*http2.IterMultiPartFile, error) {
return &http2.IterMultiPartFile{
@ -169,7 +169,11 @@ type DownloadingObject struct {
}
func (c *ObjectService) Download(req ObjectDownload) (*DownloadingObject, error) {
httpReq, err := req.MakeParam().MakeRequest(c.cfg.EndPoint)
u, err := url.JoinPath(c.cfg.EndPoint, "v1")
if err != nil {
return nil, err
}
httpReq, err := req.MakeParam().MakeRequest(u)
if err != nil {
return nil, err
}
@ -215,7 +219,11 @@ func (r *ObjectDownloadByPath) MakeParam() *sdks.RequestParam {
}
func (c *ObjectService) DownloadByPath(req ObjectDownloadByPath) (*DownloadingObject, error) {
httpReq, err := req.MakeParam().MakeRequest(c.cfg.EndPoint)
u, err := url.JoinPath(c.cfg.EndPoint, "v1")
if err != nil {
return nil, err
}
httpReq, err := req.MakeParam().MakeRequest(u)
if err != nil {
return nil, err
}

75
client/sdk/api/v1/package.go
View File

@ -2,8 +2,11 @@ package api
import (
"fmt"
"io"
"mime"
"net/http"
"net/url"
"strings"
"gitlink.org.cn/cloudream/common/consts/errorcode"
"gitlink.org.cn/cloudream/common/pkgs/iterator"
@ -23,23 +26,23 @@ func (c *Client) Package() *PackageService {
const PackageGetPath = "/package/get"
type PackageGetReq struct {
type PackageGet struct {
PackageID clitypes.PackageID `form:"packageID" url:"packageID" binding:"required"`
}
func (r *PackageGetReq) MakeParam() *sdks.RequestParam {
func (r *PackageGet) MakeParam() *sdks.RequestParam {
return sdks.MakeQueryParam(http.MethodGet, PackageGetPath, r)
}
type PackageGetResp struct {
clitypes.Package
Package clitypes.Package `json:"package"`
}
func (r *PackageGetResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}
func (c *PackageService) Get(req PackageGetReq) (*PackageGetResp, error) {
func (c *PackageService) Get(req PackageGet) (*PackageGetResp, error) {
return JSONAPI(&c.cfg, c.httpCli, &req, &PackageGetResp{})
}
@ -62,7 +65,7 @@ func (r *PackageGetByFullNameResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}
func (c *PackageService) GetByName(req PackageGetByFullName) (*PackageGetByFullNameResp, error) {
func (c *PackageService) GetByFullName(req PackageGetByFullName) (*PackageGetByFullNameResp, error) {
return JSONAPI(&c.cfg, c.httpCli, &req, &PackageGetByFullNameResp{})
}
@ -117,7 +120,7 @@ func (c *PackageService) CreateUpload(req PackageCreateUpload) (*PackageCreateUp
return nil, fmt.Errorf("upload info to json: %w", err)
}
resp, err := PostMultiPart(&c.cfg, url,
resp, err := PostMultiPart(&c.cfg, c.httpCli, url,
map[string]string{"info": string(infoJSON)},
iterator.Map(req.Files, func(src *UploadingObject) (*http2.IterMultiPartFile, error) {
return &http2.IterMultiPartFile{
@ -142,6 +145,66 @@ func (c *PackageService) CreateUpload(req PackageCreateUpload) (*PackageCreateUp
return nil, codeResp.ToError()
}
const PackageDownloadPath = "/package/download"
type PackageDownload struct {
PackageID clitypes.PackageID `url:"packageID" form:"packageID" binding:"required"`
Prefix string `url:"prefix" form:"prefix"`
NewPrefix *string `url:"newPrefix,omitempty" form:"newPrefix"`
Zip bool `url:"zip,omitempty" form:"zip"`
}
func (r *PackageDownload) MakeParam() *sdks.RequestParam {
return sdks.MakeQueryParam(http.MethodGet, PackageDownloadPath, r)
}
type DownloadingPackage struct {
Name string
File io.ReadCloser
}
func (c *PackageService) Download(req PackageDownload) (*DownloadingPackage, error) {
url, err := url.JoinPath(c.cfg.EndPoint, "v1")
if err != nil {
return nil, err
}
httpReq, err := req.MakeParam().MakeRequest(url)
if err != nil {
return nil, err
}
resp, err := c.Client.httpCli.Do(httpReq)
if err != nil {
return nil, err
}
contType := resp.Header.Get("Content-Type")
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("response status code: %d", resp.StatusCode)
}
if strings.Contains(contType, http2.ContentTypeJSON) {
var codeResp response[any]
if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil {
return nil, fmt.Errorf("parsing response: %w", err)
}
return nil, codeResp.ToError()
}
_, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition"))
if err != nil {
return nil, fmt.Errorf("parsing content disposition: %w", err)
}
return &DownloadingPackage{
Name: params["filename"],
File: resp.Body,
}, nil
}
const PackageDeletePath = "/package/delete"
type PackageDelete struct {

8
client/sdk/api/v1/storage_test.go
View File

@ -48,12 +48,12 @@ func Test_PackageGet(t *testing.T) {
})
So(err, ShouldBeNil)
getResp, err := cli.Package().Get(PackageGetReq{
getResp, err := cli.Package().Get(PackageGet{
PackageID: createResp.Package.PackageID,
})
So(err, ShouldBeNil)
So(getResp.PackageID, ShouldEqual, createResp.Package.PackageID)
So(getResp.Package.PackageID, ShouldEqual, createResp.Package.PackageID)
So(getResp.Package.Name, ShouldEqual, pkgName)
err = cli.Package().Delete(PackageDelete{
@ -266,12 +266,12 @@ func Test_Sign(t *testing.T) {
})
So(err, ShouldBeNil)
getResp, err := cli.Package().Get(PackageGetReq{
getResp, err := cli.Package().Get(PackageGet{
PackageID: createResp.Package.PackageID,
})
So(err, ShouldBeNil)
So(getResp.PackageID, ShouldEqual, createResp.Package.PackageID)
So(getResp.Package.PackageID, ShouldEqual, createResp.Package.PackageID)
So(getResp.Package.Name, ShouldEqual, pkgName)
err = cli.Package().Delete(PackageDelete{

3
client/sdk/api/v1/utils.go
View File

@ -105,7 +105,7 @@ func calcSha256(body sdks.RequestBody) string {
}
}
func PostMultiPart(cfg *api.Config, url string, info any, files http2.MultiPartFileIterator) (*http.Response, error) {
func PostMultiPart(cfg *api.Config, cli *http.Client, url string, info any, files http2.MultiPartFileIterator) (*http.Response, error) {
req, err := http.NewRequest(http.MethodPost, url, nil)
if err != nil {
return nil, err
@ -158,7 +158,6 @@ func PostMultiPart(cfg *api.Config, url string, info any, files http2.MultiPartF
req.Body = pr
cli := http.Client{}
resp, err := cli.Do(req)
if err != nil {
return nil, err

30
client/types/redundancy.go
View File

@ -21,11 +21,14 @@ var RedundancyUnion = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTyp
)), "type")
type NoneRedundancy struct {
Redundancy `json:"-"`
serder.Metadata `union:"none"`
Type string `json:"type"`
}
func (r *NoneRedundancy) GetRedundancyType() string {
return "none"
}
func NewNoneRedundancy() *NoneRedundancy {
return &NoneRedundancy{
Type: "none",
@ -35,12 +38,15 @@ func NewNoneRedundancy() *NoneRedundancy {
var DefaultRepRedundancy = *NewRepRedundancy(2)
type RepRedundancy struct {
Redundancy `json:"-"`
serder.Metadata `union:"rep"`
Type string `json:"type"`
RepCount int `json:"repCount"`
}
func (r *RepRedundancy) GetRedundancyType() string {
return "rep"
}
func NewRepRedundancy(repCount int) *RepRedundancy {
return &RepRedundancy{
Type: "rep",
@ -51,7 +57,6 @@ func NewRepRedundancy(repCount int) *RepRedundancy {
var DefaultECRedundancy = *NewECRedundancy(2, 3, 1024*1024*5)
type ECRedundancy struct {
Redundancy `json:"-"`
serder.Metadata `union:"ec"`
Type string `json:"type"`
K int `json:"k"`
@ -59,6 +64,10 @@ type ECRedundancy struct {
ChunkSize int `json:"chunkSize"`
}
func (b *ECRedundancy) GetRedundancyType() string {
return "ec"
}
func NewECRedundancy(k int, n int, chunkSize int) *ECRedundancy {
return &ECRedundancy{
Type: "ec",
@ -75,7 +84,6 @@ func (b *ECRedundancy) StripSize() int64 {
var DefaultLRCRedundancy = *NewLRCRedundancy(2, 4, []int{2}, 1024*1024*5)
type LRCRedundancy struct {
Redundancy `json:"-"`
serder.Metadata `union:"lrc"`
Type string `json:"type"`
K int `json:"k"`
@ -84,6 +92,10 @@ type LRCRedundancy struct {
ChunkSize int `json:"chunkSize"`
}
func (b *LRCRedundancy) GetRedundancyType() string {
return "lrc"
}
func NewLRCRedundancy(k int, n int, groups []int, chunkSize int) *LRCRedundancy {
return &LRCRedundancy{
Type: "lrc",
@ -132,12 +144,15 @@ func (b *LRCRedundancy) GetGroupElements(grp int) []int {
}
type SegmentRedundancy struct {
Redundancy `json:"-"`
serder.Metadata `union:"segment"`
Type string `json:"type"`
Segments []int64 `json:"segments"` // 每一段的大小
}
func (r *SegmentRedundancy) GetRedundancyType() string {
return "segment"
}
func NewSegmentRedundancy(totalSize int64, segmentCount int) *SegmentRedundancy {
return &SegmentRedundancy{
Type: "segment",
@ -201,11 +216,14 @@ func (b *SegmentRedundancy) CalcSegmentRange(start int64, end *int64) (segIdxSta
}
type MultipartUploadRedundancy struct {
Redundancy `json:"-"`
serder.Metadata `union:"multipartUpload"`
Type string `json:"type"`
}
func (r *MultipartUploadRedundancy) GetRedundancyType() string {
return "multipartUpload"
}
func NewMultipartUploadRedundancy() *MultipartUploadRedundancy {
return &MultipartUploadRedundancy{
Type: "multipartUpload",

2
common/pkgs/storage/local/dir_reader.go
View File

@ -59,7 +59,7 @@ func (r *DirReader) Next() (types.DirEntry, error) {
if entry.entry.IsDir() {
es, err := os.ReadDir(filepath.Join(r.absRootPath, entry.dir.JoinOSPath(), entry.entry.Name()))
if err != nil {
return types.DirEntry{}, nil
return types.DirEntry{}, err
}
// 多个entry对象共享同一个JPath对象，但因为不会修改JPath，所以没问题

1
go.mod
View File

@ -54,6 +54,7 @@ require (
github.com/cloudwego/base64x v0.1.4 // indirect
github.com/cloudwego/iasm v0.2.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/fatih/color v1.18.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.6 // indirect
github.com/goccy/go-json v0.10.3 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect

2
go.sum
View File

@ -56,6 +56,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc=
github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc=
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=

6
jcsctl/cmd/all/all.go
View File

@ -2,5 +2,11 @@ package all
import (
_ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/bucket"
_ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/geto"
_ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/getp"
_ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/ls"
_ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/package"
_ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/puto"
_ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/putp"
_ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/userspace"
)

3
jcsctl/cmd/bucket/bucket.go
View File

@ -6,7 +6,8 @@ import (
)
var BucketCmd = &cobra.Command{
Use: "bucket",
Use: "bucket",
Aliases: []string{"bkt"},
}
func init() {

49
jcsctl/cmd/bucket/ls.go
View File

@ -1,49 +0,0 @@
package bucket
import (
"fmt"
"github.com/jedib0t/go-pretty/v6/table"
"github.com/spf13/cobra"
"gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
"gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd"
)
func init() {
var opt lsOpt
cmd := cobra.Command{
Use: "ls",
Run: func(c *cobra.Command, args []string) {
ctx := cmd.GetCmdCtx(c)
ls(c, ctx, opt)
},
}
cmd.Flags().BoolVarP(&opt.Long, "", "l", false, "listing in long format")
BucketCmd.AddCommand(&cmd)
}
type lsOpt struct {
Long bool
}
func ls(c *cobra.Command, ctx *cmd.CommandContext, opt lsOpt) {
resp, err := ctx.Client.Bucket().ListAll(api.BucketListAll{})
if err != nil {
cmd.ErrorExitln(err.Error())
}
if opt.Long {
fmt.Printf("total: %d\n", len(resp.Buckets))
tb := table.NewWriter()
tb.AppendHeader(table.Row{"Bucket ID", "Name", "Create Time"})
for _, b := range resp.Buckets {
tb.AppendRow(table.Row{b.BucketID, b.Name, b.CreateTime})
}
fmt.Println(tb.Render())
} else {
for _, b := range resp.Buckets {
fmt.Println(b.Name)
}
}
}

6
jcsctl/cmd/cmd.go
View File

@ -74,7 +74,7 @@ func RootExecute() {
}
if endpoint == "" {
endpoint = "https://127.0.0.1:8890"
endpoint = "https://127.0.0.1:7890"
}
cli := cliapi.NewClient(api.Config{
@ -112,7 +112,3 @@ func searchCertDir() string {
return ""
}
func loadCert() {
}

143
jcsctl/cmd/geto/geto.go Normal file
View File

@ -0,0 +1,143 @@
package geto
import (
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"time"
"github.com/inhies/go-bytesize"
"github.com/spf13/cobra"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd"
)
func init() {
var opt option
c := &cobra.Command{
Use: "geto <bucket_name>/<package_name>:<object_path> <local_dir>",
Short: "download object to local disk",
Args: cobra.ExactArgs(2),
RunE: func(c *cobra.Command, args []string) error {
ctx := cmd.GetCmdCtx(c)
return geto(c, ctx, opt, args)
},
}
c.Flags().BoolVar(&opt.UseID, "id", false, "treat first argument as object id")
c.Flags().Int64Var(&opt.Offset, "offset", 0, "offset of object to download")
c.Flags().Int64Var(&opt.Length, "length", 0, "length of object to download")
c.Flags().Int64Var(&opt.Seek, "seek", 0, "seek position when save to local file, if set, will not truncate local file")
c.Flags().StringVarP(&opt.Output, "output", "o", "", "output file name")
cmd.RootCmd.AddCommand(c)
}
type option struct {
UseID bool
Offset int64
Length int64
Seek int64
Output string
}
func geto(c *cobra.Command, ctx *cmd.CommandContext, opt option, args []string) error {
var obj clitypes.Object
if opt.UseID {
id, err := strconv.ParseInt(args[0], 10, 64)
if err != nil {
return fmt.Errorf("invalid object id: %v", err)
}
resp, err := ctx.Client.Object().ListByIDs(cliapi.ObjectListByIDs{
ObjectIDs: []clitypes.ObjectID{clitypes.ObjectID(id)},
})
if err != nil {
return fmt.Errorf("list objects by ids: %v", err)
}
if resp.Objects[0] == nil {
return fmt.Errorf("object not found")
}
obj = *resp.Objects[0]
} else {
bkt, pkg, objPath, ok := cmd.SplitObjectPath(args[0])
if !ok {
return fmt.Errorf("invalid object path")
}
pkgResp, err := ctx.Client.Package().GetByFullName(cliapi.PackageGetByFullName{
BucketName: bkt,
PackageName: pkg,
})
if err != nil {
return fmt.Errorf("get package by name: %v", err)
}
objResp, err := ctx.Client.Object().ListByPath(cliapi.ObjectListByPath{
PackageID: pkgResp.Package.PackageID,
Path: objPath,
})
if err != nil {
return fmt.Errorf("list objects by path: %v", err)
}
if len(objResp.Objects) != 1 {
return fmt.Errorf("object not found")
}
obj = objResp.Objects[0]
}
filePath := args[1]
if opt.Output != "" {
filePath = filepath.Join(filePath, opt.Output)
} else {
filePath = filepath.Join(filePath, clitypes.BaseName(obj.Path))
}
flag := os.O_CREATE | os.O_WRONLY
if !c.Flags().Changed("seek") {
flag |= os.O_TRUNC
}
file, err := os.OpenFile(filePath, flag, 0644)
if err != nil {
return fmt.Errorf("open file: %v", err)
}
defer file.Close()
if opt.Seek != 0 {
if _, err := file.Seek(opt.Seek, 0); err != nil {
return fmt.Errorf("seek file: %v", err)
}
}
fmt.Printf("%v\n", filePath)
var len *int64
if c.Flags().Changed("length") {
len = &opt.Length
}
resp, err := ctx.Client.Object().Download(cliapi.ObjectDownload{
ObjectID: obj.ObjectID,
Offset: opt.Offset,
Length: len,
})
if err != nil {
return fmt.Errorf("download object: %v", err)
}
startTime := time.Now()
n, err := io.Copy(file, resp.File)
if err != nil {
return fmt.Errorf("copy object to file: %v", err)
}
dt := time.Since(startTime)
fmt.Printf("size: %v, time: %v, speed: %v/s\n", bytesize.ByteSize(n), dt, bytesize.ByteSize(int64(float64(n)/dt.Seconds())))
return nil
}

172
jcsctl/cmd/getp/getp.go Normal file
View File

@ -0,0 +1,172 @@
package getp
import (
"archive/tar"
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/inhies/go-bytesize"
"github.com/spf13/cobra"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd"
)
func init() {
var opt option
c := &cobra.Command{
Use: "getp <bucket_name>/<package_name> <local_path>",
Short: "download package all files to local disk",
Args: cobra.ExactArgs(2),
RunE: func(c *cobra.Command, args []string) error {
ctx := cmd.GetCmdCtx(c)
return getp(c, ctx, opt, args)
},
}
c.Flags().BoolVar(&opt.UseID, "id", false, "treat first argument as package id")
c.Flags().StringVar(&opt.Prefix, "prefix", "", "download objects with this prefix")
c.Flags().StringVar(&opt.NewPrefix, "new", "", "replace prefix specified by --prefix with this prefix")
c.Flags().BoolVar(&opt.Zip, "zip", false, "download as zip file")
c.Flags().StringVarP(&opt.Output, "output", "o", "", "output zip file name")
cmd.RootCmd.AddCommand(c)
}
type option struct {
UseID bool
Prefix string
NewPrefix string
Zip bool
Output string
}
func getp(c *cobra.Command, ctx *cmd.CommandContext, opt option, args []string) error {
var pkgID clitypes.PackageID
if opt.UseID {
id, err := strconv.ParseInt(args[0], 10, 64)
if err != nil {
return fmt.Errorf("invalid package id")
}
pkgID = clitypes.PackageID(id)
} else {
comps := strings.Split(args[0], "/")
if len(comps) != 2 {
return fmt.Errorf("invalid package name")
}
resp, err := ctx.Client.Package().GetByFullName(cliapi.PackageGetByFullName{
BucketName: comps[0],
PackageName: comps[1],
})
if err != nil {
return fmt.Errorf("get package by name: %w", err)
}
pkgID = resp.Package.PackageID
}
info, err := os.Stat(args[1])
if err != nil {
return err
}
if !info.IsDir() {
return fmt.Errorf("local path should be a directory")
}
req := cliapi.PackageDownload{
PackageID: pkgID,
Prefix: opt.Prefix,
}
if c.Flags().Changed("new") {
req.NewPrefix = &opt.NewPrefix
}
if opt.Zip {
req.Zip = true
}
downResp, err := ctx.Client.Package().Download(req)
if err != nil {
return fmt.Errorf("download package: %w", err)
}
defer downResp.File.Close()
if opt.Zip {
fileName := downResp.Name
if opt.Output != "" {
fileName = opt.Output
}
localFilePath := filepath.Join(args[1], fileName)
file, err := os.OpenFile(localFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
return err
}
fmt.Printf("%v\n", localFilePath)
startTime := time.Now()
n, err := io.Copy(file, downResp.File)
if err != nil {
return err
}
dt := time.Since(startTime)
fmt.Printf("size: %v, time: %v, speed: %v/s\n", bytesize.ByteSize(n), dt, bytesize.ByteSize(int64(float64(n)/dt.Seconds())))
return nil
}
startTime := time.Now()
totalSize := int64(0)
fileCnt := 0
tr := tar.NewReader(downResp.File)
for {
header, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
localPath := filepath.Join(args[1], header.Name)
fmt.Printf("%v", localPath)
dir := filepath.Dir(localPath)
err = os.MkdirAll(dir, 0755)
if err != nil {
fmt.Printf("\tx")
return err
}
fileStartTime := time.Now()
file, err := os.OpenFile(localPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
fmt.Printf("\tx")
return err
}
_, err = io.Copy(file, tr)
if err != nil {
fmt.Printf("\tx")
return err
}
dt := time.Since(fileStartTime)
fmt.Printf("\t%v\t%v\n", bytesize.ByteSize(header.Size), dt)
fileCnt++
totalSize += header.Size
}
dt := time.Since(startTime)
fmt.Printf("%v files, total size: %v, time: %v, speed: %v/s\n", fileCnt, bytesize.ByteSize(totalSize), dt, bytesize.ByteSize(int64(float64(totalSize)/dt.Seconds())))
return nil
}

76
jcsctl/cmd/ls/ls.go Normal file
View File

@ -0,0 +1,76 @@
package ls
import (
"fmt"
"strings"
"github.com/spf13/cobra"
"gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd"
)
func init() {
var opt option
c := &cobra.Command{
Use: "ls [bucket_name]/[package_name]:[object_path]",
Short: "download package all files to local disk",
Args: cobra.MaximumNArgs(1),
RunE: func(c *cobra.Command, args []string) error {
ctx := cmd.GetCmdCtx(c)
return ls(c, ctx, opt, args)
},
}
c.Flags().Int64Var(&opt.BucketID, "bid", 0, "bucket id, if set, you should not set any path")
c.Flags().Int64Var(&opt.PackageID, "pid", 0, "package id, if set, you should not set <bucket_name>/<package_name>")
c.Flags().BoolVarP(&opt.Recursive, "recursive", "r", false, "list all files in package recursively, only valid when list in a package")
c.Flags().BoolVarP(&opt.Long, "long", "l", false, "show more details")
cmd.RootCmd.AddCommand(c)
}
type option struct {
Long bool
BucketID int64
PackageID int64
Recursive bool
}
func ls(c *cobra.Command, ctx *cmd.CommandContext, opt option, args []string) error {
if opt.PackageID != 0 {
objPath := ""
if len(args) > 0 {
objPath = args[0]
}
return lsObject(ctx, opt, "", "", objPath)
}
if opt.BucketID != 0 {
if len(args) > 0 {
return fmt.Errorf("list package objects is not supported when use bucket id")
}
return lsPackage(ctx, opt, "")
}
if len(args) == 0 {
return lsBucket(ctx, opt)
}
comps := strings.SplitN(args[0], ":", 2)
if len(comps) > 1 {
objPath := comps[1]
comps = strings.SplitN(comps[0], "/", 2)
if len(comps) != 2 {
return fmt.Errorf("invalid path format, should be <bucket_name>/<package_name>:<object_path>")
}
return lsObject(ctx, opt, comps[0], comps[1], objPath)
}
comps = strings.SplitN(args[0], "/", 2)
if len(comps) == 1 {
return lsPackage(ctx, opt, comps[0])
}
return lsObject(ctx, opt, comps[0], comps[1], "")
}

33
jcsctl/cmd/ls/ls_bucket.go Normal file
View File

@ -0,0 +1,33 @@
package ls
import (
"fmt"
"github.com/jedib0t/go-pretty/v6/table"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
"gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd"
)
func lsBucket(ctx *cmd.CommandContext, opt option) error {
resp, err := ctx.Client.Bucket().ListAll(cliapi.BucketListAll{})
if err != nil {
return err
}
if opt.Long {
fmt.Printf("total: %d buckets\n", len(resp.Buckets))
tb := table.NewWriter()
tb.AppendHeader(table.Row{"Bucket ID", "Name", "Create Time"})
for _, b := range resp.Buckets {
tb.AppendRow(table.Row{b.BucketID, b.Name, b.CreateTime})
}
fmt.Println(tb.Render())
} else {
for _, b := range resp.Buckets {
fmt.Println(b.Name)
}
}
return nil
}

92
jcsctl/cmd/ls/ls_object.go Normal file
View File

@ -0,0 +1,92 @@
package ls
import (
"fmt"
"github.com/jedib0t/go-pretty/v6/table"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd"
)
func lsObject(ctx *cmd.CommandContext, opt option, bktName string, pkgName string, objPath string) error {
var pkgID clitypes.PackageID
if opt.PackageID != 0 {
pkgID = clitypes.PackageID(opt.PackageID)
} else {
resp, err := ctx.Client.Package().GetByFullName(cliapi.PackageGetByFullName{
BucketName: bktName,
PackageName: pkgName,
})
if err != nil {
return fmt.Errorf("get package %v: %w", pkgName, err)
}
pkgID = resp.Package.PackageID
}
var objs []clitypes.Object
var commonPrefixes []string
req := cliapi.ObjectListByPath{
PackageID: pkgID,
Path: objPath,
IsPrefix: true,
}
if !opt.Recursive {
req.NoRecursive = true
}
var nextConToken string
for {
req.ContinuationToken = nextConToken
resp, err := ctx.Client.Object().ListByPath(req)
if err != nil {
return fmt.Errorf("list objects: %w", err)
}
objs = append(objs, resp.Objects...)
commonPrefixes = append(commonPrefixes, resp.CommonPrefixes...)
if !resp.IsTruncated {
break
}
nextConToken = resp.NextContinuationToken
}
if opt.Long {
fmt.Printf("total %d objects, %d common prefixes in package %v\n", len(objs), len(commonPrefixes), pkgName)
}
if len(commonPrefixes) > 0 {
for _, prefix := range commonPrefixes {
fmt.Printf("%s\n", prefix)
}
fmt.Printf("\n")
}
if len(objs) > 0 {
if opt.Long {
tb := table.NewWriter()
tb.AppendHeader(table.Row{"ID", "Path", "Size", "Hash", "Redundancy", "Create Time", "Update Time"})
for _, obj := range objs {
tb.AppendRow(table.Row{
obj.ObjectID,
obj.Path,
obj.Size,
obj.FileHash,
obj.Redundancy.GetRedundancyType(),
obj.CreateTime,
obj.UpdateTime,
})
}
fmt.Println(tb.Render())
} else {
for _, obj := range objs {
fmt.Printf("%s\n", obj.Path)
}
}
}
return nil
}

48
jcsctl/cmd/ls/ls_package.go Normal file
View File

@ -0,0 +1,48 @@
package ls
import (
"fmt"
"github.com/jedib0t/go-pretty/v6/table"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd"
)
func lsPackage(ctx *cmd.CommandContext, opt option, bktName string) error {
var bktID clitypes.BucketID
if opt.BucketID == 0 {
bktResp, err := ctx.Client.Bucket().GetByName(cliapi.BucketGetByName{
Name: bktName,
})
if err != nil {
return fmt.Errorf("find bucket %v: %w", bktName, err)
}
bktID = bktResp.Bucket.BucketID
} else {
bktID = clitypes.BucketID(opt.BucketID)
}
pkgResp, err := ctx.Client.Package().ListBucketPackages(cliapi.PackageListBucketPackages{
BucketID: bktID,
})
if err != nil {
return fmt.Errorf("list packages in bucket %v: %w", bktName, err)
}
if opt.Long {
fmt.Printf("total %v:\n", len(pkgResp.Packages))
tb := table.NewWriter()
tb.AppendHeader(table.Row{"Package ID", "Bucket ID", "Name", "CreateTime"})
for _, p := range pkgResp.Packages {
tb.AppendRow(table.Row{p.PackageID, p.BucketID, p.Name, p.CreateTime})
}
fmt.Println(tb.Render())
} else {
for _, p := range pkgResp.Packages {
fmt.Println(p.Name)
}
}
return nil
}

15
jcsctl/cmd/object/object.go Normal file
View File

@ -0,0 +1,15 @@
package object
import (
"github.com/spf13/cobra"
"gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd"
)
var ObjectCmd = &cobra.Command{
Use: "object",
Aliases: []string{"obj"},
}
func init() {
cmd.RootCmd.AddCommand(ObjectCmd)
}

66
jcsctl/cmd/package/new.go Normal file
View File

@ -0,0 +1,66 @@
package pkg
import (
"fmt"
"strings"
"github.com/spf13/cobra"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd"
)
func init() {
var opt newOpt
cmd := cobra.Command{
Use: "new <bucket_name>/<package_name>",
Args: cobra.ExactArgs(1),
RunE: func(c *cobra.Command, args []string) error {
ctx := cmd.GetCmdCtx(c)
return new(c, ctx, opt, args)
},
}
cmd.Flags().Int64Var(&opt.BucketID, "bid", 0, "set bucket id, if set, you should not set bucket name")
PackageCmd.AddCommand(&cmd)
}
type newOpt struct {
BucketID int64
}
func new(c *cobra.Command, ctx *cmd.CommandContext, opt newOpt, args []string) error {
if opt.BucketID != 0 {
resp, err := ctx.Client.Package().Create(cliapi.PackageCreate{
BucketID: clitypes.BucketID(opt.BucketID),
Name: args[0],
})
if err != nil {
return err
}
printOnePackage(resp.Package)
return nil
}
comps := strings.Split(args[0], "/")
if len(comps) != 2 {
return fmt.Errorf("invalid package name")
}
bktResp, err := ctx.Client.Bucket().GetByName(cliapi.BucketGetByName{
Name: comps[0],
})
if err != nil {
return fmt.Errorf("get bucket by name %v: %v", comps[0], err)
}
resp, err := ctx.Client.Package().Create(cliapi.PackageCreate{
BucketID: bktResp.Bucket.BucketID,
Name: comps[1],
})
if err != nil {
return fmt.Errorf("create package %v: %v", args[0], err)
}
printOnePackage(resp.Package)
return nil
}

15
jcsctl/cmd/package/package.go Normal file
View File

@ -0,0 +1,15 @@
package pkg
import (
"github.com/spf13/cobra"
"gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd"
)
var PackageCmd = &cobra.Command{
Use: "package",
Aliases: []string{"pkg"},
}
func init() {
cmd.RootCmd.AddCommand(PackageCmd)
}

15
jcsctl/cmd/package/utils.go Normal file
View File

@ -0,0 +1,15 @@
package pkg
import (
"fmt"
"github.com/jedib0t/go-pretty/v6/table"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
)
func printOnePackage(pkg clitypes.Package) {
tb := table.NewWriter()
tb.AppendHeader(table.Row{"Package ID", "Bucket ID", "Name", "CreateTime"})
tb.AppendRow(table.Row{pkg.PackageID, pkg.BucketID, pkg.Name, pkg.CreateTime})
fmt.Println(tb.Render())
}

109
jcsctl/cmd/puto/puto.go Normal file
View File

@ -0,0 +1,109 @@
package puto
import (
"fmt"
"os"
"strconv"
"time"
"github.com/inhies/go-bytesize"
"github.com/spf13/cobra"
"gitlink.org.cn/cloudream/common/pkgs/iterator"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd"
)
func init() {
var opt option
c := &cobra.Command{
Use: "puto <local_path> <bucket_name>/<package_name>:<object_path>",
Short: "upload local file as a object",
Args: cobra.ExactArgs(2),
RunE: func(c *cobra.Command, args []string) error {
ctx := cmd.GetCmdCtx(c)
return puto(c, ctx, opt, args)
},
}
c.Flags().BoolVar(&opt.UseID, "id", false, "treat the second argument as object id")
cmd.RootCmd.AddCommand(c)
}
type option struct {
UseID bool
}
func puto(c *cobra.Command, ctx *cmd.CommandContext, opt option, args []string) error {
var pkgID clitypes.PackageID
var objPath string
if opt.UseID {
id, err := strconv.ParseInt(args[1], 10, 64)
if err != nil {
return fmt.Errorf("invalid object id: %v", err)
}
resp, err := ctx.Client.Object().ListByIDs(cliapi.ObjectListByIDs{
ObjectIDs: []clitypes.ObjectID{clitypes.ObjectID(id)},
})
if err != nil {
return fmt.Errorf("list objects by ids: %v", err)
}
if resp.Objects[0] == nil {
return fmt.Errorf("object not found")
}
pkgID = resp.Objects[0].PackageID
objPath = resp.Objects[0].Path
} else {
bkt, pkg, objPath2, ok := cmd.SplitObjectPath(args[1])
if !ok {
return fmt.Errorf("invalid object path")
}
pkgResp, err := ctx.Client.Package().GetByFullName(cliapi.PackageGetByFullName{
BucketName: bkt,
PackageName: pkg,
})
if err != nil {
return fmt.Errorf("get package by name: %v", err)
}
pkgID = pkgResp.Package.PackageID
objPath = objPath2
}
file, err := os.Open(args[0])
if err != nil {
return fmt.Errorf("open file: %v", err)
}
defer file.Close()
info, err := file.Stat()
if err != nil {
return err
}
fmt.Printf("%v\n", objPath)
startTime := time.Now()
_, err = ctx.Client.Object().Upload(cliapi.ObjectUpload{
ObjectUploadInfo: cliapi.ObjectUploadInfo{
PackageID: pkgID,
},
Files: iterator.Array(&cliapi.UploadingObject{
Path: objPath,
File: file,
}),
})
if err != nil {
return fmt.Errorf("upload file %v: %w", objPath, err)
}
dt := time.Since(startTime)
fmt.Printf("size: %v, time: %v, speed: %v/s\n", bytesize.ByteSize(info.Size()), dt, bytesize.ByteSize(int64(float64(info.Size())/dt.Seconds())))
return nil
}

106
jcsctl/cmd/putp/file_iterator.go Normal file
View File

@ -0,0 +1,106 @@
package putp
import (
"fmt"
"os"
"path/filepath"
"time"
"github.com/inhies/go-bytesize"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/iterator"
)
type FileIterator struct {
absRootPath string
jpathRoot clitypes.JPath
init bool
curEntries []dirEntry
lastStartTime time.Time
totalSize int64
fileCount int
}
func (i *FileIterator) MoveNext() (*cliapi.UploadingObject, error) {
if !i.init {
es, err := os.ReadDir(i.absRootPath)
if err != nil {
return nil, err
}
for _, e := range es {
i.curEntries = append(i.curEntries, dirEntry{
dir: clitypes.JPath{},
entry: e,
})
}
i.init = true
}
for {
if len(i.curEntries) == 0 {
return nil, iterator.ErrNoMoreItem
}
entry := i.curEntries[0]
i.curEntries = i.curEntries[1:]
if entry.entry.IsDir() {
es, err := os.ReadDir(filepath.Join(i.absRootPath, entry.dir.JoinOSPath(), entry.entry.Name()))
if err != nil {
return nil, err
}
// 多个entry对象共享同一个JPath对象，但因为不会修改JPath，所以没问题
dir := entry.dir.Clone()
dir.Push(entry.entry.Name())
for _, e := range es {
i.curEntries = append(i.curEntries, dirEntry{
dir: dir,
entry: e,
})
}
continue
}
file, err := os.Open(filepath.Join(i.absRootPath, entry.dir.JoinOSPath(), entry.entry.Name()))
if err != nil {
return nil, err
}
info, err := file.Stat()
if err != nil {
return nil, err
}
i.totalSize += info.Size()
i.fileCount++
jpath := i.jpathRoot.ConcatNew(entry.dir)
path := jpath.ConcatCompsNew(entry.entry.Name()).String()
now := time.Now()
if !i.lastStartTime.IsZero() {
dt := now.Sub(i.lastStartTime)
fmt.Printf("\t%v\n", dt)
}
i.lastStartTime = now
fmt.Printf("%v\t%v", path, bytesize.ByteSize(info.Size()))
return &cliapi.UploadingObject{
Path: path,
File: file,
}, nil
}
}
func (i *FileIterator) Close() {
}
type dirEntry struct {
dir clitypes.JPath
entry os.DirEntry
}

175
jcsctl/cmd/putp/putp.go Normal file
View File

@ -0,0 +1,175 @@
package putp
import (
"fmt"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/inhies/go-bytesize"
"github.com/spf13/cobra"
"gitlink.org.cn/cloudream/common/pkgs/iterator"
"gitlink.org.cn/cloudream/common/sdks"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/ecode"
"gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd"
)
func init() {
var opt option
c := &cobra.Command{
Use: "putp <local_path> <bucket_name>/<package_name>",
Short: "upload local files to a package",
Args: cobra.ExactArgs(2),
RunE: func(c *cobra.Command, args []string) error {
ctx := cmd.GetCmdCtx(c)
return putp(c, ctx, opt, args)
},
}
c.Flags().BoolVar(&opt.UseID, "id", false, "treat the second argument as package id")
c.Flags().StringVar(&opt.Prefix, "prefix", "", "add prefix to every uploaded file")
c.Flags().BoolVar(&opt.Create, "create", false, "create package if not exists")
cmd.RootCmd.AddCommand(c)
}
type option struct {
UseID bool
Prefix string
Create bool
}
func putp(c *cobra.Command, ctx *cmd.CommandContext, opt option, args []string) error {
absLocal, err := filepath.Abs(args[0])
if err != nil {
return err
}
local, err := os.Stat(absLocal)
if err != nil {
return err
}
var pkgID clitypes.PackageID
if opt.UseID {
id, err := strconv.ParseInt(args[1], 10, 64)
if err != nil {
return err
}
pkgID = clitypes.PackageID(id)
_, err = ctx.Client.Package().Get(cliapi.PackageGet{
PackageID: pkgID,
})
if err != nil {
return err
}
} else {
comps := strings.Split(args[1], "/")
if len(comps) != 2 {
return fmt.Errorf("invalid package name")
}
pkg, err := ctx.Client.Package().GetByFullName(cliapi.PackageGetByFullName{
BucketName: comps[0],
PackageName: comps[1],
})
if err != nil {
if !sdks.IsErrorCode(err, string(ecode.DataNotFound)) {
return err
}
if !opt.Create {
return fmt.Errorf("package not found")
}
bkt, err := ctx.Client.Bucket().GetByName(cliapi.BucketGetByName{
Name: comps[0],
})
if err != nil {
return fmt.Errorf("get bucket %v: %w", comps[0], err)
}
cpkg, err := ctx.Client.Package().Create(cliapi.PackageCreate{
BucketID: bkt.Bucket.BucketID,
Name: comps[1],
})
if err != nil {
return fmt.Errorf("create package %v: %w", args[1], err)
}
pkgID = cpkg.Package.PackageID
} else {
pkgID = pkg.Package.PackageID
}
}
if !local.IsDir() {
file, err := os.Open(absLocal)
if err != nil {
return err
}
defer file.Close()
info, err := file.Stat()
if err != nil {
return err
}
pat := filepath.Base(absLocal)
if opt.Prefix != "" {
pat = path.Join(opt.Prefix, pat)
}
fmt.Printf("%v\n", pat)
startTime := time.Now()
_, err = ctx.Client.Object().Upload(cliapi.ObjectUpload{
ObjectUploadInfo: cliapi.ObjectUploadInfo{
PackageID: pkgID,
},
Files: iterator.Array(&cliapi.UploadingObject{
Path: pat,
File: file,
}),
})
if err != nil {
return fmt.Errorf("upload file %v: %w", pat, err)
}
dt := time.Since(startTime)
fmt.Printf("size: %v, time: %v, speed: %v/s\n", bytesize.ByteSize(info.Size()), dt, bytesize.ByteSize(int64(float64(info.Size())/dt.Seconds())))
return nil
}
iter := &FileIterator{
absRootPath: absLocal,
jpathRoot: clitypes.PathFromJcsPathString(opt.Prefix),
}
startTime := time.Now()
_, err = ctx.Client.Object().Upload(cliapi.ObjectUpload{
ObjectUploadInfo: cliapi.ObjectUploadInfo{
PackageID: pkgID,
},
Files: iter,
})
if err != nil {
if !iter.lastStartTime.IsZero() {
fmt.Printf("\tx\n")
}
return fmt.Errorf("upload files: %w", err)
}
dt := time.Since(startTime)
if !iter.lastStartTime.IsZero() {
fileDt := time.Since(iter.lastStartTime)
fmt.Printf("\t%v\n", fileDt)
}
fmt.Printf("%v files, total size: %v, time: %v, speed: %v/s\n", iter.fileCount, bytesize.ByteSize(iter.totalSize), dt, bytesize.ByteSize(int64(float64(iter.totalSize)/dt.Seconds())))
return nil
}

24
jcsctl/cmd/userspace/ls.go
View File

@ -16,9 +16,9 @@ func init() {
cmd := cobra.Command{
Use: "ls",
Args: cobra.MaximumNArgs(1),
Run: func(c *cobra.Command, args []string) {
RunE: func(c *cobra.Command, args []string) error {
ctx := cmd.GetCmdCtx(c)
ls(c, ctx, opt, args)
return ls(c, ctx, opt, args)
},
}
@ -32,13 +32,12 @@ type lsOpt struct {
ShowPassword bool
}
func ls(c *cobra.Command, ctx *cmd.CommandContext, opt lsOpt, args []string) {
func ls(c *cobra.Command, ctx *cmd.CommandContext, opt lsOpt, args []string) error {
// 仅ls无参数
if len(args) == 0 {
resp, err := ctx.Client.UserSpaceGetAll()
if err != nil {
cmd.ErrorExitln(err.Error())
return
return err
}
fmt.Printf("total: %d\n", len(resp.UserSpaces))
@ -48,7 +47,7 @@ func ls(c *cobra.Command, ctx *cmd.CommandContext, opt lsOpt, args []string) {
tb.AppendRow(table.Row{userSpace.UserSpaceID, userSpace.Name, userSpace.Storage.GetStorageType()})
}
fmt.Println(tb.Render())
return
return nil
}
searchKey := args[0]
@ -56,16 +55,14 @@ func ls(c *cobra.Command, ctx *cmd.CommandContext, opt lsOpt, args []string) {
if opt.ByID {
id, err := strconv.Atoi(searchKey)
if err != nil {
cmd.ErrorExitln("ID必须是数字")
return
return fmt.Errorf("ID必须是数字")
}
result, err := ctx.Client.UserSpaceGet(cliapi.UserSpaceGet{
UserSpaceID: clitypes.UserSpaceID(id),
})
if err != nil {
cmd.ErrorExitln(err.Error())
return
return err
}
userSpace = &result.UserSpace
@ -74,15 +71,13 @@ func ls(c *cobra.Command, ctx *cmd.CommandContext, opt lsOpt, args []string) {
Name: searchKey,
})
if err != nil {
cmd.ErrorExitln(err.Error())
return
return err
}
userSpace = &result.UserSpace
}
if userSpace == nil {
cmd.ErrorExitln(fmt.Sprintf("未找到匹配的云存储: %s", searchKey))
return
return fmt.Errorf("未找到匹配的云存储: %s", searchKey)
}
fmt.Println("\n\033[1;36m云存储详情\033[0m")
@ -103,4 +98,5 @@ func ls(c *cobra.Command, ctx *cmd.CommandContext, opt lsOpt, args []string) {
fmt.Printf("\033[1m%-8s\033[0m %s\n", "WorkingDir:", userSpace.WorkingDir)
fmt.Println("----------------------------------")
return nil
}

3
jcsctl/cmd/userspace/userspace.go
View File

@ -6,7 +6,8 @@ import (
)
var UserSpaceCmd = &cobra.Command{
Use: "userspace",
Use: "userspace",
Aliases: []string{"us"},
}
func init() {

24
jcsctl/cmd/utils.go
View File

@ -1,16 +1,18 @@
package cmd
import (
"fmt"
"os"
)
import "strings"
func ErrorExitf(format string, args ...interface{}) {
fmt.Printf(format, args...)
os.Exit(1)
}
func SplitObjectPath(str string) (bkt string, pkg string, obj string, ok bool) {
comps := strings.Split(str, ":")
if len(comps) != 2 {
return "", "", "", false
}
func ErrorExitln(msg string) {
fmt.Println(msg)
os.Exit(1)
pat := comps[1]
comps = strings.Split(comps[0], "/")
if len(comps) != 2 {
return "", "", "", false
}
return comps[0], comps[1], pat, true
}