JointCloud
/
JCS-pub
24
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

S3分批清理无用文件

This commit is contained in:
Sydonian 2025-07-04 16:43:54 +08:00
parent c8020cf73f
commit 25b47df721
2 changed files with 27 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
common/pkgs/storage/s3/base_store.go
View File

@ -14,6 +14,7 @@ import (
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/io2"
"gitlink.org.cn/cloudream/common/utils/math2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
)
@ -173,24 +174,25 @@ func (s *BaseStore) CleanTemps() {
marker = resp.NextMarker
}
if len(deletes) == 0 {
return
}
for len(deletes) > 0 {
cnt := math2.Min(500, len(deletes))
resp, err := s.cli.DeleteObjects(context.Background(), &s3.DeleteObjectsInput{
Bucket: aws.String(s.Bucket),
Delete: &s3types.Delete{
Objects: deletes[:cnt],
},
})
if err != nil {
log.Warnf("delete temp files: %v", err)
return
}
resp, err := s.cli.DeleteObjects(context.Background(), &s3.DeleteObjectsInput{
Bucket: aws.String(s.Bucket),
Delete: &s3types.Delete{
Objects: deletes,
},
})
if err != nil {
log.Warnf("delete temp files: %v", err)
return
}
for _, del := range resp.Deleted {
obj := deleteObjs[*del.Key]
log.Infof("remove unused temp file %v, size: %v, last mod time: %v", *obj.Key, *obj.Size, *obj.LastModified)
}
for _, del := range resp.Deleted {
obj := deleteObjs[*del.Key]
log.Infof("remove unused temp file %v, size: %v, last mod time: %v", *obj.Key, *obj.Size, *obj.LastModified)
deletes = deletes[cnt:]
}
}

15
common/pkgs/storage/s3/shard_store.go
View File

@ -8,6 +8,7 @@ import (
"github.com/aws/aws-sdk-go-v2/service/s3"
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/math2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
)
@ -181,12 +182,14 @@ func (s *ShardStore) GC(avaiables []clitypes.FileHash) error {
marker = resp.NextMarker
}
cnt := 0
if len(deletes) > 0 {
resp, err := s.cli.DeleteObjects(context.Background(), &s3.DeleteObjectsInput{
totalCnt := len(deletes)
for len(deletes) > 0 {
cnt := math2.Min(500, len(deletes))
_, err := s.cli.DeleteObjects(context.Background(), &s3.DeleteObjectsInput{
Bucket: aws.String(s.Bucket),
Delete: &s3types.Delete{
Objects: deletes,
Objects: deletes[:cnt],
},
})
if err != nil {
@ -194,10 +197,10 @@ func (s *ShardStore) GC(avaiables []clitypes.FileHash) error {
return err
}
cnt = len(resp.Deleted)
deletes = deletes[cnt:]
}
s.getLogger().Infof("purge %d files", cnt)
s.getLogger().Infof("purge %d files", totalCnt)
// TODO 无法保证原子性，所以删除失败只打日志
return nil
}