Merge pull request #1142 from swordqiu/automated-cherry-pick-of-#1140…

…-upstream-master

Automated cherry pick of #1140: fix: ignore put object acl error

go.mod

@@ -31,10 +31,10 @@ require (
 	golang.org/x/text v0.14.0
 	gopkg.in/fatih/set.v0 v0.2.1
 	moul.io/http2curl/v2 v2.3.0
-	yunion.io/x/jsonutils v1.0.1-0.20240203102553-4096f103b401
+	yunion.io/x/jsonutils v1.0.1-0.20240930100528-1671a2d0d22f
 	yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91
-	yunion.io/x/pkg v1.10.1-0.20240905110705-77c46e716318
-	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
+	yunion.io/x/pkg v1.10.2
+	yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1
 	yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c
 )
 

go.sum

@@ -415,15 +415,15 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 moul.io/http2curl/v2 v2.3.0 h1:9r3JfDzWPcbIklMOs2TnIFzDYvfAZvjeavG6EzP7jYs=
 moul.io/http2curl/v2 v2.3.0/go.mod h1:RW4hyBjTWSYDOxapodpNEtX0g5Eb16sxklBqmd2RHcE=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
-yunion.io/x/jsonutils v1.0.1-0.20240203102553-4096f103b401 h1:4l6ELFSQ0MBVInscZ8/yOtSWF0cwH5BT1ATN6dCtAqc=
-yunion.io/x/jsonutils v1.0.1-0.20240203102553-4096f103b401/go.mod h1:VK4Z93dgiKgAijcSqbMKmGaBMJuHulR16Hz4K015ZPo=
+yunion.io/x/jsonutils v1.0.1-0.20240930100528-1671a2d0d22f h1:N7V0uLqX9mED1HEPeIKcYpkWglnHL5hyn4ENX+VBeqM=
+yunion.io/x/jsonutils v1.0.1-0.20240930100528-1671a2d0d22f/go.mod h1:VK4Z93dgiKgAijcSqbMKmGaBMJuHulR16Hz4K015ZPo=
 yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE=
 yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91 h1:inY5o3LDa/zgsIZuPN0HmpzKIsu/lLgsBmMttuDPGj4=
 yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v1.10.1-0.20240905110705-77c46e716318 h1:Fm7I8ypXHxeObY4u/VUGz78NsambemzTZ9fECyGKNi8=
-yunion.io/x/pkg v1.10.1-0.20240905110705-77c46e716318/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM=
-yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
-yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
+yunion.io/x/pkg v1.10.2 h1:oZhedvlvDsebIWcOvrOMJQ31SxzjxxqUXUBYseBjh7w=
+yunion.io/x/pkg v1.10.2/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM=
+yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 h1:1KJ3YYinydPHpDEQRXdr/T8SYcKZ5Er+m489H+PnaQ4=
+yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
 yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c h1:QuLab2kSRECZRxo4Lo2KcYn6XjQFDGaZ1+x0pYDVVwQ=
 yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/multicloud/objectstore/buckets.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 	"time"
 
+	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/s3cli"
@@ -243,7 +244,8 @@ func (bucket *SBucket) PutObject(ctx context.Context, key string, input io.Reade
 	}
 	err = obj.SetAcl(cannedAcl)
 	if err != nil && errors.Cause(err) != cloudprovider.ErrNotImplemented {
-		return errors.Wrap(err, "obj.SetAcl")
+		log.Errorf("PubObject SetAcl fail %s", err)
+		// return errors.Wrap(err, "obj.SetAcl")
 	}
 	return nil
 }

pkg/multicloud/objectstore/shell.go

@@ -838,6 +838,35 @@ func S3Shell() {
 		return nil
 	})
 
+	type BucketObjectBatchDownloadOptions struct {
+		BUCKET string `help:"name of bucket"`
+		PREFIX string `help:"Prefix of object"`
+		Output string `help:"target output directory, default to current directory"`
+	}
+	shellutils.R(&BucketObjectBatchDownloadOptions{}, "batch-download", "Download objects recursively", func(cli cloudprovider.ICloudRegion, args *BucketObjectBatchDownloadOptions) error {
+		bucket, err := cli.GetIBucketById(args.BUCKET)
+		if err != nil {
+			return err
+		}
+		marker := ""
+		const maxCount = 1000
+		for {
+			results, err := bucket.ListObjects(args.PREFIX, marker, "", maxCount)
+			if err != nil {
+				return errors.Wrapf(err, "ListObjects prefix %s marker %s", args.PREFIX, marker)
+			}
+			for _, obj := range results.Objects {
+				fmt.Println(obj.GetKey())
+			}
+			if !results.IsTruncated {
+				break
+			} else {
+				marker = results.NextMarker
+			}
+		}
+		return nil
+	})
+
 	type BucketObjectDownloadOptions struct {
 		BUCKET string `help:"name of bucket"`
 		KEY    string `help:"Key of object"`

vendor/modules.txt

@@ -662,14 +662,14 @@ gopkg.in/yaml.v2
 # moul.io/http2curl/v2 v2.3.0
 ## explicit; go 1.13
 moul.io/http2curl/v2
-# yunion.io/x/jsonutils v1.0.1-0.20240203102553-4096f103b401
+# yunion.io/x/jsonutils v1.0.1-0.20240930100528-1671a2d0d22f
 ## explicit; go 1.18
 yunion.io/x/jsonutils
 # yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91
 ## explicit; go 1.12
 yunion.io/x/log
 yunion.io/x/log/hooks
-# yunion.io/x/pkg v1.10.1-0.20240905110705-77c46e716318
+# yunion.io/x/pkg v1.10.2
 ## explicit; go 1.18
 yunion.io/x/pkg/appctx
 yunion.io/x/pkg/errors
@@ -708,7 +708,7 @@ yunion.io/x/pkg/util/timeutils
 yunion.io/x/pkg/util/version
 yunion.io/x/pkg/util/vmdkutils
 yunion.io/x/pkg/utils
-# yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
+# yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1
 ## explicit; go 1.12
 yunion.io/x/s3cli
 # yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c