// Copyright 2019 Yunion // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package objectstore import ( "context" "fmt" "io" "net/http" "os" "strings" "time" "yunion.io/x/pkg/errors" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/util/printutils" "yunion.io/x/onecloud/pkg/util/shellutils" "yunion.io/x/onecloud/pkg/util/streamutils" ) type ObjectHeaderOptions struct { CacheControl string `help:"Cache-Control"` ContentType string `help:"Content-Type"` ContentEncoding string `help:"Content-Encoding"` ContentLanguage string `help:"Content-Language"` ContentDisposition string `help:"Content-Disposition"` ContentMD5 string `help:"Content-MD5"` Meta []string `help:"header, common seperatored key and value, e.g. max-age:100"` } func (args ObjectHeaderOptions) Options2Header() http.Header { meta := http.Header{} for _, kv := range args.Meta { parts := strings.Split(kv, ":") if len(parts) == 2 { key := strings.TrimSpace(parts[0]) value := strings.TrimSpace(parts[1]) if len(key) > 0 && len(value) > 0 { meta.Add(key, value) } } } if len(args.CacheControl) > 0 { meta.Set(cloudprovider.META_HEADER_CACHE_CONTROL, args.CacheControl) } if len(args.ContentType) > 0 { meta.Set(cloudprovider.META_HEADER_CONTENT_TYPE, args.ContentType) } if len(args.ContentEncoding) > 0 { meta.Set(cloudprovider.META_HEADER_CONTENT_ENCODING, args.ContentEncoding) } if len(args.ContentMD5) > 0 { meta.Set(cloudprovider.META_HEADER_CONTENT_MD5, args.ContentMD5) } if len(args.ContentLanguage) > 0 { meta.Set(cloudprovider.META_HEADER_CONTENT_LANGUAGE, args.ContentLanguage) } if len(args.ContentDisposition) > 0 { meta.Set(cloudprovider.META_HEADER_CONTENT_DISPOSITION, args.ContentDisposition) } return meta } func S3Shell() { type BucketListOptions struct { } shellutils.R(&BucketListOptions{}, "bucket-list", "List all bucket", func(cli cloudprovider.ICloudRegion, args *BucketListOptions) error { buckets, err := cli.GetIBuckets() if err != nil { return err } printutils.PrintGetterList(buckets, nil) return nil }) type BucketCreateOptions struct { NAME string `help:"name of bucket to create"` Acl string `help:"ACL string" choices:"private|public-read|public-read-write"` StorageClass string `help:"StorageClass"` } shellutils.R(&BucketCreateOptions{}, "bucket-create", "Create bucket", func(cli cloudprovider.ICloudRegion, args *BucketCreateOptions) error { err := cli.CreateIBucket(args.NAME, args.StorageClass, args.Acl) if err != nil { return err } return nil }) type BucketAclOptions struct { BUCKET string `help:"name of bucket"` ACL string `help:"ACL string" choices:"private|public-read|public-read-write"` } shellutils.R(&BucketAclOptions{}, "bucket-set-acl", "Create bucket", func(cli cloudprovider.ICloudRegion, args *BucketAclOptions) error { bucket, err := cli.GetIBucketById(args.BUCKET) if err != nil { return err } err = bucket.SetAcl(cloudprovider.TBucketACLType(args.ACL)) if err != nil { return err } return nil }) type BucketDeleteOptions struct { NAME string `help:"name of bucket to delete"` } shellutils.R(&BucketDeleteOptions{}, "bucket-delete", "Delete bucket", func(cli cloudprovider.ICloudRegion, args *BucketDeleteOptions) error { err := cli.DeleteIBucket(args.NAME) if err != nil { return err } return nil }) type BucketLimitOptions struct { NAME string `help:"name of bucket to set limit"` SizeGB int `help:"limit of volumes in GB"` Objects int `help:"limit of object count"` Off bool `help:"Turn off limit"` } shellutils.R(&BucketLimitOptions{}, "bucket-set-limit", "Set bucket limit", func(cli cloudprovider.ICloudRegion, args *BucketLimitOptions) error { bucket, err := cli.GetIBucketByName(args.NAME) if err != nil { return err } if args.Off { err = bucket.SetLimit(cloudprovider.SBucketStats{}) } else { fmt.Println("set limit") err = bucket.SetLimit(cloudprovider.SBucketStats{SizeBytes: int64(args.SizeGB * 1000 * 1000 * 1000), ObjectCount: args.Objects}) } if err != nil { return err } return nil }) type BucketExistOptions struct { NAME string `help:"name of bucket to delete"` } shellutils.R(&BucketExistOptions{}, "bucket-exist", "Test existence of a bucket", func(cli cloudprovider.ICloudRegion, args *BucketExistOptions) error { exist, err := cli.IBucketExist(args.NAME) if err != nil { return err } fmt.Printf("Exist: %v\n", exist) return nil }) type BucketObjectsOptions struct { BUCKET string `help:"name of bucket to list objects"` Prefix string `help:"prefix"` Marker string `help:"marker"` Demiliter string `help:"delimiter"` Max int `help:"Max count"` } shellutils.R(&BucketObjectsOptions{}, "bucket-object", "List objects in a bucket", func(cli cloudprovider.ICloudRegion, args *BucketObjectsOptions) error { bucket, err := cli.GetIBucketById(args.BUCKET) if err != nil { return err } result, err := bucket.ListObjects(args.Prefix, args.Marker, args.Demiliter, args.Max) if err != nil { return err } if result.IsTruncated { fmt.Printf("NextMarker: %s IsTruncated: %v\n", result.NextMarker, result.IsTruncated) } fmt.Println("Common prefixes:") printutils.PrintGetterList(result.CommonPrefixes, []string{"key", "size_bytes"}) fmt.Println("Objects:") printutils.PrintGetterList(result.Objects, []string{"key", "size_bytes"}) return nil }) type BucketListObjectsOptions struct { BUCKET string `help:"name of bucket to list objects"` Prefix string `help:"prefix"` Limit int `help:"limit per page request" default:"20"` Marker string `help:"offset marker"` } shellutils.R(&BucketListObjectsOptions{}, "bucket-list-object", "List objects in a bucket", func(cli cloudprovider.ICloudRegion, args *BucketListObjectsOptions) error { bucket, err := cli.GetIBucketById(args.BUCKET) if err != nil { return err } objects, marker, err := cloudprovider.GetPagedObjects(bucket, args.Prefix, true, args.Marker, args.Limit) if err != nil { return err } printutils.PrintGetterList(objects, []string{"key", "size_bytes"}) if len(marker) > 0 { fmt.Println("Next marker:", marker) } return nil }) shellutils.R(&BucketListObjectsOptions{}, "bucket-dir-object", "List objects in a bucket like directory", func(cli cloudprovider.ICloudRegion, args *BucketListObjectsOptions) error { bucket, err := cli.GetIBucketById(args.BUCKET) if err != nil { return err } objects, marker, err := cloudprovider.GetPagedObjects(bucket, args.Prefix, false, args.Marker, args.Limit) if err != nil { return err } printutils.PrintGetterList(objects, []string{"key", "size_bytes"}) if len(marker) > 0 { fmt.Println("Next marker:", marker) } return nil }) type BucketMakrdirOptions struct { BUCKET string `help:"name of bucket to put object"` DIR string `help:"dir to make"` } shellutils.R(&BucketMakrdirOptions{}, "bucket-mkdir", "Mkdir in a bucket", func(cli cloudprovider.ICloudRegion, args *BucketMakrdirOptions) error { bucket, err := cli.GetIBucketById(args.BUCKET) if err != nil { return err } err = cloudprovider.Makedir(context.Background(), bucket, args.DIR) if err != nil { return err } fmt.Printf("Mkdir success\n") return nil }) type BucketPutObjectOptions struct { BUCKET string `help:"name of bucket to put object"` KEY string `help:"key of object"` Path string `help:"Path of file to upload"` BlockSize int64 `help:"blocksz in MB" default:"100"` Acl string `help:"acl" choices:"private|public-read|public-read-write"` StorageClass string `help:"storage class"` ObjectHeaderOptions } shellutils.R(&BucketPutObjectOptions{}, "put-object", "Put object into a bucket", func(cli cloudprovider.ICloudRegion, args *BucketPutObjectOptions) error { bucket, err := cli.GetIBucketById(args.BUCKET) if err != nil { return err } var input io.ReadSeeker var fSize int64 if len(args.Path) > 0 { finfo, err := os.Stat(args.Path) if err != nil { return errors.Wrap(err, "os.Stat") } fSize = finfo.Size() file, err := os.Open(args.Path) if err != nil { return errors.Wrap(err, "os.Open") } defer file.Close() input = file } else { input = os.Stdout } meta := args.ObjectHeaderOptions.Options2Header() err = cloudprovider.UploadObject(context.Background(), bucket, args.KEY, args.BlockSize*1000*1000, input, fSize, cloudprovider.TBucketACLType(args.Acl), args.StorageClass, meta, true) if err != nil { return err } fmt.Printf("Upload success\n") return nil }) type BucketDeleteObjectOptions struct { BUCKET string `help:"name of bucket to put object"` KEY string `help:"key of object"` } shellutils.R(&BucketDeleteObjectOptions{}, "delete-object", "Delete object from a bucket", func(cli cloudprovider.ICloudRegion, args *BucketDeleteObjectOptions) error { bucket, err := cli.GetIBucketById(args.BUCKET) if err != nil { return err } err = bucket.DeleteObject(context.Background(), args.KEY) if err != nil { return err } fmt.Printf("Delete success\n") return nil }) shellutils.R(&BucketDeleteObjectOptions{}, "delete-prefix", "Delete object from a bucket", func(cli cloudprovider.ICloudRegion, args *BucketDeleteObjectOptions) error { bucket, err := cli.GetIBucketById(args.BUCKET) if err != nil { return err } err = cloudprovider.DeletePrefix(context.Background(), bucket, args.KEY) if err != nil { return err } fmt.Printf("Delete success\n") return nil }) type BucketTempUrlOption struct { BUCKET string `help:"name of bucket to put object"` METHOD string `help:"http method" choices:"GET|PUT|DELETE"` KEY string `help:"key of object"` Duration int `help:"duration in seconds" default:"60"` } shellutils.R(&BucketTempUrlOption{}, "temp-url", "generate temp url", func(cli cloudprovider.ICloudRegion, args *BucketTempUrlOption) error { bucket, err := cli.GetIBucketById(args.BUCKET) if err != nil { return err } urlStr, err := bucket.GetTempUrl(args.METHOD, args.KEY, time.Duration(args.Duration)*time.Second) if err != nil { return err } fmt.Println(urlStr) return nil }) type BucketAclOption struct { BUCKET string `help:"name of bucket to put object"` KEY string `help:"key of object"` } shellutils.R(&BucketAclOption{}, "object-acl", "Get object acl", func(cli cloudprovider.ICloudRegion, args *BucketAclOption) error { bucket, err := cli.GetIBucketById(args.BUCKET) if err != nil { return err } object, err := cloudprovider.GetIObject(bucket, args.KEY) if err != nil { return err } fmt.Println(object.GetAcl()) return nil }) type BucketSetAclOption struct { BUCKET string `help:"name of bucket to put object"` KEY string `help:"key of object"` ACL string `help:"Target acl" choices:"default|private|public-read|public-read-write"` } shellutils.R(&BucketSetAclOption{}, "object-set-acl", "Get object acl", func(cli cloudprovider.ICloudRegion, args *BucketSetAclOption) error { bucket, err := cli.GetIBucketById(args.BUCKET) if err != nil { return err } object, err := cloudprovider.GetIObject(bucket, args.KEY) if err != nil { return err } err = object.SetAcl(cloudprovider.TBucketACLType(args.ACL)) if err != nil { return err } fmt.Println("Success!") return nil }) type BucketObjectDownloadOptions struct { BUCKET string `help:"name of bucket"` KEY string `help:"Key of object"` Output string `help:"target output, default to stdout"` Start int64 `help:"partial download start"` End int64 `help:"partial download end"` } shellutils.R(&BucketObjectDownloadOptions{}, "object-download", "Download", func(cli cloudprovider.ICloudRegion, args *BucketObjectDownloadOptions) error { bucket, err := cli.GetIBucketById(args.BUCKET) if err != nil { return err } obj, err := cloudprovider.GetIObject(bucket, args.KEY) if err != nil { return err } var rangeOpt *cloudprovider.SGetObjectRange if args.Start != 0 || args.End != 0 { if args.End <= 0 { args.End = obj.GetSizeBytes() - 1 } rangeOpt = &cloudprovider.SGetObjectRange{Start: args.Start, End: args.End} } output, err := bucket.GetObject(context.Background(), args.KEY, rangeOpt) if err != nil { return err } defer output.Close() var target io.Writer if len(args.Output) == 0 { target = os.Stdout } else { fp, err := os.Create(args.Output) if err != nil { return err } defer fp.Close() target = fp } prop, err := streamutils.StreamPipe(output, target, false, nil) if err != nil { return err } if len(args.Output) > 0 { fmt.Println("Success:", prop.Size, "written") } return nil }) type BucketObjectCopyOptions struct { SRC string `help:"name of source bucket"` SRCKEY string `help:"Key of source object"` DST string `help:"name of destination bucket"` DSTKEY string `help:"key of destination object"` Debug bool `help:"show debug info"` BlockSize int64 `help:"block size in MB"` Native bool `help:"Use native copy"` ObjectHeaderOptions } shellutils.R(&BucketObjectCopyOptions{}, "object-copy", "Copy object", func(cli cloudprovider.ICloudRegion, args *BucketObjectCopyOptions) error { ctx := context.Background() dstBucket, err := cli.GetIBucketByName(args.DST) if err != nil { return err } srcBucket, err := cli.GetIBucketByName(args.SRC) if err != nil { return err } srcObj, err := cloudprovider.GetIObject(srcBucket, args.SRCKEY) if err != nil { return err } meta := args.ObjectHeaderOptions.Options2Header() if args.Native { err = dstBucket.CopyObject(ctx, args.DSTKEY, args.SRC, args.SRCKEY, srcObj.GetAcl(), srcObj.GetStorageClass(), meta) if err != nil { return err } } else { err = cloudprovider.CopyObject(ctx, args.BlockSize*1000*1000, dstBucket, args.DSTKEY, srcBucket, args.SRCKEY, meta, args.Debug) if err != nil { return err } } fmt.Println("Success!") return nil }) type ObjectMetaOptions struct { BUCKET string `help:"bucket name"` KEY string `help:"object key"` } shellutils.R(&ObjectMetaOptions{}, "object-meta", "Show object meta header", func(cli cloudprovider.ICloudRegion, args *ObjectMetaOptions) error { bucket, err := cli.GetIBucketByName(args.BUCKET) if err != nil { return err } obj, err := cloudprovider.GetIObject(bucket, args.KEY) if err != nil { return err } meta := obj.GetMeta() for k, v := range meta { fmt.Println(k, ": ", v[0]) } return nil }) type ObjectSetMetaOptions struct { BUCKET string `help:"bucket name"` KEY string `help:"object key"` ObjectHeaderOptions } shellutils.R(&ObjectSetMetaOptions{}, "object-set-meta", "Set object meta header", func(cli cloudprovider.ICloudRegion, args *ObjectSetMetaOptions) error { bucket, err := cli.GetIBucketByName(args.BUCKET) if err != nil { return err } obj, err := cloudprovider.GetIObject(bucket, args.KEY) if err != nil { return err } meta := args.ObjectHeaderOptions.Options2Header() err = obj.SetMeta(context.Background(), meta) if err != nil { return err } return nil }) }