From 2edde62fca0aa6f1064ff3efd4bb02b50fafc27a Mon Sep 17 00:00:00 2001 From: lvyangyang Date: Fri, 4 Dec 2020 10:11:25 +0800 Subject: [PATCH] feat(region): add bucket cloud tags support --- pkg/cloudprovider/objectstore.go | 34 +++++++++++++ pkg/compute/models/buckets.go | 22 +++++++++ pkg/multicloud/aliyun/bucket.go | 65 +++++++++++++++++++++++- pkg/multicloud/aws/bucket.go | 76 +++++++++++++++++++++++++++++ pkg/multicloud/bucket_base.go | 12 +++++ pkg/multicloud/huawei/bucket.go | 66 ++++++++++++++++++++++++- pkg/multicloud/objectstore/shell.go | 38 +++++++++++++++ pkg/multicloud/qcloud/bucket.go | 66 +++++++++++++++++++++++++ pkg/util/logclient/consts_i18n.go | 1 + 9 files changed, 378 insertions(+), 2 deletions(-) diff --git a/pkg/cloudprovider/objectstore.go b/pkg/cloudprovider/objectstore.go index 664d69b7cf..1efd7d6dca 100644 --- a/pkg/cloudprovider/objectstore.go +++ b/pkg/cloudprovider/objectstore.go @@ -260,6 +260,10 @@ type ICloudBucket interface { GetPolicy() ([]SBucketPolicyStatement, error) SetPolicy(policy SBucketPolicyStatementInput) error DeletePolicy(id []string) ([]SBucketPolicyStatement, error) + + GetTags() (map[string]string, error) + SetTags(tags map[string]string) error + DeleteTags() error } type ICloudObject interface { @@ -822,3 +826,33 @@ func DeleteBucketCORS(ibucket ICloudBucket, id []string) ([]SBucketCORSRule, err return deletedRules, nil } + +func SetBucketMetadata(ibucket ICloudBucket, tags map[string]string, replace bool) error { + newTags := map[string]string{} + if replace { + newTags = tags + } else { + oldTags, err := ibucket.GetTags() + if err != nil { + return errors.Wrap(err, "b.getTags()") + } + for k, v := range oldTags { + if _, ok := tags[k]; !ok { + tags[k] = v + } + } + newTags = tags + } + if len(newTags) == 0 { + err := ibucket.DeleteTags() + if err != nil { + return errors.Wrap(err, "b.DeleteTags()") + } + } else { + err := ibucket.SetTags(newTags) + if err != nil { + return errors.Wrapf(err, "b.setTags(%s)", jsonutils.Marshal(newTags).String()) + } + } + return nil +} diff --git a/pkg/compute/models/buckets.go b/pkg/compute/models/buckets.go index dcf02d96c0..f52dbb750f 100644 --- a/pkg/compute/models/buckets.go +++ b/pkg/compute/models/buckets.go @@ -228,6 +228,7 @@ func (manager *SBucketManager) newFromCloudBucket( SyncCloudProject(userCred, &bucket, provider.GetOwnerId(), extBucket, provider.Id) bucket.SyncShareState(ctx, userCred, provider.getAccountShareInfo()) + syncVirtualResourceMetadata(ctx, userCred, &bucket, extBucket) db.OpsLog.LogEvent(&bucket, db.ACT_CREATE, bucket.GetShortDesc(ctx), userCred) return &bucket, nil @@ -298,6 +299,8 @@ func (bucket *SBucket) syncWithCloudBucket( return errors.Wrap(err, "db.UpdateWithLock") } + syncVirtualResourceMetadata(ctx, userCred, bucket, extBucket) + db.OpsLog.LogSyncUpdate(bucket, diff, userCred) if !oStats.Equals(extBucket.GetStats()) { @@ -539,6 +542,11 @@ func (bucket *SBucket) RemoteCreate(ctx context.Context, userCred mcclient.Token if err != nil { return errors.Wrap(err, "bucket.syncWithCloudBucket") } + tags, _ := bucket.GetAllUserMetadata() + err = cloudprovider.SetBucketMetadata(extBucket, tags, false) + if err != nil { + log.Errorf("iBucket.SetMetadata failed: %s", err) + } return nil } @@ -1859,6 +1867,20 @@ func (bucket *SBucket) processObjectsActionInput(input api.BucketObjectsActionIn return iBucket, objects, nil } +func (bucket *SBucket) OnMetadataUpdated(ctx context.Context, userCred mcclient.TokenCredential) { + iBucket, err := bucket.GetIBucket() + if err != nil { + log.Errorf("bucket.GetIBucket() failed: %s", err) + return + } + tags, _ := bucket.GetAllUserMetadata() + err = cloudprovider.SetBucketMetadata(iBucket, tags, false) + if err != nil { + log.Errorf("iBucket.SetMetadata failed: %s", err) + } + db.OpsLog.LogEvent(bucket, db.ACT_UPDATE_TAGS, tags, userCred) +} + func (manager *SBucketManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, diff --git a/pkg/multicloud/aliyun/bucket.go b/pkg/multicloud/aliyun/bucket.go index 285baea9eb..a55c3eca7f 100644 --- a/pkg/multicloud/aliyun/bucket.go +++ b/pkg/multicloud/aliyun/bucket.go @@ -630,7 +630,7 @@ func (b *SBucket) SetReferer(conf cloudprovider.SBucketRefererConf) error { } err = osscli.SetBucketReferer(b.Name, conf.WhiteList, conf.AllowEmptyRefer) if err != nil { - return errors.Wrapf(err, "osscli.SetBucketReferer(%s,%s,%d)", b.Name, conf.WhiteList, conf.AllowEmptyRefer) + return errors.Wrapf(err, "osscli.SetBucketReferer(%s,%s,%t)", b.Name, conf.WhiteList, conf.AllowEmptyRefer) } return nil } @@ -715,3 +715,66 @@ func (b *SBucket) GetCdnDomains() ([]cloudprovider.SCdnDomain, error) { } return result, nil } + +func (b *SBucket) GetTags() (map[string]string, error) { + osscli, err := b.region.GetOssClient() + if err != nil { + return nil, errors.Wrap(err, "GetOssClient") + } + + tagresult, err := osscli.GetBucketTagging(b.Name) + if err != nil { + if strings.Contains(err.Error(), "404") { + return nil, nil + } + return nil, errors.Wrapf(err, "osscli.GetBucketTagging(%s)", b.Name) + } + result := map[string]string{} + for i := range tagresult.Tags { + result[tagresult.Tags[i].Key] = tagresult.Tags[i].Value + } + return result, nil +} + +func (b *SBucket) SetTags(tags map[string]string) error { + osscli, err := b.region.GetOssClient() + if err != nil { + return errors.Wrap(err, "GetOssClient") + } + + input := []oss.Tag{} + for k, v := range tags { + input = append(input, oss.Tag{Key: k, Value: v}) + } + + err = osscli.SetBucketTagging(b.Name, oss.Tagging{Tags: input}) + if err != nil { + return errors.Wrapf(err, "osscli.SetBucketTagging(%s)", jsonutils.Marshal(input)) + } + return nil +} + +func (b *SBucket) DeleteTags() error { + osscli, err := b.region.GetOssClient() + if err != nil { + return errors.Wrap(err, "GetOssClient") + } + err = osscli.DeleteBucketTagging(b.Name) + if err != nil { + return errors.Wrapf(err, "osscli.DeleteBucketTagging(%s)", b.Name) + } + return nil +} + +func (b *SBucket) GetMetadata() *jsonutils.JSONDict { + meta := jsonutils.NewDict() + tags, err := b.GetTags() + if err != nil { + log.Errorf("error:%s b.getTags()", err) + return meta + } + for k, v := range tags { + meta.Add(jsonutils.NewString(v), k) + } + return meta +} diff --git a/pkg/multicloud/aws/bucket.go b/pkg/multicloud/aws/bucket.go index 42935c1102..297c48da6d 100644 --- a/pkg/multicloud/aws/bucket.go +++ b/pkg/multicloud/aws/bucket.go @@ -678,3 +678,79 @@ func (b *SBucket) DeleteCORS() error { } return nil } + +func (b *SBucket) GetTags() (map[string]string, error) { + s3cli, err := b.region.GetS3Client() + if err != nil { + return nil, errors.Wrap(err, "GetS3Client") + } + tagresult, err := s3cli.GetBucketTagging(&s3.GetBucketTaggingInput{Bucket: &b.Name}) + if err != nil { + if strings.Contains(err.Error(), "NoSuchTagSet") { + return nil, nil + } + return nil, errors.Wrapf(err, "osscli.GetBucketTagging(%s)", b.Name) + } + if tagresult == nil { + return nil, nil + } + result := map[string]string{} + for i := range tagresult.TagSet { + if tagresult.TagSet[i].Key != nil && tagresult.TagSet[i].Value != nil { + result[*tagresult.TagSet[i].Key] = *tagresult.TagSet[i].Value + } + + } + return result, nil +} + +func (b *SBucket) SetTags(tags map[string]string) error { + s3cli, err := b.region.GetS3Client() + if err != nil { + return errors.Wrap(err, "GetS3Client") + } + + input := s3.PutBucketTaggingInput{Tagging: &s3.Tagging{}} + input.Bucket = &b.Name + apiTagKeys := []string{} + apiTagValues := []string{} + for k, v := range tags { + apiTagKeys = append(apiTagKeys, k) + apiTagValues = append(apiTagValues, v) + + } + for i := range apiTagKeys { + input.Tagging.TagSet = append(input.Tagging.TagSet, &s3.Tag{Key: &apiTagKeys[i], Value: &apiTagValues[i]}) + } + + _, err = s3cli.PutBucketTagging(&input) + if err != nil { + return errors.Wrapf(err, "obscli.SetBucketTagging(%s)", jsonutils.Marshal(input)) + } + return nil +} + +func (b *SBucket) DeleteTags() error { + s3cli, err := b.region.GetS3Client() + if err != nil { + return errors.Wrap(err, "GetS3Client") + } + _, err = s3cli.DeleteBucketTagging(&s3.DeleteBucketTaggingInput{Bucket: &b.Name}) + if err != nil { + return errors.Wrapf(err, "osscli.DeleteBucketTagging(%s)", b.Name) + } + return nil +} + +func (b *SBucket) GetMetadata() *jsonutils.JSONDict { + meta := jsonutils.NewDict() + tags, err := b.GetTags() + if err != nil { + log.Errorf("error:%s b.getTags()", err) + return meta + } + for k, v := range tags { + meta.Add(jsonutils.NewString(v), k) + } + return meta +} diff --git a/pkg/multicloud/bucket_base.go b/pkg/multicloud/bucket_base.go index 13bc014f82..5abbf6a6e4 100644 --- a/pkg/multicloud/bucket_base.go +++ b/pkg/multicloud/bucket_base.go @@ -121,3 +121,15 @@ func (b *SBaseBucket) SetPolicy(policy cloudprovider.SBucketPolicyStatementInput func (b *SBaseBucket) DeletePolicy(id []string) ([]cloudprovider.SBucketPolicyStatement, error) { return nil, cloudprovider.ErrNotImplemented } + +func (b *SBaseBucket) GetTags() (map[string]string, error) { + return nil, cloudprovider.ErrNotImplemented +} + +func (b *SBaseBucket) SetTags(tags map[string]string) error { + return cloudprovider.ErrNotImplemented +} + +func (b *SBaseBucket) DeleteTags() error { + return cloudprovider.ErrNotImplemented +} diff --git a/pkg/multicloud/huawei/bucket.go b/pkg/multicloud/huawei/bucket.go index 0f5e509a67..89f38464ed 100644 --- a/pkg/multicloud/huawei/bucket.go +++ b/pkg/multicloud/huawei/bucket.go @@ -23,6 +23,7 @@ import ( "strings" "time" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/utils" @@ -628,7 +629,7 @@ func (b *SBucket) SetCORS(rules []cloudprovider.SBucketCORSRule) error { input.BucketCors.CorsRules = opts _, err = obscli.SetBucketCors(&input) if err != nil { - return errors.Wrapf(err, "obscli.SetBucketCors(%s)", input) + return errors.Wrapf(err, "obscli.SetBucketCors(%s)", jsonutils.Marshal(input).String()) } return nil } @@ -673,3 +674,66 @@ func (b *SBucket) DeleteCORS() error { } return nil } + +func (b *SBucket) GetTags() (map[string]string, error) { + obscli, err := b.region.getOBSClient() + if err != nil { + return nil, errors.Wrap(err, "GetOBSClient") + } + tagresult, err := obscli.GetBucketTagging(b.Name) + if err != nil { + if strings.Contains(err.Error(), "404") { + return nil, nil + } + return nil, errors.Wrapf(err, "osscli.GetBucketTagging(%s)", b.Name) + } + result := map[string]string{} + for i := range tagresult.Tags { + result[tagresult.Tags[i].Key] = tagresult.Tags[i].Value + } + return result, nil +} + +func (b *SBucket) SetTags(tags map[string]string) error { + obscli, err := b.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "GetOBSClient") + } + + input := obs.SetBucketTaggingInput{BucketTagging: obs.BucketTagging{}} + input.Bucket = b.Name + for k, v := range tags { + input.BucketTagging.Tags = append(input.BucketTagging.Tags, obs.Tag{Key: k, Value: v}) + } + + _, err = obscli.SetBucketTagging(&input) + if err != nil { + return errors.Wrapf(err, "obscli.SetBucketTagging(%s)", jsonutils.Marshal(input).String()) + } + return nil +} + +func (b *SBucket) DeleteTags() error { + obscli, err := b.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "GetOBSClient") + } + _, err = obscli.DeleteBucketTagging(b.Name) + if err != nil { + return errors.Wrapf(err, "osscli.DeleteBucketTagging(%s)", b.Name) + } + return nil +} + +func (b *SBucket) GetMetadata() *jsonutils.JSONDict { + meta := jsonutils.NewDict() + tags, err := b.GetTags() + if err != nil { + log.Errorf("error:%s b.getTags()", err) + return meta + } + for k, v := range tags { + meta.Add(jsonutils.NewString(v), k) + } + return meta +} diff --git a/pkg/multicloud/objectstore/shell.go b/pkg/multicloud/objectstore/shell.go index 7e1a63532e..981a50af8b 100644 --- a/pkg/multicloud/objectstore/shell.go +++ b/pkg/multicloud/objectstore/shell.go @@ -631,6 +631,44 @@ func S3Shell() { return nil }) + type BucketGetMetadata struct { + BUCKET string `help:"name of bucket to put object"` + } + shellutils.R(&BucketGetMetadata{}, "bucket-get-metadata", "get bucket metadata", func(cli cloudprovider.ICloudRegion, args *BucketGetMetadata) error { + bucket, err := cli.GetIBucketById(args.BUCKET) + if err != nil { + return err + } + meta := bucket.GetMetadata() + printObject(meta) + return nil + }) + + type BucketSetMetadate struct { + BUCKET string `help:"name of bucket to put object"` + Tags []string `help:"Tags info, eg: hypervisor=aliyun、os_type=Linux、os_version"` + Replace bool + } + shellutils.R(&BucketSetMetadate{}, "bucket-set-metadata", "set bucket metadata", func(cli cloudprovider.ICloudRegion, args *BucketSetMetadate) error { + bucket, err := cli.GetIBucketById(args.BUCKET) + if err != nil { + return err + } + tags := map[string]string{} + for _, tag := range args.Tags { + pair := strings.Split(tag, "=") + if len(pair) == 2 { + tags[pair[0]] = pair[1] + } + } + err = cloudprovider.SetBucketMetadata(bucket, tags, args.Replace) + if err != nil { + return err + } + fmt.Println("Success!") + return nil + }) + type BucketObjectDownloadOptions struct { BUCKET string `help:"name of bucket"` KEY string `help:"Key of object"` diff --git a/pkg/multicloud/qcloud/bucket.go b/pkg/multicloud/qcloud/bucket.go index 8f9cc29221..1f3465b3cb 100644 --- a/pkg/multicloud/qcloud/bucket.go +++ b/pkg/multicloud/qcloud/bucket.go @@ -1069,3 +1069,69 @@ func (b *SBucket) DeletePolicy(id []string) ([]cloudprovider.SBucketPolicyStatem } return deletedPolicy, nil } + +func (b *SBucket) GetTags() (map[string]string, error) { + coscli, err := b.region.GetCosClient(b) + if err != nil { + log.Errorf("GetCosClient fail %s", err) + return nil, errors.Wrap(err, "b.region.GetCosClient(b)") + } + + tagresult, _, err := coscli.Bucket.GetTagging(context.Background()) + if err != nil { + if strings.Contains(err.Error(), "404") { + return nil, nil + } + return nil, errors.Wrap(err, "coscli.Bucket.GetTagging(context.Background())") + } + result := map[string]string{} + for i := range tagresult.TagSet { + result[tagresult.TagSet[i].Key] = tagresult.TagSet[i].Value + } + return result, nil +} + +func (b *SBucket) SetTags(tags map[string]string) error { + coscli, err := b.region.GetCosClient(b) + if err != nil { + log.Errorf("GetCosClient fail %s", err) + return errors.Wrap(err, "b.region.GetCosClient(b)") + } + + input := cos.BucketPutTaggingOptions{} + for k, v := range tags { + input.TagSet = append(input.TagSet, cos.BucketTaggingTag{Key: k, Value: v}) + } + + _, err = coscli.Bucket.PutTagging(context.Background(), &input) + if err != nil { + return errors.Wrapf(err, "coscli.Bucket.PutTagging(%s)", jsonutils.Marshal(input)) + } + return nil +} + +func (b *SBucket) DeleteTags() error { + coscli, err := b.region.GetCosClient(b) + if err != nil { + log.Errorf("GetCosClient fail %s", err) + return errors.Wrap(err, "b.region.GetCosClient(b)") + } + _, err = coscli.Bucket.DeleteTagging(context.Background()) + if err != nil { + return errors.Wrap(err, "coscli.Bucket.DeleteTagging(context.Background())") + } + return nil +} + +func (b *SBucket) GetMetadata() *jsonutils.JSONDict { + meta := jsonutils.NewDict() + tags, err := b.GetTags() + if err != nil { + log.Errorf("error:%s b.getTags()", err) + return meta + } + for k, v := range tags { + meta.Add(jsonutils.NewString(v), k) + } + return meta +} diff --git a/pkg/util/logclient/consts_i18n.go b/pkg/util/logclient/consts_i18n.go index 006fc80c80..f1e160a1ab 100644 --- a/pkg/util/logclient/consts_i18n.go +++ b/pkg/util/logclient/consts_i18n.go @@ -656,6 +656,7 @@ func init() { t.Set(ACT_UPDATE_RULE, i18n.NewTableEntry(). EN("Update RuleConfig"). CN("调整规则配置"), + ) t.Set(ACT_UPDATE_TAGS, i18n.NewTableEntry(). EN("Update Tags"). CN("修改标签"),