fix: fail to reset policy org_node_id to empty (#19425)

Co-authored-by: Qiu Jian <qiujian@yunionyun.com>

go.mod

@@ -90,7 +90,7 @@ require (
 	moul.io/http2curl/v2 v2.3.0
 	yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20240202081635-8602b7fb2751
 	yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32
-	yunion.io/x/jsonutils v1.0.1-0.20230613121553-0f3b41e2ef19
+	yunion.io/x/jsonutils v1.0.1-0.20240203102553-4096f103b401
 	yunion.io/x/log v1.0.1-0.20230411060016-feb3f46ab361
 	yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900
 	yunion.io/x/pkg v1.10.1-0.20240127153242-cdf9dc071f4f

go.sum

@@ -1206,8 +1206,8 @@ yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20240202081635-8602b7fb2751/go.mod h1:d
 yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32 h1:v7POYkQwo1XzOxBoIoRVr/k0V9Y5JyjpshlIFa9raug=
 yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
-yunion.io/x/jsonutils v1.0.1-0.20230613121553-0f3b41e2ef19 h1:mZh6vuo7oOiOtE5HF4667+XBNnn7uYpmGN5eIgCVMLk=
-yunion.io/x/jsonutils v1.0.1-0.20230613121553-0f3b41e2ef19/go.mod h1:VK4Z93dgiKgAijcSqbMKmGaBMJuHulR16Hz4K015ZPo=
+yunion.io/x/jsonutils v1.0.1-0.20240203102553-4096f103b401 h1:4l6ELFSQ0MBVInscZ8/yOtSWF0cwH5BT1ATN6dCtAqc=
+yunion.io/x/jsonutils v1.0.1-0.20240203102553-4096f103b401/go.mod h1:VK4Z93dgiKgAijcSqbMKmGaBMJuHulR16Hz4K015ZPo=
 yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
 yunion.io/x/log v1.0.1-0.20230411060016-feb3f46ab361 h1:c5LyNdhbvBe/92pXs9jgXOU/S+RgYh/DHe538LpT/Mo=

pkg/apis/identity/input.go

@@ -426,14 +426,14 @@ type IdentityProviderUpdateInput struct {
 
 type PolicyTagInput struct {
 	// 匹配的资源标签
-	ObjectTags tagutils.TTagSet `json:"object_tags"`
+	ObjectTags tagutils.TTagSet `json:"object_tags,allowempty"`
 	// 匹配的项目标签
-	ProjectTags tagutils.TTagSet `json:"project_tags"`
+	ProjectTags tagutils.TTagSet `json:"project_tags,allowempty"`
 	// 匹配的域标签
-	DomainTags tagutils.TTagSet `json:"domain_tags"`
+	DomainTags tagutils.TTagSet `json:"domain_tags,allowempty"`
 
 	// 组织架构节点ID
-	OrgNodeId []string `json:"org_node_id"`
+	OrgNodeId []string `json:"org_node_id,allowempty"`
 }
 
 type PolicyUpdateInput struct {

pkg/cloudcommon/db/db_dispatcher.go

@@ -266,7 +266,10 @@ func listItemQueryFiltersRaw(manager IModelManager,
 		return nil, httperrors.NewGeneralError(err)
 	}
 
-	query.(*jsonutils.JSONDict).Update(policyTagFilters.Json())
+	if !policyTagFilters.IsEmpty() {
+		query.(*jsonutils.JSONDict).Update(policyTagFilters.Json())
+		log.Debugf("policyTagFilers: %s", query)
+	}
 
 	if !useRawQuery {
 		// Specifically for joint resource, these filters will exclude
@@ -1745,11 +1748,11 @@ func reflectDispatcherInternal(
 	} else {
 		if model != nil {
 			if _, ok := model.(IStandaloneModel); ok {
-				Metadata.rawSetValues(ctx, model.Keyword(), model.GetId(), tagutils.Tagset2MapString(result.ObjectTags.Flattern()), false, "")
+				Metadata.rawSetValues(ctx, model.Keyword(), model.GetId(), tagutils.TagsetMap2MapString(result.ObjectTags.Flattern()), false, "")
 				if model.Keyword() == "project" {
-					Metadata.rawSetValues(ctx, model.Keyword(), model.GetId(), tagutils.Tagset2MapString(result.ProjectTags.Flattern()), false, "")
+					Metadata.rawSetValues(ctx, model.Keyword(), model.GetId(), tagutils.TagsetMap2MapString(result.ProjectTags.Flattern()), false, "")
 				} else if model.Keyword() == "domain" {
-					Metadata.rawSetValues(ctx, model.Keyword(), model.GetId(), tagutils.Tagset2MapString(result.DomainTags.Flattern()), false, "")
+					Metadata.rawSetValues(ctx, model.Keyword(), model.GetId(), tagutils.TagsetMap2MapString(result.DomainTags.Flattern()), false, "")
 				}
 			}
 		}

pkg/cloudcommon/db/standalone_anon.go

@@ -722,12 +722,12 @@ func (model *SStandaloneAnonResourceBase) applyPolicyTags(ctx context.Context, u
 	data.Unmarshal(&tags)
 	log.Debugf("applyPolicyTags: %s", jsonutils.Marshal(tags))
 	if len(tags.PolicyObjectTags) > 0 {
-		model.PerformMetadata(ctx, userCred, nil, tagutils.Tagset2MapString(tags.PolicyObjectTags.Flattern()))
+		model.PerformMetadata(ctx, userCred, nil, tagutils.TagsetMap2MapString(tags.PolicyObjectTags.Flattern()))
 	}
 	if model.Keyword() == "project" && len(tags.PolicyProjectTags) > 0 {
-		model.PerformMetadata(ctx, userCred, nil, tagutils.Tagset2MapString(tags.PolicyProjectTags.Flattern()))
+		model.PerformMetadata(ctx, userCred, nil, tagutils.TagsetMap2MapString(tags.PolicyProjectTags.Flattern()))
 	} else if model.Keyword() == "domain" && len(tags.PolicyDomainTags) > 0 {
-		model.PerformMetadata(ctx, userCred, nil, tagutils.Tagset2MapString(tags.PolicyDomainTags.Flattern()))
+		model.PerformMetadata(ctx, userCred, nil, tagutils.TagsetMap2MapString(tags.PolicyDomainTags.Flattern()))
 	}
 }
 

pkg/cloudcommon/policy/policy.go

@@ -363,6 +363,9 @@ func (manager *SPolicyManager) allowWithoutCache(policies rbacutils.TPolicySet,
 		log.Warningf("no policies fetched for scope %s", scope)
 	} else {
 		matchRules = policies.GetMatchRules(service, resource, action, extra...)
+		if consts.IsRbacDebug() {
+			log.Debugf("service %s resource %s action %s extra %s matchRules: %s", service, resource, action, jsonutils.Marshal(extra), jsonutils.Marshal(matchRules))
+		}
 	}
 
 	scopedDeny := false
@@ -393,26 +396,31 @@ func (manager *SPolicyManager) allowWithoutCache(policies rbacutils.TPolicySet,
 		matchRules = append(matchRules, rule)
 	}
 
-	// try default policies
-	defaultPolicies, ok := manager.defaultPolicies[scope]
-	if ok {
-		for i := range defaultPolicies {
-			isMatched, _ := defaultPolicies[i].Match(userCred)
-			if !isMatched {
-				continue
-			}
-			rule := defaultPolicies[i].Rules.GetMatchRule(service, resource, action, extra...)
-			if rule != nil {
-				matchRules = append(matchRules,
-					rbacutils.SPolicyMatch{
-						Rule: *rule,
-					},
-				)
+	result := matchRules.GetResult()
+	if result.Result.IsDeny() {
+		// denied, try default policies
+		defaultPolicies, ok := manager.defaultPolicies[scope]
+		if ok {
+			for i := range defaultPolicies {
+				isMatched, _ := defaultPolicies[i].Match(userCred)
+				if !isMatched {
+					continue
+				}
+				rule := defaultPolicies[i].Rules.GetMatchRule(service, resource, action, extra...)
+				if rule != nil {
+					if consts.IsRbacDebug() {
+						log.Debugf("service: %s resource: %s action: %s extra: %s match default policy: %s match rule: %s", service, resource, action, jsonutils.Marshal(extra), jsonutils.Marshal(defaultPolicies[i]), jsonutils.Marshal(rule))
+					}
+					matchRules = append(matchRules,
+						rbacutils.SPolicyMatch{
+							Rule: *rule,
+						},
+					)
+				}
 			}
 		}
+		result = matchRules.GetResult()
 	}
-
-	result := matchRules.GetResult()
 	if consts.IsRbacDebug() {
 		log.Debugf("[RBAC: %s] %s %s %s %s permission %s userCred: %s MatchRules: %d(%s)", scope, service, resource, action, jsonutils.Marshal(extra), result, userCred, len(matchRules), jsonutils.Marshal(matchRules))
 	}

pkg/keystone/models/policies.go

@@ -448,10 +448,12 @@ func (policy *SPolicy) ValidateUpdateData(ctx context.Context, userCred mcclient
 		input.OrgNodeId = nodeIds
 	case api.TAG_UPDATE_POLICY_REPLACE:
 		// do nothing
+		tagChanged = true
 	default:
 		for i := range policy.OrgNodeId {
 			if !utils.IsInArray(policy.OrgNodeId[i], input.OrgNodeId) {
 				input.OrgNodeId = append(input.OrgNodeId, policy.OrgNodeId[i])
+				tagChanged = true
 			}
 		}
 	}

pkg/keystone/models/projects.go

@@ -249,6 +249,8 @@ func (manager *SProjectManager) ListItemFilter(
 	userCred mcclient.TokenCredential,
 	query api.ProjectListInput,
 ) (*sqlchemy.SQuery, error) {
+	log.Debugf("ProjectManager ListItemFilter query %s", jsonutils.Marshal(query).String())
+
 	var err error
 
 	q, err = manager.SIdentityBaseResourceManager.ListItemFilter(ctx, q, userCred, query.IdentityBaseResourceListInput)

pkg/keystone/models/rolepolicies.go

@@ -472,6 +472,9 @@ func (manager *SRolePolicyManager) GetMatchPolicyGroupByCred(userCred api.IRbacI
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "GetMatchPolicyGroup")
 	}
+	if !options.Options.EnableDefaultDashboardPolicy {
+		return names, policies, nil
+	}
 	userId := userCred.GetUserId()
 	if len(userId) == 0 {
 		// anonymous access

pkg/keystone/options/options.go

@@ -60,9 +60,10 @@ type SKeystoneOptions struct {
 	DomainAdminRoleToNotify string `help:"domain admin role to notify" default:"domainadmin"`
 	AdminRoleToNotify       string `help:"admin role to notify" default:"admin"`
 
-	SystemDashboardPolicy  string `help:"dashboard policy name for system view" default:""`
-	DomainDashboardPolicy  string `help:"dashboard policy name for domain view" default:""`
-	ProjectDashboardPolicy string `help:"dashboard policy name for project view" default:""`
+	EnableDefaultDashboardPolicy bool   `default:"true" help:"enable default dashboard policy"`
+	SystemDashboardPolicy        string `help:"dashboard policy name for system view" default:""`
+	DomainDashboardPolicy        string `help:"dashboard policy name for domain view" default:""`
+	ProjectDashboardPolicy       string `help:"dashboard policy name for project view" default:""`
 
 	NoPolicyViolationCheck bool `help:"do not check policy violation when modify or assign policy" default:"false"`
 

pkg/mcclient/auth/auth.go

@@ -190,7 +190,7 @@ func (a *authManager) refreshRevokeTokens(ctx context.Context) error {
 	if a.adminCredential == nil {
 		return fmt.Errorf("refreshRevokeTokens: No valid admin token credential")
 	}
-	tokens, err := a.client.FetchInvalidTokens(ctx, a.adminCredential.GetTokenString())
+	tokens, err := a.client.FetchInvalidTokens(getContext(ctx), a.adminCredential.GetTokenString())
 	if err != nil {
 		return errors.Wrap(err, "client.FetchInvalidTokens")
 	}
@@ -344,6 +344,17 @@ func (a *authManager) getAdminSession(ctx context.Context, region, zone, endpoin
 	return a.getSession(ctx, manager.adminCredential, region, zone, endpointType)
 }
 
+func getContext(ctx context.Context) context.Context {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	srvType := consts.GetServiceType()
+	if len(srvType) > 0 && len(appctx.AppContextServiceName(ctx)) == 0 {
+		ctx = context.WithValue(ctx, appctx.APP_CONTEXT_KEY_APPNAME, srvType)
+	}
+	return ctx
+}
+
 func (a *authManager) getSession(ctx context.Context, token mcclient.TokenCredential, region, zone, endpointType string) *mcclient.ClientSession {
 	cli := Client()
 	if cli == nil {
@@ -352,11 +363,7 @@ func (a *authManager) getSession(ctx context.Context, token mcclient.TokenCreden
 	if endpointType == "" && globalEndpointType != "" {
 		endpointType = globalEndpointType
 	}
-	srvType := consts.GetServiceType()
-	if len(srvType) > 0 && len(appctx.AppContextServiceName(ctx)) == 0 {
-		ctx = context.WithValue(ctx, appctx.APP_CONTEXT_KEY_APPNAME, srvType)
-	}
-	return cli.NewSession(ctx, region, zone, endpointType, token)
+	return cli.NewSession(getContext(ctx), region, zone, endpointType, token)
 }
 
 func GetCatalogData(serviceTypes []string, region string) jsonutils.JSONObject {

pkg/util/rbacutils/policytags.go

@@ -48,9 +48,9 @@ func (policy SPolicy) GetMatchRule(service string, resource string, action strin
 
 func DecodePolicy(policyJson jsonutils.JSONObject) (*SPolicy, error) {
 	tags := []tagutils.TTagSetList{
-		tagutils.TTagSetList{}, // domain
-		tagutils.TTagSetList{}, // project
-		tagutils.TTagSetList{}, // resource
+		{}, // domain
+		{}, // project
+		{}, // resource
 	}
 	for i, key := range []string{
 		DomainTagsKey,
@@ -63,9 +63,9 @@ func DecodePolicy(policyJson jsonutils.JSONObject) (*SPolicy, error) {
 				tmpTagSet := make(tagutils.TTagSet, 0)
 				err2 := policyJson.Unmarshal(&tmpTagSet, key)
 				if err2 == nil {
-					tags[i] = tags[i].Append(tmpTagSet)
+					tags[i] = append(tags[i], tmpTagSet)
 				} else {
-					return nil, errors.Wrapf(errors.NewAggregate([]error{err, err2}), "Unmarshal %s", key)
+					return nil, errors.Wrapf(errors.NewAggregate([]error{err, err2}), "Unmarshal TTagSetList %s", key)
 				}
 			}
 		}
@@ -98,14 +98,21 @@ func DecodePolicyData(domainTags, projectTags, objectTags tagutils.TTagSetList,
 func (policy SPolicy) Encode() jsonutils.JSONObject {
 	ret := rules2Json(policy.Rules)
 	if dict, ok := ret.(*jsonutils.JSONDict); ok {
-		if len(policy.DomainTags) > 0 {
+		// In order to make compatible with old policy client that supports TTagset
+		if len(policy.DomainTags) > 1 {
 			dict.Add(jsonutils.Marshal(policy.DomainTags), DomainTagsKey)
+		} else if len(policy.DomainTags) == 1 {
+			dict.Add(jsonutils.Marshal(policy.DomainTags[0]), DomainTagsKey)
 		}
-		if len(policy.ProjectTags) > 0 {
+		if len(policy.ProjectTags) > 1 {
 			dict.Add(jsonutils.Marshal(policy.ProjectTags), ProjectTagsKey)
+		} else if len(policy.ProjectTags) == 1 {
+			dict.Add(jsonutils.Marshal(policy.ProjectTags[0]), ProjectTagsKey)
 		}
-		if len(policy.ObjectTags) > 0 {
+		if len(policy.ObjectTags) > 1 {
 			dict.Add(jsonutils.Marshal(policy.ObjectTags), ObjectTagsKey)
+		} else if len(policy.ObjectTags) == 1 {
+			dict.Add(jsonutils.Marshal(policy.ObjectTags[0]), ObjectTagsKey)
 		}
 	} else {
 		log.Fatalf("rule2Json output a NonJSON???")

pkg/util/rbacutils/policytags_test.go

@@ -17,6 +17,8 @@ package rbacutils
 import (
 	"testing"
 
+	"yunion.io/x/jsonutils"
+
 	"yunion.io/x/onecloud/pkg/util/tagutils"
 )
 
@@ -137,3 +139,89 @@ func TestSPolicy_Contains(t *testing.T) {
 		}
 	}
 }
+
+func TestDecodePolicy(t *testing.T) {
+	cases := []struct {
+		policy SPolicy
+	}{
+		{
+			policy: SPolicy{
+				Rules: TPolicy{
+					SRbacRule{
+						Service:  WILD_MATCH,
+						Resource: WILD_MATCH,
+						Action:   WILD_MATCH,
+						Result:   Allow,
+					},
+				},
+			},
+		},
+		{
+			policy: SPolicy{
+				Rules: TPolicy{
+					SRbacRule{
+						Service:  WILD_MATCH,
+						Resource: WILD_MATCH,
+						Action:   WILD_MATCH,
+						Result:   Allow,
+					},
+				},
+				ProjectTags: tagutils.TTagSetList{
+					tagutils.TTagSet{
+						tagutils.STag{
+							Key:   "user:部门",
+							Value: "技术",
+						},
+						tagutils.STag{
+							Key:   "user:环境",
+							Value: "UAT",
+						},
+					},
+				},
+			},
+		},
+		{
+			policy: SPolicy{
+				Rules: TPolicy{
+					SRbacRule{
+						Service:  WILD_MATCH,
+						Resource: WILD_MATCH,
+						Action:   WILD_MATCH,
+						Result:   Allow,
+					},
+				},
+				ProjectTags: tagutils.TTagSetList{
+					tagutils.TTagSet{
+						tagutils.STag{
+							Key:   "user:部门",
+							Value: "技术",
+						},
+						tagutils.STag{
+							Key:   "user:环境",
+							Value: "UAT",
+						},
+					},
+					tagutils.TTagSet{
+						tagutils.STag{
+							Key:   "org:部门",
+							Value: "技术",
+						},
+						tagutils.STag{
+							Key:   "org:环境",
+							Value: "UAT",
+						},
+					},
+				},
+			},
+		},
+	}
+	for _, c := range cases {
+		json := c.policy.Encode()
+		got, err := DecodePolicy(json)
+		if err != nil {
+			t.Errorf("DecodePolicy fail %s", err)
+		} else if jsonutils.Marshal(c.policy).String() != jsonutils.Marshal(got).String() {
+			t.Errorf("want %s got %s", jsonutils.Marshal(c.policy), jsonutils.Marshal(got))
+		}
+	}
+}

pkg/util/rbacutils/results.go

@@ -85,11 +85,21 @@ func (result SPolicyResult) String() string {
 	return fmt.Sprintf("[%s] domain:%s project:%s object:%s", result.Result, result.DomainTags.String(), result.ProjectTags.String(), result.ObjectTags.String())
 }
 
+func (result SPolicyResult) IsEmpty() bool {
+	return len(result.ObjectTags) == 0 && len(result.ProjectTags) == 0 && len(result.DomainTags) == 0
+}
+
 func (result SPolicyResult) Json() jsonutils.JSONObject {
 	ret := jsonutils.NewDict()
-	ret.Add(jsonutils.Marshal(result.ObjectTags), "policy_object_tags")
-	ret.Add(jsonutils.Marshal(result.ProjectTags), "policy_project_tags")
-	ret.Add(jsonutils.Marshal(result.DomainTags), "policy_domain_tags")
+	if len(result.ObjectTags) > 0 {
+		ret.Add(jsonutils.Marshal(result.ObjectTags), "policy_object_tags")
+	}
+	if len(result.ProjectTags) > 0 {
+		ret.Add(jsonutils.Marshal(result.ProjectTags), "policy_project_tags")
+	}
+	if len(result.DomainTags) > 0 {
+		ret.Add(jsonutils.Marshal(result.DomainTags), "policy_domain_tags")
+	}
 	return ret
 }
 

pkg/util/tagutils/tag.go

@@ -14,6 +14,11 @@
 
 package tagutils
 
+import (
+	"fmt"
+	"strings"
+)
+
 const (
 	NoValue  = "___no_value__"
 	AnyValue = ""
@@ -26,6 +31,18 @@ type STag struct {
 	Value string `json:"value"`
 }
 
+func (t STag) String() string {
+	return fmt.Sprintf("%s=%s", t.Key, t.Value)
+}
+
+func (t STag) KeyPrefix() string {
+	commaPos := strings.Index(t.Key, ":")
+	if commaPos > 0 {
+		return t.Key[:commaPos]
+	}
+	return ""
+}
+
 func Compare(t1, t2 STag) int {
 	if t1.Key < t2.Key {
 		return -1

pkg/util/tagutils/tag_test.go

@@ -89,3 +89,38 @@ func TestSTagCompare(t *testing.T) {
 		}
 	}
 }
+
+func TestKeyPrefix(t *testing.T) {
+	cases := []struct {
+		tag  STag
+		want string
+	}{
+		{
+			tag: STag{
+				Key:   "user:abc",
+				Value: "1",
+			},
+			want: "user",
+		},
+		{
+			tag: STag{
+				Key:   "org:abc",
+				Value: "1",
+			},
+			want: "org",
+		},
+		{
+			tag: STag{
+				Key:   "cls:abc",
+				Value: "1",
+			},
+			want: "cls",
+		},
+	}
+	for _, c := range cases {
+		got := c.tag.KeyPrefix()
+		if got != c.want {
+			t.Errorf("tag %s keyprefix %s got %s", c.tag.String(), c.want, got)
+		}
+	}
+}

pkg/util/tagutils/tagset.go

@@ -30,6 +30,22 @@ func (t TTagSet) IsZero() bool {
 	return len(t) == 0
 }
 
+func (ts TTagSet) KeyPrefix() string {
+	var pref *string
+	for i := range ts {
+		prefix := ts[i].KeyPrefix()
+		if pref == nil {
+			pref = &prefix
+		} else if *pref != prefix {
+			return ""
+		}
+	}
+	if pref != nil {
+		return *pref
+	}
+	return ""
+}
+
 func (ts TTagSet) index(needle STag) (int, bool) {
 	i := 0
 	j := len(ts) - 1
@@ -220,3 +236,14 @@ func Tagset2MapString(oTags TTagSet) map[string]string {
 	}
 	return tags
 }
+
+func TagsetMap2MapString(oTags map[string]TTagSet) map[string]string {
+	ret := make(map[string]string)
+	for k := range oTags {
+		keyMap := Tagset2MapString(oTags[k])
+		for k, v := range keyMap {
+			ret[k] = v
+		}
+	}
+	return ret
+}

pkg/util/tagutils/tagset_test.go

@@ -575,3 +575,61 @@ func TestTagSetAppend(t *testing.T) {
 		}
 	}
 }
+
+func TestTagSetKeyPrefix(t *testing.T) {
+	cases := []struct {
+		tags TTagSet
+		want string
+	}{
+		{
+			tags: TTagSet{
+				STag{
+					Key:   "user:abc",
+					Value: "1",
+				},
+				STag{
+					Key:   "user:efd",
+					Value: "1",
+				},
+			},
+			want: "user",
+		},
+		{
+			tags: TTagSet{
+				STag{
+					Key:   "org:abc",
+					Value: "1",
+				},
+			},
+			want: "org",
+		},
+		{
+			tags: TTagSet{
+				STag{
+					Key:   "cls:abc",
+					Value: "1",
+				},
+			},
+			want: "cls",
+		},
+		{
+			tags: TTagSet{
+				STag{
+					Key:   "cls:abc",
+					Value: "1",
+				},
+				STag{
+					Key:   "user:abc",
+					Value: "1",
+				},
+			},
+			want: "",
+		},
+	}
+	for _, c := range cases {
+		got := c.tags.KeyPrefix()
+		if got != c.want {
+			t.Errorf("tag %s keyprefix %s got %s", c.tags.String(), c.want, got)
+		}
+	}
+}

pkg/util/tagutils/tagsetlist.go

@@ -137,10 +137,25 @@ func (a TTagSetList) Less(i, j int) bool {
 	return false
 }
 
-func (tsl TTagSetList) Flattern() TTagSet {
+func (tsl TTagSetList) Flattern() map[string]TTagSet {
 	if len(tsl) == 0 {
-		return TTagSet{}
+		return nil
 	}
-	sort.Sort(tsl)
-	return tsl[len(tsl)-1]
+
+	splitMap := make(map[string]TTagSetList)
+	for i := range tsl {
+		prefix := tsl[i].KeyPrefix()
+		if ts, ok := splitMap[prefix]; ok {
+			splitMap[prefix] = ts.Append(tsl[i])
+		} else {
+			splitMap[prefix] = TTagSetList{tsl[i]}
+		}
+	}
+	ret := make(map[string]TTagSet)
+	for k := range splitMap {
+		tsl := splitMap[k]
+		sort.Sort(tsl)
+		ret[k] = tsl[len(tsl)-1]
+	}
+	return ret
 }

pkg/util/tagutils/tagsetlist_test.go

@@ -252,7 +252,7 @@ func TestTTagSetList_Append(t *testing.T) {
 func TestTTagSetList_Flattern(t *testing.T) {
 	cases := []struct {
 		tsl  TTagSetList
-		want TTagSet
+		want map[string]TTagSet
 	}{
 		{
 			tsl: TTagSetList{
@@ -273,14 +273,16 @@ func TestTTagSetList_Flattern(t *testing.T) {
 					},
 				},
 			},
-			want: TTagSet{
-				STag{
-					Key:   "project",
-					Value: "a",
-				},
-				STag{
-					Key:   "env",
-					Value: "product",
+			want: map[string]TTagSet{
+				"": {
+					STag{
+						Key:   "project",
+						Value: "a",
+					},
+					STag{
+						Key:   "env",
+						Value: "product",
+					},
 				},
 			},
 		},
@@ -427,3 +429,149 @@ func TestIntersects(t *testing.T) {
 		}
 	}
 }
+
+func TestFlattern(t *testing.T) {
+	cases := []struct {
+		tsl  TTagSetList
+		want map[string]TTagSet
+	}{
+		{
+			tsl: TTagSetList{
+				TTagSet{
+					STag{
+						Key:   "user:部门",
+						Value: "技术",
+					},
+					STag{
+						Key:   "user:环境",
+						Value: "UAT",
+					},
+				},
+				TTagSet{
+					STag{
+						Key:   "org:部门",
+						Value: "技术",
+					},
+					STag{
+						Key:   "org:环境",
+						Value: "UAT",
+					},
+				},
+			},
+			want: map[string]TTagSet{
+				"user": {
+					STag{
+						Key:   "user:部门",
+						Value: "技术",
+					},
+					STag{
+						Key:   "user:环境",
+						Value: "UAT",
+					},
+				},
+				"org": {
+					STag{
+						Key:   "org:部门",
+						Value: "技术",
+					},
+					STag{
+						Key:   "org:环境",
+						Value: "UAT",
+					},
+				},
+			},
+		},
+		{
+			tsl: TTagSetList{
+				TTagSet{
+					STag{
+						Key:   "user:国家",
+						Value: "中国",
+					},
+					STag{
+						Key:   "user:城市",
+						Value: "天津",
+					},
+					STag{
+						Key:   "user:部门",
+						Value: "技术",
+					},
+					STag{
+						Key:   "user:环境",
+						Value: "Product",
+					},
+				},
+				TTagSet{
+					STag{
+						Key:   "user:部门",
+						Value: "技术",
+					},
+					STag{
+						Key:   "user:环境",
+						Value: "UAT",
+					},
+				},
+				TTagSet{
+					STag{
+						Key:   "user:城市",
+						Value: "北京",
+					},
+					STag{
+						Key:   "user:部门",
+						Value: "研发",
+					},
+					STag{
+						Key:   "user:环境",
+						Value: "UAT",
+					},
+				},
+				TTagSet{
+					STag{
+						Key:   "org:部门",
+						Value: "技术",
+					},
+					STag{
+						Key:   "org:环境",
+						Value: "UAT",
+					},
+				},
+			},
+			want: map[string]TTagSet{
+				"user": {
+					STag{
+						Key:   "user:国家",
+						Value: "中国",
+					},
+					STag{
+						Key:   "user:城市",
+						Value: "天津",
+					},
+					STag{
+						Key:   "user:部门",
+						Value: "技术",
+					},
+					STag{
+						Key:   "user:环境",
+						Value: "Product",
+					},
+				},
+				"org": {
+					STag{
+						Key:   "org:部门",
+						Value: "技术",
+					},
+					STag{
+						Key:   "org:环境",
+						Value: "UAT",
+					},
+				},
+			},
+		},
+	}
+	for _, c := range cases {
+		got := c.tsl.Flattern()
+		if jsonutils.Marshal(got).String() != jsonutils.Marshal(c.want).String() {
+			t.Errorf("tsl %s flattern got %s want %s", jsonutils.Marshal(c.tsl), jsonutils.Marshal(got), jsonutils.Marshal(c.want))
+		}
+	}
+}

vendor/modules.txt

@@ -1548,7 +1548,7 @@ yunion.io/x/cloudmux/pkg/multicloud/zstack/provider
 yunion.io/x/executor/apis
 yunion.io/x/executor/client
 yunion.io/x/executor/server
-# yunion.io/x/jsonutils v1.0.1-0.20230613121553-0f3b41e2ef19
+# yunion.io/x/jsonutils v1.0.1-0.20240203102553-4096f103b401
 ## explicit; go 1.18
 yunion.io/x/jsonutils
 # yunion.io/x/log v1.0.1-0.20230411060016-feb3f46ab361

vendor/yunion.io/x/jsonutils/unmarshal.go

@@ -455,7 +455,7 @@ func (this *JSONArray) _unmarshalValue(s *sJsonUnmarshalSession, val reflect.Val
 			}
 		} else if val.Kind() == reflect.Slice {
 			dataLen := len(this.data)
-			if val.Cap() < dataLen {
+			if val.IsNil() || val.Cap() < dataLen {
 				newVal := reflect.MakeSlice(val.Type(), dataLen, dataLen)
 				val.Set(newVal)
 			} else if val.Len() != dataLen {