From aabdb3fa97bc7918ba9de386fa4afc6224da4e3d Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Thu, 27 Jun 2019 16:35:32 +0800 Subject: [PATCH] fix: misc bugfixes. --- cmd/climc/shell/identityproviders.go | 43 ++++++++++++ pkg/apis/identity/config.go | 5 +- pkg/apis/identity/consts.go | 1 + pkg/cloudcommon/db/fetch.go | 4 ++ pkg/cloudcommon/db/tenantcache.go | 25 ++++--- pkg/keystone/driver/base.go | 11 ++- pkg/keystone/driver/driver.go | 2 +- pkg/keystone/driver/ldap/class.go | 4 +- pkg/keystone/driver/ldap/ldap.go | 19 ++++-- pkg/keystone/driver/ldap/sync.go | 67 +++++++++++++++---- pkg/keystone/driver/ldap/template.go | 6 -- pkg/keystone/driver/sql/class.go | 4 +- pkg/keystone/driver/sql/sql.go | 4 +- pkg/keystone/models/domains.go | 23 +++++-- pkg/keystone/models/expandidps.go | 19 +++--- pkg/keystone/models/groups.go | 4 ++ pkg/keystone/models/id_mappings.go | 7 +- pkg/keystone/models/identity_provider.go | 27 +++++++- pkg/keystone/models/syncworker.go | 2 +- pkg/keystone/models/users.go | 34 ++++++++++ .../tasks/identity_provider_delete_task.go | 2 + pkg/keystone/tokens/auth.go | 2 +- pkg/util/stringutils2/i18n.go | 24 +++++++ pkg/util/stringutils2/i18n_test.go | 35 ++++++++++ 24 files changed, 307 insertions(+), 67 deletions(-) create mode 100644 pkg/util/stringutils2/i18n.go create mode 100644 pkg/util/stringutils2/i18n_test.go diff --git a/cmd/climc/shell/identityproviders.go b/cmd/climc/shell/identityproviders.go index a5cc0cdeba..95fec712a0 100644 --- a/cmd/climc/shell/identityproviders.go +++ b/cmd/climc/shell/identityproviders.go @@ -116,11 +116,27 @@ func init() { type IdentityProviderCreateOptions struct { NAME string `help:"name of identity provider" json:"-"` + + AutoCreateProject bool `help:"automatically create a default project when importing domain" json:"-"` + NoAutoCreateProject bool `help:"do not create default project when importing domain" json:"-"` + + TargetDomain string `help:"target domain without creating new domain"` + api.SLDAPIdpConfigOptions } R(&IdentityProviderCreateOptions{}, "idp-create-ldap", "Create an identity provider with LDAP driver", func(s *mcclient.ClientSession, args *IdentityProviderCreateOptions) error { params := jsonutils.NewDict() params.Add(jsonutils.NewString(args.NAME), "name") + + if len(args.TargetDomain) > 0 { + params.Add(jsonutils.NewString(args.TargetDomain), "target_domain") + } + if args.AutoCreateProject { + params.Add(jsonutils.JSONTrue, "auto_create_project") + } else if args.NoAutoCreateProject { + params.Add(jsonutils.JSONFalse, "auto_create_project") + } + params.Add(jsonutils.NewString("ldap"), "driver") params.Add(jsonutils.Marshal(args), "config", "ldap") @@ -150,6 +166,12 @@ func init() { type IdentityProviderCreateLDAPSingleDomainOptions struct { NAME string `help:"name of identity provider" json:"-"` TEMPLATE string `help:"configuration template name" choices:"msad_one_domain|openldap_one_domain" json:"-"` + + AutoCreateProject bool `help:"automatically create a default project when importing domain" json:"-"` + NoAutoCreateProject bool `help:"do not create default project when importing domain" json:"-"` + + TargetDomain string `help:"target domain without creating new domain"` + api.SLDAPIdpConfigSingleDomainOptions } R(&IdentityProviderCreateLDAPSingleDomainOptions{}, "idp-create-ldap-single-domain", "Create an identity provider with LDAP driver/single domain template", func(s *mcclient.ClientSession, args *IdentityProviderCreateLDAPSingleDomainOptions) error { @@ -157,6 +179,16 @@ func init() { params.Add(jsonutils.NewString(args.NAME), "name") params.Add(jsonutils.NewString("ldap"), "driver") params.Add(jsonutils.NewString(args.TEMPLATE), "template") + + if len(args.TargetDomain) > 0 { + params.Add(jsonutils.NewString(args.TargetDomain), "target_domain") + } + if args.AutoCreateProject { + params.Add(jsonutils.JSONTrue, "auto_create_project") + } else if args.NoAutoCreateProject { + params.Add(jsonutils.JSONFalse, "auto_create_project") + } + params.Add(jsonutils.Marshal(args), "config", "ldap") idp, err := modules.IdentityProviders.Create(s, params) @@ -185,6 +217,10 @@ func init() { type IdentityProviderCreateLDAPMultiDomainOptions struct { NAME string `help:"name of identity provider" json:"-"` TEMPLATE string `help:"configuration template name" choices:"msad_multi_domain" json:"-"` + + AutoCreateProject bool `help:"automatically create a default project when importing domain" json:"-"` + NoAutoCreateProject bool `help:"do not create default project when importing domain" json:"-"` + api.SLDAPIdpConfigMultiDomainOptions } R(&IdentityProviderCreateLDAPMultiDomainOptions{}, "idp-create-ldap-multi-domain", "Create an identity provider with LDAP driver/single domain template", func(s *mcclient.ClientSession, args *IdentityProviderCreateLDAPMultiDomainOptions) error { @@ -192,6 +228,13 @@ func init() { params.Add(jsonutils.NewString(args.NAME), "name") params.Add(jsonutils.NewString("ldap"), "driver") params.Add(jsonutils.NewString(args.TEMPLATE), "template") + + if args.AutoCreateProject { + params.Add(jsonutils.JSONTrue, "auto_create_project") + } else if args.NoAutoCreateProject { + params.Add(jsonutils.JSONFalse, "auto_create_project") + } + params.Add(jsonutils.Marshal(args), "config", "ldap") idp, err := modules.IdentityProviders.Create(s, params) diff --git a/pkg/apis/identity/config.go b/pkg/apis/identity/config.go index 017f620473..28eafbdb96 100644 --- a/pkg/apis/identity/config.go +++ b/pkg/apis/identity/config.go @@ -47,13 +47,10 @@ type SLDAPIdpConfigOptions struct { Url string `json:"url,omitempty" help:"LDAP server URL" required:"true"` Suffix string `json:"suffix,omitempty" required:"true"` QueryScope string `json:"query_scope,omitempty" help:"Query scope, either one or sub" choices:"one|sub"` - // PageSize int `json:"page_size,omitzero" help:"Page size, default 20" default:"20"` + User string `json:"user,omitempty"` Password string `json:"password,omitempty"` - AutoCreateProject bool `json:"auto_create_project,omitfalse"` - ImportDomain bool `json:"import_domain,omitfalse"` - DomainTreeDN string `json:"domain_tree_dn,omitempty" help:"Domain tree root node dn(distinguished name)"` DomainFilter string `json:"domain_filter,omitempty"` DomainObjectclass string `json:"domain_objectclass,omitempty"` diff --git a/pkg/apis/identity/consts.go b/pkg/apis/identity/consts.go index 07f5c2d342..36469e9436 100644 --- a/pkg/apis/identity/consts.go +++ b/pkg/apis/identity/consts.go @@ -59,6 +59,7 @@ const ( IdentityDriverStatusConnected = "connected" IdentityDriverStatusDisconnected = "disconnected" IdentityDriverStatusDeleting = "deleting" + IdentityDriverStatusDeleteFailed = "delete_fail" IdentityProviderSyncLocal = "local" IdentityProviderSyncFull = "full" diff --git a/pkg/cloudcommon/db/fetch.go b/pkg/cloudcommon/db/fetch.go index ede9eb14c5..4a665465de 100644 --- a/pkg/cloudcommon/db/fetch.go +++ b/pkg/cloudcommon/db/fetch.go @@ -29,6 +29,7 @@ import ( "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/rbacutils" + "yunion.io/x/onecloud/pkg/util/stringutils2" ) func FetchJointByIds(manager IJointModelManager, masterId, slaveId string, query jsonutils.JSONObject) (IJointModel, error) { @@ -127,6 +128,9 @@ func FetchByName(manager IModelManager, userCred mcclient.IIdentityProvider, idS } func FetchByIdOrName(manager IModelManager, userCred mcclient.IIdentityProvider, idStr string) (IModel, error) { + if stringutils2.IsUtf8(idStr) { + return FetchByName(manager, userCred, idStr) + } obj, err := FetchById(manager, idStr) if err == sql.ErrNoRows { return FetchByName(manager, userCred, idStr) diff --git a/pkg/cloudcommon/db/tenantcache.go b/pkg/cloudcommon/db/tenantcache.go index 5b289876f1..12109e1fc2 100644 --- a/pkg/cloudcommon/db/tenantcache.go +++ b/pkg/cloudcommon/db/tenantcache.go @@ -32,6 +32,7 @@ import ( "yunion.io/x/onecloud/pkg/mcclient/auth" "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/util/httputils" + "yunion.io/x/onecloud/pkg/util/stringutils2" ) type STenantCacheManager struct { @@ -123,10 +124,14 @@ func (manager *STenantCacheManager) fetchTenant(ctx context.Context, idStr strin func (manager *STenantCacheManager) FetchTenantByIdOrName(ctx context.Context, idStr string) (*STenant, error) { return manager.fetchTenant(ctx, idStr, false, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { - return q.Filter(sqlchemy.OR( - sqlchemy.Equals(q.Field("id"), idStr), - sqlchemy.Equals(q.Field("name"), idStr), - )) + if stringutils2.IsUtf8(idStr) { + return q.Equals("name", idStr) + } else { + return q.Filter(sqlchemy.OR( + sqlchemy.Equals(q.Field("id"), idStr), + sqlchemy.Equals(q.Field("name"), idStr), + )) + } }) } @@ -167,10 +172,14 @@ func (manager *STenantCacheManager) fetchTenantFromKeystone(ctx context.Context, func (manager *STenantCacheManager) FetchDomainByIdOrName(ctx context.Context, idStr string) (*STenant, error) { return manager.fetchTenant(ctx, idStr, true, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { - return q.Filter(sqlchemy.OR( - sqlchemy.Equals(q.Field("id"), idStr), - sqlchemy.Equals(q.Field("name"), idStr), - )) + if stringutils2.IsUtf8(idStr) { + return q.Equals("name", idStr) + } else { + return q.Filter(sqlchemy.OR( + sqlchemy.Equals(q.Field("id"), idStr), + sqlchemy.Equals(q.Field("name"), idStr), + )) + } }) } diff --git a/pkg/keystone/driver/base.go b/pkg/keystone/driver/base.go index d66701a806..18112abe23 100644 --- a/pkg/keystone/driver/base.go +++ b/pkg/keystone/driver/base.go @@ -34,12 +34,12 @@ func GetDriverClass(drv string) IIdentityBackendClass { return nil } -func GetDriver(driver string, idpId, idpName, template string, conf api.TIdentityProviderConfigs) (IIdentityBackend, error) { +func GetDriver(driver string, idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TIdentityProviderConfigs) (IIdentityBackend, error) { drvCls := GetDriverClass(driver) if drvCls == nil { return nil, ErrNoSuchDriver } - return drvCls.NewDriver(idpId, idpName, template, conf) + return drvCls.NewDriver(idpId, idpName, template, targetDomainId, autoCreateProject, conf) } type SBaseIdentityDriver struct { @@ -49,17 +49,22 @@ type SBaseIdentityDriver struct { IdpId string IdpName string Template string + + TargetDomainId string + AutoCreateProject bool } func (base *SBaseIdentityDriver) IIdentityBackend() IIdentityBackend { return base.GetVirtualObject().(IIdentityBackend) } -func NewBaseIdentityDriver(idpId, idpName, template string, conf api.TIdentityProviderConfigs) (SBaseIdentityDriver, error) { +func NewBaseIdentityDriver(idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TIdentityProviderConfigs) (SBaseIdentityDriver, error) { drv := SBaseIdentityDriver{} drv.IdpId = idpId drv.IdpName = idpName drv.Template = template + drv.TargetDomainId = targetDomainId + drv.AutoCreateProject = autoCreateProject drv.Config = conf return drv, nil } diff --git a/pkg/keystone/driver/driver.go b/pkg/keystone/driver/driver.go index 0e7fcb6ef6..4398a87e27 100644 --- a/pkg/keystone/driver/driver.go +++ b/pkg/keystone/driver/driver.go @@ -25,7 +25,7 @@ type IIdentityBackendClass interface { SingletonInstance() bool SyncMethod() string Name() string - NewDriver(idpId, idpName, template string, conf api.TIdentityProviderConfigs) (IIdentityBackend, error) + NewDriver(idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TIdentityProviderConfigs) (IIdentityBackend, error) } type IIdentityBackend interface { diff --git a/pkg/keystone/driver/ldap/class.go b/pkg/keystone/driver/ldap/class.go index ac7211f23a..0b0916b22a 100644 --- a/pkg/keystone/driver/ldap/class.go +++ b/pkg/keystone/driver/ldap/class.go @@ -29,8 +29,8 @@ func (self *SLDAPDriverClass) SyncMethod() string { return api.IdentityProviderSyncFull } -func (self *SLDAPDriverClass) NewDriver(idpId, idpName, template string, conf api.TIdentityProviderConfigs) (driver.IIdentityBackend, error) { - return NewLDAPDriver(idpId, idpName, template, conf) +func (self *SLDAPDriverClass) NewDriver(idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TIdentityProviderConfigs) (driver.IIdentityBackend, error) { + return NewLDAPDriver(idpId, idpName, template, targetDomainId, autoCreateProject, conf) } func (self *SLDAPDriverClass) Name() string { diff --git a/pkg/keystone/driver/ldap/ldap.go b/pkg/keystone/driver/ldap/ldap.go index ff10b891c9..dc18b95cae 100644 --- a/pkg/keystone/driver/ldap/ldap.go +++ b/pkg/keystone/driver/ldap/ldap.go @@ -21,10 +21,9 @@ import ( "gopkg.in/ldap.v3" - "github.com/pkg/errors" - "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" "yunion.io/x/pkg/utils" api "yunion.io/x/onecloud/pkg/apis/identity" @@ -40,8 +39,8 @@ type SLDAPDriver struct { ldapConfig *api.SLDAPIdpConfigOptions } -func NewLDAPDriver(idpId, idpName, template string, conf api.TIdentityProviderConfigs) (driver.IIdentityBackend, error) { - base, err := driver.NewBaseIdentityDriver(idpId, idpName, template, conf) +func NewLDAPDriver(idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TIdentityProviderConfigs) (driver.IIdentityBackend, error) { + base, err := driver.NewBaseIdentityDriver(idpId, idpName, template, targetDomainId, autoCreateProject, conf) if err != nil { return nil, errors.Wrap(err, "NewBaseIdentityDriver") } @@ -242,12 +241,20 @@ func (self *SLDAPDriver) Authenticate(ctx context.Context, ident mcclient.SAuthe } var userTreeDN string - if self.ldapConfig.ImportDomain { + if len(self.ldapConfig.DomainTreeDN) > 0 { + // import domains idMap, err := models.IdmappingManager.FetchEntity(usrExt.DomainId, api.IdMappingEntityDomain) if err != nil { return nil, errors.Wrap(err, "IdmappingManager.FetchEntity for domain") } - userTreeDN = idMap.IdpEntityId + entries, err := self.searchDomainEntries(cli, idMap.IdpEntityId) + if err != nil { + return nil, errors.Wrap(err, "self.searchDomainEntries") + } + if len(entries) == 0 { + return nil, errors.Error("fail to find domain DN") + } + userTreeDN = entries[0].DN } else { userTreeDN = self.getUserTreeDN() } diff --git a/pkg/keystone/driver/ldap/sync.go b/pkg/keystone/driver/ldap/sync.go index 13a67a87b3..9f6390998a 100644 --- a/pkg/keystone/driver/ldap/sync.go +++ b/pkg/keystone/driver/ldap/sync.go @@ -19,6 +19,8 @@ import ( "database/sql" "fmt" + "gopkg.in/ldap.v3" + "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/tristate" @@ -46,7 +48,7 @@ func (self *SLDAPDriver) Sync(ctx context.Context) error { } defer cli.Close() - if self.ldapConfig.ImportDomain { + if len(self.ldapConfig.DomainTreeDN) > 0 { return self.syncDomains(ctx, cli) } else { return self.syncSingleDomain(ctx, cli) @@ -54,10 +56,25 @@ func (self *SLDAPDriver) Sync(ctx context.Context) error { } func (self *SLDAPDriver) syncSingleDomain(ctx context.Context, cli *ldaputils.SLDAPClient) error { - domainInfo := SDomainInfo{DN: self.ldapConfig.Suffix, Id: api.DefaultRemoteDomainId, Name: self.IdpName} - domain, err := self.syncDomainInfo(ctx, domainInfo) - if err != nil { - return errors.Wrap(err, "syncDomainInfo") + var domain *models.SDomain + if len(self.TargetDomainId) > 0 { + targetDomain, err := models.DomainManager.FetchDomainById(self.TargetDomainId) + if err != nil && err != sql.ErrNoRows { + return errors.Wrap(err, "models.DomainManager.FetchDomainById") + } + if targetDomain == nil { + log.Warningln("target domain not exist!") + } else { + domain = targetDomain + } + } + if domain == nil { + domainInfo := SDomainInfo{DN: self.ldapConfig.Suffix, Id: api.DefaultRemoteDomainId, Name: self.IdpName} + newDomain, err := self.syncDomainInfo(ctx, domainInfo) + if err != nil { + return errors.Wrap(err, "syncDomainInfo") + } + domain = newDomain } userIdMap, err := self.syncUsers(ctx, cli, domain.Id, self.getUserTreeDN()) if err != nil { @@ -70,14 +87,22 @@ func (self *SLDAPDriver) syncSingleDomain(ctx context.Context, cli *ldaputils.SL return nil } -func (self *SLDAPDriver) syncDomains(ctx context.Context, cli *ldaputils.SLDAPClient) error { - entries, err := cli.Search(self.getDomainTreeDN(), +func (self *SLDAPDriver) searchDomainEntries(cli *ldaputils.SLDAPClient, domainid string) ([]*ldap.Entry, error) { + attrMap := make(map[string]string) + if len(domainid) > 0 { + attrMap[self.ldapConfig.DomainIdAttribute] = domainid + } + return cli.Search(self.getDomainTreeDN(), self.ldapConfig.DomainObjectclass, - nil, + attrMap, self.ldapConfig.DomainFilter, self.domainAttributeList(), self.domainQueryScope(), ) +} + +func (self *SLDAPDriver) syncDomains(ctx context.Context, cli *ldaputils.SLDAPClient) error { + entries, err := self.searchDomainEntries(cli, "") if err != nil { return errors.Wrap(err, "searchLDAP") } @@ -99,7 +124,7 @@ func (self *SLDAPDriver) syncDomains(ctx context.Context, cli *ldaputils.SLDAPCl } } // remove any obsolete domain Id_mappings - err = models.IdmappingManager.DeleteAny(self.IdpId, api.IdMappingEntityDomain, domainLocalIds) + err = models.IdmappingManager.DeleteAny(self.IdpId, api.IdMappingEntityDomain, domainLocalIds, nil) if err != nil { log.Errorf("delete remvoed remote domain fail %s", err) } @@ -155,7 +180,7 @@ func (self *SLDAPDriver) syncDomainInfo(ctx context.Context, info SDomainInfo) ( return nil, errors.Wrap(err, "insert") } - if self.ldapConfig.AutoCreateProject { + if self.AutoCreateProject { project := &models.SProject{} project.SetModelManager(models.ProjectManager, project) projectName := models.NormalizeProjectName(fmt.Sprintf("%s_default_project", info.Name)) @@ -205,9 +230,15 @@ func (self *SLDAPDriver) syncUsers(ctx context.Context, cli *ldaputils.SLDAPClie userIdMap[userInfo.DN] = userId } } - err = models.IdmappingManager.DeleteAny(self.IdpId, api.IdMappingEntityUser, userLocalIds) + deleteUsrIds, err := models.UserManager.FetchUserLocalIdsInDomain(domainId, userLocalIds) if err != nil { - log.Errorf("delete removed remote user fail %s", err) + return nil, errors.Wrap(err, "models.UserManager.FetchUserIdsInDomain") + } + if len(deleteUsrIds) > 0 { + err = models.IdmappingManager.DeleteAny(self.IdpId, api.IdMappingEntityUser, nil, deleteUsrIds) + if err != nil { + log.Errorf("delete removed remote user fail %s", err) + } } return userIdMap, nil } @@ -272,6 +303,8 @@ func (self *SLDAPDriver) syncUserDB(ctx context.Context, ui SUserInfo, domainId return "", errors.Wrap(err, "models.IdmappingManager.RegisterIdMap") } + log.Debugf("syncUserDB: %s", userId) + // insert nonlocal user err = registerNonlocalUser(ctx, ui, userId, domainId) if err != nil { @@ -301,9 +334,15 @@ func (self *SLDAPDriver) syncGroups(ctx context.Context, cli *ldaputils.SLDAPCli return errors.Wrap(err, "syncGroupDB") } } - err = models.IdmappingManager.DeleteAny(self.IdpId, api.IdMappingEntityGroup, groupLocalIds) + deleteGroupIds, err := models.GroupManager.FetchGroupLocalIdsInDomain(domainId, groupLocalIds) if err != nil { - log.Errorf("delete removed remote group fail %s", err) + return errors.Wrap(err, "models.GroupManager.FetchGroupIdsInDomain") + } + if len(groupLocalIds) > 0 { + err = models.IdmappingManager.DeleteAny(self.IdpId, api.IdMappingEntityGroup, nil, deleteGroupIds) + if err != nil { + log.Errorf("delete removed remote group fail %s", err) + } } return nil } diff --git a/pkg/keystone/driver/ldap/template.go b/pkg/keystone/driver/ldap/template.go index 8056e26888..ace8bce0f3 100644 --- a/pkg/keystone/driver/ldap/template.go +++ b/pkg/keystone/driver/ldap/template.go @@ -20,8 +20,6 @@ import ( var ( MicrosoftActiveDirectorySingleDomainTemplate = api.SLDAPIdpConfigOptions{ - ImportDomain: false, - AutoCreateProject: true, UserObjectclass: "organizationalPerson", UserIdAttribute: "sAMAccountName", UserNameAttribute: "sAMAccountName", @@ -44,8 +42,6 @@ var ( } MicrosoftActiveDirectoryMultipleDomainTemplate = api.SLDAPIdpConfigOptions{ - ImportDomain: true, - AutoCreateProject: true, DomainObjectclass: "organizationalUnit", DomainIdAttribute: "objectGUID", DomainNameAttribute: "name", @@ -72,8 +68,6 @@ var ( } OpenLdapSingleDomainTemplate = api.SLDAPIdpConfigOptions{ - ImportDomain: false, - AutoCreateProject: true, UserObjectclass: "person", UserIdAttribute: "uid", UserNameAttribute: "uid", diff --git a/pkg/keystone/driver/sql/class.go b/pkg/keystone/driver/sql/class.go index 5ac9806b53..9b6bda2a71 100644 --- a/pkg/keystone/driver/sql/class.go +++ b/pkg/keystone/driver/sql/class.go @@ -29,8 +29,8 @@ func (self *SSQLDriverClass) SyncMethod() string { return api.IdentityProviderSyncLocal } -func (self *SSQLDriverClass) NewDriver(idpId, idpName, template string, conf api.TIdentityProviderConfigs) (driver.IIdentityBackend, error) { - return NewSQLDriver(idpId, idpName, template, conf) +func (self *SSQLDriverClass) NewDriver(idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TIdentityProviderConfigs) (driver.IIdentityBackend, error) { + return NewSQLDriver(idpId, idpName, template, targetDomainId, autoCreateProject, conf) } func (self *SSQLDriverClass) Name() string { diff --git a/pkg/keystone/driver/sql/sql.go b/pkg/keystone/driver/sql/sql.go index 0dc467c622..97b9ae6d77 100644 --- a/pkg/keystone/driver/sql/sql.go +++ b/pkg/keystone/driver/sql/sql.go @@ -29,8 +29,8 @@ type SSQLDriver struct { driver.SBaseIdentityDriver } -func NewSQLDriver(idpId, idpName, template string, conf api.TIdentityProviderConfigs) (driver.IIdentityBackend, error) { - base, err := driver.NewBaseIdentityDriver(idpId, idpName, template, conf) +func NewSQLDriver(idpId, idpName, template, targetDomainId string, autoCreateProject bool, conf api.TIdentityProviderConfigs) (driver.IIdentityBackend, error) { + base, err := driver.NewBaseIdentityDriver(idpId, idpName, template, targetDomainId, autoCreateProject, conf) if err != nil { return nil, err } diff --git a/pkg/keystone/models/domains.go b/pkg/keystone/models/domains.go index 76ea715274..9577841136 100644 --- a/pkg/keystone/models/domains.go +++ b/pkg/keystone/models/domains.go @@ -153,10 +153,14 @@ func (manager *SDomainManager) FetchDomainByIdOrName(domain string) (*SDomain, e return nil, err } q := manager.Query().NotEquals("id", api.KeystoneDomainRoot) - q = q.Filter(sqlchemy.OR( - sqlchemy.Equals(q.Field("id"), domain), - sqlchemy.Equals(q.Field("name"), domain), - )) + if stringutils2.IsUtf8(domain) { + q = q.Equals("name", domain) + } else { + q = q.Filter(sqlchemy.OR( + sqlchemy.Equals(q.Field("id"), domain), + sqlchemy.Equals(q.Field("name"), domain), + )) + } err = q.First(obj) if err != nil { return nil, err @@ -205,6 +209,11 @@ func (domain *SDomain) GetGroupCount() (int, error) { return q.CountWithError() } +func (domain *SDomain) GetIdpCount() (int, error) { + q := IdentityProviderManager.Query().Equals("target_domain_id", domain.Id) + return q.CountWithError() +} + func (domain *SDomain) ValidatePurgeCondition(ctx context.Context) error { if domain.Enabled.IsTrue() { return httperrors.NewInvalidStatusError("domain is enabled") @@ -301,6 +310,12 @@ func domainExtra(domain *SDomain, extra *jsonutils.JSONDict) *jsonutils.JSONDict extra.Add(jsonutils.NewInt(int64(grpCnt)), "group_count") prjCnt, _ := domain.GetProjectCount() extra.Add(jsonutils.NewInt(int64(prjCnt)), "project_count") + roleCnt, _ := domain.GetRoleCount() + extra.Add(jsonutils.NewInt(int64(roleCnt)), "role_count") + policyCnt, _ := domain.GetPolicyCount() + extra.Add(jsonutils.NewInt(int64(policyCnt)), "policy_count") + idpCnt, _ := domain.GetIdpCount() + extra.Add(jsonutils.NewInt(int64(idpCnt)), "idp_count") return extra } diff --git a/pkg/keystone/models/expandidps.go b/pkg/keystone/models/expandidps.go index fda305211d..2ae8f6a38d 100644 --- a/pkg/keystone/models/expandidps.go +++ b/pkg/keystone/models/expandidps.go @@ -28,19 +28,20 @@ import ( func expandIdpAttributes(rows []*jsonutils.JSONDict, objs []db.IModel, fields stringutils2.SSortedStrings, entType string) []*jsonutils.JSONDict { if len(fields) == 0 || fields.Contains("idp_id") || fields.Contains("idp") || fields.Contains("idp_entity_id") || fields.Contains("idp_driver") { + log.Debugf("objs %d", len(objs)) idList := make([]string, len(objs)) for i := range objs { - idList = append(idList, objs[i].GetId()) + idList[i] = objs[i].GetId() } idps, err := fetchIdmappings(idList, entType) if err == nil && idps != nil { for i := range rows { if idp, ok := idps[objs[i].GetId()]; ok { if len(fields) == 0 || fields.Contains("idp_id") { - rows[i].Set("idp_id", jsonutils.NewString(idp.Id)) + rows[i].Set("idp_id", jsonutils.NewString(idp.IdpId)) } if len(fields) == 0 || fields.Contains("idp") { - rows[i].Set("idp", jsonutils.NewString(idp.Name)) + rows[i].Set("idp", jsonutils.NewString(idp.IdpName)) } if len(fields) == 0 || fields.Contains("idp_entity_id") { rows[i].Set("idp_entity_id", jsonutils.NewString(idp.EntityId)) @@ -58,8 +59,8 @@ func expandIdpAttributes(rows []*jsonutils.JSONDict, objs []db.IModel, fields st } type sIdpInfo struct { - Id string - Name string + IdpId string + IdpName string EntityId string Driver string PublicId string @@ -69,13 +70,13 @@ func fetchIdmappings(idList []string, resType string) (map[string]sIdpInfo, erro idmappings := IdmappingManager.Query().SubQuery() idps := IdentityProviderManager.Query().SubQuery() - q := idmappings.Query(idmappings.Field("domain_id", "id"), + q := idmappings.Query(idmappings.Field("domain_id", "idp_id"), idmappings.Field("local_id", "entity_id"), - idps.Field("name"), + idps.Field("name", "idp_name"), idps.Field("driver"), idmappings.Field("public_id"), - ).Join(idps, sqlchemy.Equals(idps.Field("id"), - idmappings.Field("domain_id"))) + ) + q = q.Join(idps, sqlchemy.Equals(idps.Field("id"), idmappings.Field("domain_id"))) q = q.Filter(sqlchemy.In(idmappings.Field("public_id"), idList)) q = q.Filter(sqlchemy.Equals(idmappings.Field("entity_type"), resType)) diff --git a/pkg/keystone/models/groups.go b/pkg/keystone/models/groups.go index 60f9fb9e79..5654458183 100644 --- a/pkg/keystone/models/groups.go +++ b/pkg/keystone/models/groups.go @@ -274,3 +274,7 @@ func (manager *SGroupManager) FetchCustomizeColumns(ctx context.Context, userCre rows := manager.SIdentityBaseResourceManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields) return expandIdpAttributes(rows, objs, fields, api.IdMappingEntityGroup) } + +func (manager *SGroupManager) FetchGroupLocalIdsInDomain(domainId string, excludes []string) ([]string, error) { + return fetchLocalIdsInDomain(manager, domainId, excludes) +} diff --git a/pkg/keystone/models/id_mappings.go b/pkg/keystone/models/id_mappings.go index 00a75e3708..4a34d01b7b 100644 --- a/pkg/keystone/models/id_mappings.go +++ b/pkg/keystone/models/id_mappings.go @@ -125,10 +125,10 @@ func (manager *SIdmappingManager) FetchEntity(idStr string, entType string) (*SI } func (manager *SIdmappingManager) deleteByIdpId(idpId string) error { - return manager.DeleteAny(idpId, "", nil) + return manager.DeleteAny(idpId, "", nil, nil) } -func (manager *SIdmappingManager) DeleteAny(idpId string, entityType string, excludeLocalIds []string) error { +func (manager *SIdmappingManager) DeleteAny(idpId string, entityType string, excludeLocalIds []string, includeLocalIds []string) error { q := manager.Query().Equals("domain_id", idpId) if len(entityType) > 0 { q = q.Equals("entity_type", entityType) @@ -136,6 +136,9 @@ func (manager *SIdmappingManager) DeleteAny(idpId string, entityType string, exc if len(excludeLocalIds) > 0 { q = q.NotIn("local_id", excludeLocalIds) } + if len(includeLocalIds) > 0 { + q = q.In("local_id", includeLocalIds) + } idmappings := make([]SIdmapping, 0) err := db.FetchModelObjects(manager, q, &idmappings) if err != nil && err != sql.ErrNoRows { diff --git a/pkg/keystone/models/identity_provider.go b/pkg/keystone/models/identity_provider.go index 94c0cdf469..a6b3a30473 100644 --- a/pkg/keystone/models/identity_provider.go +++ b/pkg/keystone/models/identity_provider.go @@ -19,10 +19,10 @@ import ( "database/sql" "time" - "yunion.io/x/pkg/errors" - "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/tristate" "yunion.io/x/sqlchemy" api "yunion.io/x/onecloud/pkg/apis/identity" @@ -73,6 +73,10 @@ type SIdentityProvider struct { Driver string `width:"32" charset:"ascii" nullable:"false" list:"admin" create:"admin_required"` Template string `width:"32" charset:"ascii" nullable:"true" list:"admin" create:"admin_optional"` + TargetDomainId string `width:"64" charset:"ascii" nullable:"true" list:"admin" create:"admin_optional" update:"admin"` + + AutoCreateProject tristate.TriState `default:"true" nullable:"true" list:"admin" create:"admin_optional" update:"admin"` + ErrorCount int `list:"admin"` SyncStatus string `width:"10" charset:"ascii" default:"idle" list:"admin"` @@ -300,6 +304,19 @@ func (manager *SIdentityProviderManager) ValidateCreateData(ctx context.Context, } } + targetDomainStr, _ := data.GetString("target_domain") + if len(targetDomainStr) > 0 { + domain, err := DomainManager.FetchDomainById(targetDomainStr) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(DomainManager.Keyword(), targetDomainStr) + } else { + return nil, httperrors.NewGeneralError(err) + } + } + data.Set("target_domain_id", jsonutils.NewString(domain.Id)) + } + opts := api.TIdentityProviderConfigs{} err := data.Unmarshal(&opts, "config") if err != nil { @@ -408,6 +425,12 @@ func (self *SIdentityProvider) GetExtraDetails(ctx context.Context, userCred mcc func (self *SIdentityProvider) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict { extra = db.FetchModelExtraCountProperties(self, extra) extra.Set("sync_interval_seconds", jsonutils.NewInt(int64(self.getSyncIntervalSeconds()))) + if len(self.TargetDomainId) > 0 { + domain, _ := DomainManager.FetchDomainById(self.TargetDomainId) + if domain != nil { + extra.Set("target_domain", jsonutils.NewString(domain.Name)) + } + } return extra } diff --git a/pkg/keystone/models/syncworker.go b/pkg/keystone/models/syncworker.go index 465ab77188..5981d583a4 100644 --- a/pkg/keystone/models/syncworker.go +++ b/pkg/keystone/models/syncworker.go @@ -50,7 +50,7 @@ func submitIdpSyncTask(ctx context.Context, userCred mcclient.TokenCredential, i idp.MarkDisconnected(ctx, userCred) return } - driver, err := driver.GetDriver(idp.Driver, idp.Id, idp.Name, idp.Template, conf) + driver, err := driver.GetDriver(idp.Driver, idp.Id, idp.Name, idp.Template, idp.TargetDomainId, idp.AutoCreateProject.Bool(), conf) if err != nil { log.Errorf("GetDriver for idp %s fail %s", idp.Name, err) idp.MarkDisconnected(ctx, userCred) diff --git a/pkg/keystone/models/users.go b/pkg/keystone/models/users.go index c5ef4c264b..9e831011c7 100644 --- a/pkg/keystone/models/users.go +++ b/pkg/keystone/models/users.go @@ -656,3 +656,37 @@ func (manager *SUserManager) FetchCustomizeColumns(ctx context.Context, userCred rows := manager.SEnabledIdentityBaseResourceManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields) return expandIdpAttributes(rows, objs, fields, api.IdMappingEntityUser) } + +func (manager *SUserManager) FetchUserLocalIdsInDomain(domainId string, excludes []string) ([]string, error) { + return fetchLocalIdsInDomain(manager, domainId, excludes) +} + +func fetchLocalIdsInDomain(manager db.IModelManager, domainId string, excludes []string) ([]string, error) { + idmappings := IdmappingManager.Query().SubQuery() + users := manager.Query().SubQuery() + q := idmappings.Query(idmappings.Field("local_id")) + q = q.Join(users, sqlchemy.AND( + sqlchemy.Equals(idmappings.Field("entity_type"), manager.Keyword()), + sqlchemy.Equals(idmappings.Field("public_id"), users.Field("id")), + )) + q = q.Filter(sqlchemy.Equals(users.Field("domain_id"), domainId)) + q = q.Filter(sqlchemy.NotIn(idmappings.Field("local_id"), excludes)) + rows, err := q.Rows() + if err != nil && err != sql.ErrNoRows { + return nil, errors.Wrap(err, "query") + } + if rows == nil { + return nil, nil + } + defer rows.Close() + ret := make([]string, 0) + for rows.Next() { + var idStr string + err = rows.Scan(&idStr) + if err != nil { + return nil, errors.Wrap(err, "scan") + } + ret = append(ret, idStr) + } + return ret, nil +} diff --git a/pkg/keystone/tasks/identity_provider_delete_task.go b/pkg/keystone/tasks/identity_provider_delete_task.go index 9f8d8a8dff..e5f220a41d 100644 --- a/pkg/keystone/tasks/identity_provider_delete_task.go +++ b/pkg/keystone/tasks/identity_provider_delete_task.go @@ -20,6 +20,7 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/identity" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/keystone/models" @@ -39,6 +40,7 @@ func (self *IdentityProviderDeleteTask) OnInit(ctx context.Context, obj db.IStan err := idp.Purge(ctx, self.UserCred) if err != nil { + idp.SetStatus(self.UserCred, api.IdentityDriverStatusDeleteFailed, err.Error()) self.SetStageFailed(ctx, fmt.Sprintf("purge failed %s", err)) logclient.AddActionLogWithStartable(self, idp, logclient.ACT_DELETE, err, self.UserCred, false) return diff --git a/pkg/keystone/tokens/auth.go b/pkg/keystone/tokens/auth.go index 07c647f17c..2a6a13dc9e 100644 --- a/pkg/keystone/tokens/auth.go +++ b/pkg/keystone/tokens/auth.go @@ -140,7 +140,7 @@ func authUserByIdentity(ctx context.Context, ident mcclient.SAuthenticationIdent return nil, errors.Wrap(err, "GetConfig") } - backend, err := driver.GetDriver(idp.Driver, idp.Id, idp.Name, idp.Template, conf) + backend, err := driver.GetDriver(idp.Driver, idp.Id, idp.Name, idp.Template, idp.TargetDomainId, idp.AutoCreateProject.Bool(), conf) if err != nil { return nil, errors.Wrap(err, "driver.GetDriver") } diff --git a/pkg/util/stringutils2/i18n.go b/pkg/util/stringutils2/i18n.go new file mode 100644 index 0000000000..38ca99d8bb --- /dev/null +++ b/pkg/util/stringutils2/i18n.go @@ -0,0 +1,24 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stringutils2 + +func IsUtf8(str string) bool { + for _, runeVal := range str { + if runeVal > 0x7f { + return true + } + } + return false +} diff --git a/pkg/util/stringutils2/i18n_test.go b/pkg/util/stringutils2/i18n_test.go new file mode 100644 index 0000000000..d2b1cfd358 --- /dev/null +++ b/pkg/util/stringutils2/i18n_test.go @@ -0,0 +1,35 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stringutils2 + +import ( + "testing" +) + +func TestIsUtf8(t *testing.T) { + cases := []struct { + In string + Want bool + }{ + {"中文", true}, + {"this is eng", false}, + } + for _, c := range cases { + got := IsUtf8(c.In) + if got != c.Want { + t.Errorf("IsUtf8 %s got %v want %v", c.In, got, c.Want) + } + } +}