From f7c61a9750a5a9d35c2ac0d21e38aaa32818879c Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Tue, 12 Mar 2019 13:39:22 +0800 Subject: [PATCH 1/5] fix: 1. usage hide system servers 2. host undo convert shows wrong status --- pkg/cloudcommon/db/taskman/tasks.go | 14 +++++++++++--- pkg/compute/guestdrivers/baremetals.go | 4 ++++ pkg/compute/tasks/guest_sync_task.go | 8 ++++++-- pkg/compute/usages/handler.go | 4 +++- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/pkg/cloudcommon/db/taskman/tasks.go b/pkg/cloudcommon/db/taskman/tasks.go index e9c703ac6a..7760200e87 100644 --- a/pkg/cloudcommon/db/taskman/tasks.go +++ b/pkg/cloudcommon/db/taskman/tasks.go @@ -10,17 +10,17 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/appctx" - "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/onecloud/pkg/util/httputils" "yunion.io/x/pkg/util/reflectutils" "yunion.io/x/pkg/util/stringutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/appctx" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/httputils" ) const ( @@ -447,6 +447,14 @@ func (task *STask) ScheduleRun(data jsonutils.JSONObject) { runTask(task.Id, data) } +func (self *STask) HasParentTask() bool { + parentTaskId, _ := self.Params.GetString(PARENT_TASK_ID_KEY) + if len(parentTaskId) > 0 { + return true + } + return false +} + func (self *STask) GetParentTask() *STask { parentTaskId, _ := self.Params.GetString(PARENT_TASK_ID_KEY) if len(parentTaskId) > 0 { diff --git a/pkg/compute/guestdrivers/baremetals.go b/pkg/compute/guestdrivers/baremetals.go index c58cd61034..d5f4adb5a1 100644 --- a/pkg/compute/guestdrivers/baremetals.go +++ b/pkg/compute/guestdrivers/baremetals.go @@ -136,6 +136,7 @@ func (self *SBaremetalGuestDriver) GetRandomNetworkTypes() []string { } func (self *SBaremetalGuestDriver) Attach2RandomNetwork(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, netConfig *models.SNetworkConfig, pendingUsage quotas.IQuota) ([]models.SGuestnetwork, error) { + log.Debugf("SBaremetalGuestDriver Attach2RandomNetwork") netifs := host.GetNetInterfaces() netsAvaiable := make([]models.SNetwork, 0) netifIndexs := make(map[string][]models.SNetInterface, 0) @@ -144,6 +145,7 @@ func (self *SBaremetalGuestDriver) Attach2RandomNetwork(guest *models.SGuest, ct if len(netConfig.NetType) > 0 { netTypes = []string{netConfig.NetType} } + log.Debugf("netTypes %#v", netTypes) var wirePattern *regexp.Regexp if len(netConfig.Wire) > 0 { wirePattern = regexp.MustCompile(netConfig.Wire) @@ -162,8 +164,10 @@ func (self *SBaremetalGuestDriver) Attach2RandomNetwork(guest *models.SGuest, ct var net *models.SNetwork if netConfig.Private { net, _ = wire.GetCandidatePrivateNetwork(userCred, netConfig.Exit, netTypes) + log.Debugf("choose private network %#v", net) } else { net, _ = wire.GetCandidatePublicNetwork(netConfig.Exit, netTypes) + log.Debugf("choose public network %#v", net) } if net != nil { netsAvaiable = append(netsAvaiable, *net) diff --git a/pkg/compute/tasks/guest_sync_task.go b/pkg/compute/tasks/guest_sync_task.go index c8c65ac534..b2cbdfe090 100644 --- a/pkg/compute/tasks/guest_sync_task.go +++ b/pkg/compute/tasks/guest_sync_task.go @@ -47,8 +47,12 @@ func (self *GuestSyncConfTask) OnSyncComplete(ctx context.Context, obj db.IStand } func (self *GuestSyncConfTask) OnDiskSyncComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { - self.SetStage("on_sync_status_complete", nil) - guest.StartSyncstatus(ctx, self.GetUserCred(), self.GetTaskId()) + if self.HasParentTask() { + self.OnSyncStatusComplete(ctx, guest, nil) + } else { + self.SetStage("on_sync_status_complete", nil) + guest.StartSyncstatus(ctx, self.GetUserCred(), self.GetTaskId()) + } } func (self *GuestSyncConfTask) OnDiskSyncCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { diff --git a/pkg/compute/usages/handler.go b/pkg/compute/usages/handler.go index 0376564ae6..2ea515a08d 100644 --- a/pkg/compute/usages/handler.go +++ b/pkg/compute/usages/handler.go @@ -509,7 +509,9 @@ func guestHypervisorsUsage( if userCred != nil { projectId = userCred.GetProjectId() } - guest := models.GuestManager.TotalCount(projectId, rangeObj, status, hypervisors, true, pendingDelete, hostTypes, resourceTypes, providers) + // temporarily hide system resources + // XXX needs more work later + guest := models.GuestManager.TotalCount(projectId, rangeObj, status, hypervisors, false, pendingDelete, hostTypes, resourceTypes, providers) count := make(map[string]interface{}) count[prefix] = guest.TotalGuestCount count[fmt.Sprintf("%s.cpu", prefix)] = guest.TotalCpuCount From 4e85b1cd5a011d3c54d5939e87053e3f2f840dad Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Tue, 12 Mar 2019 15:07:13 +0800 Subject: [PATCH 2/5] fix: usage port state not correct --- pkg/compute/models/networks.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pkg/compute/models/networks.go b/pkg/compute/models/networks.go index 1e18825561..911082c89f 100644 --- a/pkg/compute/models/networks.go +++ b/pkg/compute/models/networks.go @@ -633,7 +633,7 @@ func (manager *SNetworkManager) allNetworksQ(providers []string, rangeObj db.ISt networks := manager.Query().SubQuery() hostwires := HostwireManager.Query().SubQuery() hosts := HostManager.Query().SubQuery() - q := networks.Query() + q := networks.Query(networks.Field("id")) q = q.Join(hostwires, sqlchemy.Equals(hostwires.Field("wire_id"), networks.Field("wire_id"))) q = q.Join(hosts, sqlchemy.Equals(hosts.Field("id"), hostwires.Field("host_id"))) q = q.Filter(sqlchemy.IsTrue(hosts.Field("enabled"))) @@ -645,13 +645,12 @@ func (manager *SNetworkManager) allNetworksQ(providers []string, rangeObj db.ISt func (manager *SNetworkManager) totalPortCountQ(userCred mcclient.TokenCredential, providers []string, rangeObj db.IStandaloneModel) *sqlchemy.SQuery { q := manager.allNetworksQ(providers, rangeObj) - networks := manager.Query().SubQuery() if userCred != nil && !db.IsAdminAllowList(userCred, manager) { q = q.Filter(sqlchemy.OR( - sqlchemy.Equals(networks.Field("tenant_id"), userCred.GetProjectId()), - sqlchemy.IsTrue(networks.Field("is_public")))) + sqlchemy.Equals(q.Field("tenant_id"), userCred.GetProjectId()), + sqlchemy.IsTrue(q.Field("is_public")))) } - return q + return manager.Query().In("id", q.Distinct().SubQuery()) } type NetworkPortStat struct { @@ -659,7 +658,11 @@ type NetworkPortStat struct { CountExt int } -func (manager *SNetworkManager) TotalPortCount(userCred mcclient.TokenCredential, providers []string, rangeObj db.IStandaloneModel) NetworkPortStat { +func (manager *SNetworkManager) TotalPortCount( + userCred mcclient.TokenCredential, + providers []string, + rangeObj db.IStandaloneModel, +) NetworkPortStat { nets := make([]SNetwork, 0) err := manager.totalPortCountQ(userCred, providers, rangeObj).All(&nets) if err != nil { From 8b460aa21ff3971d360fcb0f36e920d2402b6a10 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Tue, 12 Mar 2019 15:11:27 +0800 Subject: [PATCH 3/5] remove debug logs --- pkg/compute/guestdrivers/baremetals.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/compute/guestdrivers/baremetals.go b/pkg/compute/guestdrivers/baremetals.go index d5f4adb5a1..c58cd61034 100644 --- a/pkg/compute/guestdrivers/baremetals.go +++ b/pkg/compute/guestdrivers/baremetals.go @@ -136,7 +136,6 @@ func (self *SBaremetalGuestDriver) GetRandomNetworkTypes() []string { } func (self *SBaremetalGuestDriver) Attach2RandomNetwork(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, netConfig *models.SNetworkConfig, pendingUsage quotas.IQuota) ([]models.SGuestnetwork, error) { - log.Debugf("SBaremetalGuestDriver Attach2RandomNetwork") netifs := host.GetNetInterfaces() netsAvaiable := make([]models.SNetwork, 0) netifIndexs := make(map[string][]models.SNetInterface, 0) @@ -145,7 +144,6 @@ func (self *SBaremetalGuestDriver) Attach2RandomNetwork(guest *models.SGuest, ct if len(netConfig.NetType) > 0 { netTypes = []string{netConfig.NetType} } - log.Debugf("netTypes %#v", netTypes) var wirePattern *regexp.Regexp if len(netConfig.Wire) > 0 { wirePattern = regexp.MustCompile(netConfig.Wire) @@ -164,10 +162,8 @@ func (self *SBaremetalGuestDriver) Attach2RandomNetwork(guest *models.SGuest, ct var net *models.SNetwork if netConfig.Private { net, _ = wire.GetCandidatePrivateNetwork(userCred, netConfig.Exit, netTypes) - log.Debugf("choose private network %#v", net) } else { net, _ = wire.GetCandidatePublicNetwork(netConfig.Exit, netTypes) - log.Debugf("choose public network %#v", net) } if net != nil { netsAvaiable = append(netsAvaiable, *net) From cb9fea22867825a89b5ef93bfbc47f7bf12e94ee Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Tue, 12 Mar 2019 18:24:12 +0800 Subject: [PATCH 4/5] fix: filter cached images by storage status --- pkg/compute/models/cachedimages.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/compute/models/cachedimages.go b/pkg/compute/models/cachedimages.go index 7efebcc156..631b3e1080 100644 --- a/pkg/compute/models/cachedimages.go +++ b/pkg/compute/models/cachedimages.go @@ -336,14 +336,16 @@ func (self *SCachedimage) ChooseSourceStoragecacheInRange(hostType string, exclu scimgs := make([]SStoragecachedimage, 0) q := storageCachedImage.Query(). - Join(storage, sqlchemy.AND(sqlchemy.Equals(storage.Field("storagecache_id"), storageCachedImage.Field("storagecache_id")))). - Join(hostStorage, sqlchemy.AND(sqlchemy.Equals(hostStorage.Field("storage_id"), storage.Field("id")))). - Join(host, sqlchemy.AND(sqlchemy.Equals(hostStorage.Field("host_id"), host.Field("id")))). + Join(storage, sqlchemy.Equals(storage.Field("storagecache_id"), storageCachedImage.Field("storagecache_id"))). + Join(hostStorage, sqlchemy.Equals(hostStorage.Field("storage_id"), storage.Field("id"))). + Join(host, sqlchemy.Equals(hostStorage.Field("host_id"), host.Field("id"))). Filter(sqlchemy.Equals(storageCachedImage.Field("cachedimage_id"), self.Id)). Filter(sqlchemy.Equals(storageCachedImage.Field("status"), CACHED_IMAGE_STATUS_READY)). Filter(sqlchemy.Equals(host.Field("status"), HOST_STATUS_RUNNING)). Filter(sqlchemy.IsTrue(host.Field("enabled"))). - Filter(sqlchemy.Equals(host.Field("host_status"), HOST_ONLINE)) + Filter(sqlchemy.Equals(host.Field("host_status"), HOST_ONLINE)). + Filter(sqlchemy.IsTrue(storage.Field("enabled"))). + Filter(sqlchemy.In(storage.Field("status"), []string{STORAGE_ENABLED, STORAGE_ONLINE})) if len(excludes) > 0 { q = q.Filter(sqlchemy.NotIn(host.Field("id"), excludes)) From 1ca00c14c6d0e3e9342f8e2a0a5586f6b2fc7a2c Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Tue, 12 Mar 2019 21:48:40 +0800 Subject: [PATCH 5/5] fix: find image peer host with local storage only --- pkg/compute/models/cachedimages.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/compute/models/cachedimages.go b/pkg/compute/models/cachedimages.go index 631b3e1080..6421b4b06a 100644 --- a/pkg/compute/models/cachedimages.go +++ b/pkg/compute/models/cachedimages.go @@ -345,7 +345,8 @@ func (self *SCachedimage) ChooseSourceStoragecacheInRange(hostType string, exclu Filter(sqlchemy.IsTrue(host.Field("enabled"))). Filter(sqlchemy.Equals(host.Field("host_status"), HOST_ONLINE)). Filter(sqlchemy.IsTrue(storage.Field("enabled"))). - Filter(sqlchemy.In(storage.Field("status"), []string{STORAGE_ENABLED, STORAGE_ONLINE})) + Filter(sqlchemy.In(storage.Field("status"), []string{STORAGE_ENABLED, STORAGE_ONLINE})). + Filter(sqlchemy.Equals(storage.Field("storage_type"), STORAGE_LOCAL)) if len(excludes) > 0 { q = q.Filter(sqlchemy.NotIn(host.Field("id"), excludes))