diff --git a/Gopkg.lock b/Gopkg.lock index 67f3b37f1e..f6212bb5f6 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1749,11 +1749,11 @@ [[projects]] branch = "master" - digest = "1:4137f40ed548a5dc02719ad2e6b6bdfd76a9d881fcd1a72eb5970956a7219d03" + digest = "1:71e1d99440696f89cfdd93aba9f349fbd6a1273b30114ff5b6e5ba2457832b80" name = "yunion.io/x/sqlchemy" packages = ["."] pruneopts = "UT" - revision = "4d8d8e9b16804336f16570787421b9be24a6da1f" + revision = "1214fd27cb9c02b2d650557a76cfaa8c13220b54" [[projects]] branch = "master" diff --git a/pkg/appsrv/appsrv.go b/pkg/appsrv/appsrv.go index 083f212912..d26636341a 100644 --- a/pkg/appsrv/appsrv.go +++ b/pkg/appsrv/appsrv.go @@ -41,6 +41,7 @@ type Application struct { name string context context.Context session *SWorkerManager + readSession *SWorkerManager systemSession *SWorkerManager roots map[string]*RadixNode rootLock *sync.RWMutex @@ -74,6 +75,7 @@ func NewApplication(name string, connMax int, db bool) *Application { context: context.Background(), connMax: connMax, session: NewWorkerManager("HttpRequestWorkerManager", connMax, DEFAULT_BACKLOG, db), + readSession: NewWorkerManager("HttpGetRequestWorkerManager", connMax, DEFAULT_BACKLOG, db), systemSession: NewWorkerManager("InternalHttpRequestWorkerManager", 1, DEFAULT_BACKLOG, false), roots: make(map[string]*RadixNode), rootLock: &sync.RWMutex{}, @@ -242,7 +244,11 @@ func (app *Application) defaultHandle(w http.ResponseWriter, r *http.Request, ri defer cancel() session := hand.workerMan if session == nil { - session = app.session + if r.Method == "GET" || r.Method == "HEAD" { + session = app.readSession + } else { + session = app.session + } } appParams := hand.GetAppParams(params, segs) appParams.Request = r diff --git a/pkg/appsrv/workers.go b/pkg/appsrv/workers.go index a517917208..dbc3b0c3ef 100644 --- a/pkg/appsrv/workers.go +++ b/pkg/appsrv/workers.go @@ -222,7 +222,7 @@ func (wm *SWorkerManager) scheduleWithLock() { log.Debugf("no enough worker, add new worker %s", worker) } go worker.run() - } else { + } else if wm.queue.Size() > 10 { log.Warningf("[%s] BUSY activeWork %d detachedWork %d max %d queue: %d", wm, wm.ActiveWorkerCount(), wm.DetachedWorkerCount(), wm.workerCount, wm.queue.Size()) } } diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index f0d4ae1d7d..36232b8f9e 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -39,6 +39,7 @@ import ( "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/util/httputils" "yunion.io/x/onecloud/pkg/util/rbacutils" + "yunion.io/x/onecloud/pkg/util/stringutils2" ) type DBModelDispatcher struct { @@ -323,13 +324,28 @@ func listItemQueryFilters(manager IModelManager, ctx context.Context, q *sqlchem return q, nil } -func Query2List(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, q *sqlchemy.SQuery, query jsonutils.JSONObject) ([]jsonutils.JSONObject, error) { - listF := listFields(manager, userCred) - fieldFilter := jsonutils.GetQueryStringArray(query, "field") - if len(fieldFilter) > 0 && IsAdminAllowList(userCred, manager) { - // only sysadmin can specify list Fields - listF = fieldFilter +func mergeFields(metaFields, queryFields []string, isAdmin bool) stringutils2.SSortedStrings { + meta := stringutils2.NewSortedStrings(metaFields) + if len(queryFields) == 0 { + return meta } + + query := stringutils2.NewSortedStrings(queryFields) + _, mAndQ, qNoM := stringutils2.Split(meta, query) + + if !isAdmin { + return mAndQ + } + + // only sysadmin can specify list Fields + return stringutils2.Merge(mAndQ, qNoM) +} + +func Query2List(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, q *sqlchemy.SQuery, query jsonutils.JSONObject) ([]jsonutils.JSONObject, error) { + metaFields := listFields(manager, userCred) + fieldFilter := jsonutils.GetQueryStringArray(query, "field") + listF := mergeFields(metaFields, fieldFilter, IsAdminAllowList(userCred, manager)) + showDetails := false showDetailsJson, _ := query.Get("details") if showDetailsJson != nil { @@ -337,15 +353,7 @@ func Query2List(manager IModelManager, ctx context.Context, userCred mcclient.To } else { showDetails = true } - item, err := NewModelObject(manager) - if err != nil { - return nil, err - } - itemInitValue := reflect.Indirect(reflect.ValueOf(item)) - item, err = NewModelObject(manager) - if err != nil { - return nil, err - } + items := make([]IModel, 0) results := make([]jsonutils.JSONObject, 0) rows, err := q.Rows() if err != nil { @@ -353,8 +361,10 @@ func Query2List(manager IModelManager, ctx context.Context, userCred mcclient.To } defer rows.Close() for rows.Next() { - itemValue := reflect.Indirect(reflect.ValueOf(item)) - itemValue.Set(itemInitValue) + item, err := NewModelObject(manager) + if err != nil { + return nil, err + } extraData := jsonutils.NewDict() if query.Contains("export_keys") { RowMap, err := q.Row2Map(rows) @@ -376,12 +386,8 @@ func Query2List(manager IModelManager, ctx context.Context, userCred mcclient.To } } - jsonData := jsonutils.Marshal(item) - jsonDict, ok := jsonData.(*jsonutils.JSONDict) - if !ok { - return nil, fmt.Errorf("invalid model data structure, not a dict") - } - jsonDict = jsonDict.CopyIncludes(listF...) + jsonDict := jsonutils.Marshal(item).(*jsonutils.JSONDict) + jsonDict = jsonDict.CopyIncludes([]string(listF)...) jsonDict.Update(extraData) if showDetails && !query.Contains("export_keys") { extraDict := item.GetCustomizeColumns(ctx, userCred, query) @@ -391,6 +397,16 @@ func Query2List(manager IModelManager, ctx context.Context, userCred mcclient.To jsonDict = getModelExtraDetails(item, ctx, jsonDict) } results = append(results, jsonDict) + items = append(items, item) + } + if showDetails && !query.Contains("export_keys") { + extraRows := manager.FetchCustomizeColumns(ctx, userCred, query, items, stringutils2.NewSortedStrings(fieldFilter)) + log.Debugf("manager.FetchCustomizeColumns: %s %s", extraRows, listF) + if len(extraRows) == len(results) { + for i := range results { + results[i].(*jsonutils.JSONDict).Update(extraRows[i]) + } + } } return results, nil } @@ -617,20 +633,27 @@ func getItemDetails(manager IModelManager, item IModel, ctx context.Context, use extraDict, err := item.GetExtraDetails(ctx, userCred, query) if err != nil { return nil, httperrors.NewGeneralError(err) - } else if extraDict != nil { - jsonData := jsonutils.Marshal(item) - jsonDict, ok := jsonData.(*jsonutils.JSONDict) - if !ok { - return nil, fmt.Errorf("fail to convert model to json") - } - jsonDict = jsonDict.CopyIncludes(GetDetailFields(manager, userCred)...) - jsonDict.Update(extraDict) - jsonDict = getModelExtraDetails(item, ctx, jsonDict) - return jsonDict, nil - } else { + } + if extraDict == nil { // override GetExtraDetails return nil, nil } + + metaFields := GetDetailFields(manager, userCred) + fieldFilter := jsonutils.GetQueryStringArray(query, "field") + getFields := mergeFields(metaFields, fieldFilter, IsAdminAllowGet(userCred, item)) + + jsonDict := jsonutils.Marshal(item).(*jsonutils.JSONDict) + jsonDict = jsonDict.CopyIncludes(getFields...) + jsonDict.Update(extraDict) + jsonDict = getModelExtraDetails(item, ctx, jsonDict) + + extraRows := manager.FetchCustomizeColumns(ctx, userCred, query, []IModel{item}, stringutils2.NewSortedStrings(fieldFilter)) + if len(extraRows) == 1 { + jsonDict.Update(extraRows[0]) + } + + return jsonDict, nil } func (dispatcher *DBModelDispatcher) tryGetModelProperty(ctx context.Context, property string, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { diff --git a/pkg/cloudcommon/db/interface.go b/pkg/cloudcommon/db/interface.go index 96b59bbae7..f57f88cc91 100644 --- a/pkg/cloudcommon/db/interface.go +++ b/pkg/cloudcommon/db/interface.go @@ -24,6 +24,7 @@ import ( "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/stringutils2" ) type IModelManager interface { @@ -86,6 +87,9 @@ type IModelManager interface { IsCustomizedGetDetailsBody() bool ListSkipLog(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool GetSkipLog(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool + + // list extend colums hook + FetchCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, objs []IModel, fields stringutils2.SSortedStrings) []*jsonutils.JSONDict } type IModel interface { diff --git a/pkg/cloudcommon/db/modelbase.go b/pkg/cloudcommon/db/modelbase.go index e2e774bb67..5a49a5a496 100644 --- a/pkg/cloudcommon/db/modelbase.go +++ b/pkg/cloudcommon/db/modelbase.go @@ -25,6 +25,7 @@ import ( "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/stringutils2" ) type SModelBase struct { @@ -207,6 +208,14 @@ func (manager *SModelBaseManager) GetSkipLog(ctx context.Context, userCred mccli return false } +func (manager *SModelBaseManager) FetchCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, objs []IModel, fields stringutils2.SSortedStrings) []*jsonutils.JSONDict { + ret := make([]*jsonutils.JSONDict, len(objs)) + for i := range objs { + ret[i] = jsonutils.NewDict() + } + return ret +} + func (model *SModelBase) GetId() string { return "" } diff --git a/pkg/cloudcommon/db/taskman/tasks.go b/pkg/cloudcommon/db/taskman/tasks.go index 83ad12b181..179bdb4ffa 100644 --- a/pkg/cloudcommon/db/taskman/tasks.go +++ b/pkg/cloudcommon/db/taskman/tasks.go @@ -65,7 +65,7 @@ var TaskManager *STaskManager func init() { TaskManager = &STaskManager{SResourceBaseManager: db.NewResourceBaseManager(STask{}, "tasks_tbl", "task", "tasks")} - TaskManager.TableSpec().AddIndex(true, "deleted", "obj_name", "obj_id", "created_at", "stage") + TaskManager.TableSpec().AddIndex(true, "created_at", "stage", "obj_id", "obj_name") } type STask struct { diff --git a/pkg/compute/hostdrivers/base.go b/pkg/compute/hostdrivers/base.go index 6765af8c3f..09ca1508cb 100644 --- a/pkg/compute/hostdrivers/base.go +++ b/pkg/compute/hostdrivers/base.go @@ -145,7 +145,7 @@ func (self *SBaseHostDriver) FinishConvert(userCred mcclient.TokenCredential, ho db.Update(host, func() error { host.CpuReserved = 0 host.MemReserved = 0 - host.AccessIp = guest.GetRealIps()[0] + host.AccessIp = guest.GetRealIPs()[0] host.Enabled = false host.HostStatus = models.HOST_OFFLINE host.HostType = hostType diff --git a/pkg/compute/models/elasticips.go b/pkg/compute/models/elasticips.go index 6e8331d9ca..b116e7fe05 100644 --- a/pkg/compute/models/elasticips.go +++ b/pkg/compute/models/elasticips.go @@ -74,6 +74,7 @@ func init() { "eips", ), } + ElasticipManager.TableSpec().AddIndex(true, "associate_id", "associate_type") } type SElasticip struct { diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index 49e6f7346b..9f1412987c 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -269,7 +269,7 @@ func (self *SGuest) PerformLiveMigrate(ctx context.Context, userCred mcclient.To return nil, httperrors.NewBadRequestError("Guest have backup, can't migrate") } if utils.IsInStringArray(self.Status, []string{VM_RUNNING, VM_SUSPEND}) { - cdrom := self.getCdrom() + cdrom := self.getCdrom(false) if cdrom != nil && len(cdrom.ImageId) > 0 { return nil, httperrors.NewBadRequestError("Cannot migrate with cdrom") } @@ -676,17 +676,17 @@ func (self *SGuest) StartGuestStopTask(ctx context.Context, userCred mcclient.To } func (self *SGuest) insertIso(imageId string) bool { - cdrom := self.getCdrom() + cdrom := self.getCdrom(true) return cdrom.insertIso(imageId) } func (self *SGuest) InsertIsoSucc(imageId string, path string, size int, name string) bool { - cdrom := self.getCdrom() + cdrom := self.getCdrom(false) return cdrom.insertIsoSucc(imageId, path, size, name) } func (self *SGuest) GetDetailsIso(userCred mcclient.TokenCredential) jsonutils.JSONObject { - cdrom := self.getCdrom() + cdrom := self.getCdrom(false) desc := jsonutils.NewDict() if len(cdrom.ImageId) > 0 { desc.Set("image_id", jsonutils.NewString(cdrom.ImageId)) @@ -705,12 +705,8 @@ func (self *SGuest) AllowPerformInsertiso(ctx context.Context, userCred mcclient } func (self *SGuest) PerformInsertiso(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { - cdrom := self.getCdrom() - if cdrom == nil { - return nil, fmt.Errorf("Failed to get cdrom") - } - - if len(cdrom.ImageId) > 0 { + cdrom := self.getCdrom(false) + if cdrom != nil && len(cdrom.ImageId) > 0 { return nil, httperrors.NewBadRequestError("CD-ROM not empty, please eject first") } imageId, _ := data.GetString("image_id") @@ -736,11 +732,8 @@ func (self *SGuest) PerformEjectiso(ctx context.Context, userCred mcclient.Token if self.Hypervisor != HYPERVISOR_KVM { return nil, httperrors.NewNotAcceptableError("Not allow for hypervisor %s", self.Hypervisor) } - cdrom := self.getCdrom() - if cdrom == nil { - return nil, fmt.Errorf("Failed to get cdrom") - } - if len(cdrom.ImageId) == 0 { + cdrom := self.getCdrom(false) + if cdrom == nil || len(cdrom.ImageId) == 0 { return nil, httperrors.NewBadRequestError("No ISO to eject") } if utils.IsInStringArray(self.Status, []string{VM_RUNNING, VM_READY}) { diff --git a/pkg/compute/models/guest_queries.go b/pkg/compute/models/guest_queries.go new file mode 100644 index 0000000000..dc689916ff --- /dev/null +++ b/pkg/compute/models/guest_queries.go @@ -0,0 +1,404 @@ +package models + +import ( + "context" + "database/sql" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/tristate" + "yunion.io/x/sqlchemy" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/stringutils2" +) + +func (manager *SGuestManager) FetchCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, objs []db.IModel, fields stringutils2.SSortedStrings) []*jsonutils.JSONDict { + rows := manager.SVirtualResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields) + guestIds := make([]string, len(objs)) + for i := range objs { + guestIds[i] = objs[i].GetId() + } + if len(fields) == 0 || fields.Contains("disk") { + gds := fetchGuestDiskSizes(guestIds) + if gds != nil { + for i := range rows { + if gd, ok := gds[objs[i].GetId()]; ok { + rows[i].Add(jsonutils.NewInt(gd.DiskSizeMb), "disk") + } + } + } + } + if len(fields) == 0 || fields.Contains("ips") { + gips := fetchGuestIPs(guestIds, tristate.False) + if gips != nil { + for i := range rows { + if gip, ok := gips[objs[i].GetId()]; ok { + rows[i].Add(jsonutils.NewString(strings.Join(gip, ",")), "ips") + } + } + } + } + if len(fields) == 0 || fields.Contains("vpc") || fields.Contains("vpc_id") { + gvpcs := fetchGuestVpcs(guestIds) + if gvpcs != nil { + for i := range rows { + if gvpc, ok := gvpcs[objs[i].GetId()]; ok { + if len(fields) == 0 || fields.Contains("vpc") { + rows[i].Add(jsonutils.NewString(strings.Join(gvpc.Vpc, ",")), "vpc") + } + if len(fields) == 0 || fields.Contains("vpc_id") { + rows[i].Add(jsonutils.NewString(strings.Join(gvpc.VpcId, ",")), "vpc_id") + } + } + } + } + } + if len(fields) == 0 || fields.Contains("secgroups") || fields.Contains("secgroup") { + gsgs := fetchSecgroups(guestIds) + if gsgs != nil { + for i := range rows { + if gsg, ok := gsgs[objs[i].GetId()]; ok { + if len(fields) == 0 || fields.Contains("secgroups") { + rows[i].Add(jsonutils.Marshal(gsg), "secgroups") + } + if len(fields) == 0 || fields.Contains("secgroup") { + rows[i].Add(jsonutils.NewString(gsg[0].Name), "secgroup") + } + } + } + } + } + if len(fields) == 0 || fields.Contains("eip") || fields.Contains("eip_mode") { + geips := fetchGuestEips(guestIds) + if geips != nil { + for i := range rows { + if eip, ok := geips[objs[i].GetId()]; ok { + if len(fields) == 0 || fields.Contains("eip") { + rows[i].Add(jsonutils.NewString(eip.IpAddr), "eip") + } + if len(fields) == 0 || fields.Contains("eip_mode") { + rows[i].Add(jsonutils.NewString(eip.Mode), "eip_mode") + } + } + } + } + } + if len(fields) == 0 || fields.Contains("keypair") { + gkps := fetchGuestKeypairs(guestIds) + if gkps != nil { + for i := range rows { + if kps, ok := gkps[objs[i].GetId()]; ok { + rows[i].Add(jsonutils.NewString(kps.Keypair), "keypair") + } + } + } + } + if len(fields) == 0 || fields.Contains("isolated_devices") || fields.Contains("is_gpu") { + gdevs := fetchGuestIsolatedDevices(guestIds) + if gdevs != nil { + for i := range rows { + if gdev, ok := gdevs[objs[i].GetId()]; ok { + if len(fields) == 0 || fields.Contains("isolated_devices") { + rows[i].Add(jsonutils.NewString(getIsolatedDeviceDetails(gdev)), "isolated_devices") + } + if len(fields) == 0 || fields.Contains("is_gpu") { + if len(gdev) > 0 { + rows[i].Add(jsonutils.JSONTrue, "is_gpu") + } else { + rows[i].Add(jsonutils.JSONFalse, "is_gpu") + } + } + } else { + if len(fields) == 0 || fields.Contains("is_gpu") { + rows[i].Add(jsonutils.JSONFalse, "is_gpu") + } + } + } + } + } + if len(fields) == 0 || fields.Contains("cdrom") { + gcds := fetchGuestCdroms(guestIds) + if gcds != nil { + for i := range rows { + if gcd, ok := gcds[objs[i].GetId()]; ok { + rows[i].Add(jsonutils.NewString(gcd.GetDetails()), "cdrom") + } else { + rows[i].Add(jsonutils.NewString(""), "cdrom") + } + } + } + } + return rows +} + +type sGustDiskSize struct { + GuestId string + DiskSizeMb int64 +} + +func fetchGuestDiskSizes(guestIds []string) map[string]sGustDiskSize { + disks := DiskManager.Query().SubQuery() + guestdisks := GuestdiskManager.Query().SubQuery() + + q := disks.Query(guestdisks.Field("guest_id"), sqlchemy.SUM("disk_size_mb", disks.Field("disk_size"))) + q = q.Join(guestdisks, sqlchemy.Equals(guestdisks.Field("disk_id"), disks.Field("id"))) + q = q.Filter(sqlchemy.In(guestdisks.Field("guest_id"), guestIds)) + q = q.GroupBy(guestdisks.Field("guest_id")) + + gds := make([]sGustDiskSize, 0) + err := q.All(&gds) + if err != nil && err != sql.ErrNoRows { + log.Errorf("query sGustDiskSize fail %s") + return nil + } + + ret := make(map[string]sGustDiskSize) + for i := range gds { + ret[gds[i].GuestId] = gds[i] + } + return ret +} + +func (guest *SGuest) getDiskSize() int { + result := fetchGuestDiskSizes([]string{guest.Id}) + if result == nil { + return -1 + } + if gs, ok := result[guest.Id]; ok { + return int(gs.DiskSizeMb) + } else { + return -1 + } +} + +func fetchGuestIPs(guestIds []string, virtual tristate.TriState) map[string][]string { + guestnetworks := GuestnetworkManager.Query().SubQuery() + q := guestnetworks.Query(guestnetworks.Field("guest_id"), guestnetworks.Field("ip_addr")) + q = q.In("guest_id", guestIds) + if virtual.IsTrue() { + q = q.IsTrue("virtual") + } else if virtual.IsFalse() { + q = q.IsFalse("virtual") + } + q = q.IsNotEmpty("ip_addr") + q = q.Asc("ip_addr") + type sGuestIdIpAddr struct { + GuestId string + IpAddr string + } + gias := make([]sGuestIdIpAddr, 0) + err := q.All(&gias) + if err != nil && err != sql.ErrNoRows { + return nil + } + ret := make(map[string][]string) + for i := range gias { + if _, ok := ret[gias[i].GuestId]; !ok { + ret[gias[i].GuestId] = make([]string, 0) + } + ret[gias[i].GuestId] = append(ret[gias[i].GuestId], gias[i].IpAddr) + } + return ret +} + +func (self *SGuest) GetRealIPs() []string { + result := fetchGuestIPs([]string{self.Id}, tristate.False) + if result == nil { + return nil + } + if ret, ok := result[self.Id]; ok { + return ret + } + return nil +} + +type sGuestVpcsInfo struct { + GuestId string + Vpc []string + VpcId []string +} + +func fetchGuestVpcs(guestIds []string) map[string]sGuestVpcsInfo { + vpcs := VpcManager.Query().SubQuery() + wires := WireManager.Query().SubQuery() + networks := NetworkManager.Query().SubQuery() + guestnetworks := GuestnetworkManager.Query().SubQuery() + + q := vpcs.Query(guestnetworks.Field("guest_id"), vpcs.Field("id"), vpcs.Field("name")) + q = q.Join(wires, sqlchemy.Equals(vpcs.Field("id"), wires.Field("vpc_id"))) + q = q.Join(networks, sqlchemy.Equals(wires.Field("id"), networks.Field("wire_id"))) + q = q.Join(guestnetworks, sqlchemy.Equals(networks.Field("id"), guestnetworks.Field("network_id"))) + q = q.Filter(sqlchemy.In(guestnetworks.Field("guest_id"), guestIds)) + q = q.Distinct() + + type sGuestVpcInfo struct { + GuestId string + Id string + Name string + } + gvpcs := make([]sGuestVpcInfo, 0) + err := q.All(&gvpcs) + if err != nil { + return nil + } + + ret := make(map[string]sGuestVpcsInfo) + for i := range gvpcs { + gvpc, ok := ret[gvpcs[i].GuestId] + if !ok { + gvpc = sGuestVpcsInfo{ + GuestId: gvpcs[i].GuestId, + Vpc: make([]string, 0), + VpcId: make([]string, 0), + } + } + gvpc.VpcId = append(gvpc.VpcId, gvpcs[i].Id) + gvpc.Vpc = append(gvpc.Vpc, gvpcs[i].Name) + ret[gvpcs[i].GuestId] = gvpc + } + + return ret +} + +type sSecgroupInfo struct { + Id string + Name string +} + +func fetchSecgroups(guestIds []string) map[string][]sSecgroupInfo { + secgroups := SecurityGroupManager.Query().SubQuery() + guestsecgroups := GuestsecgroupManager.Query().SubQuery() + guests := GuestManager.Query().SubQuery() + + q1 := guests.Query(guests.Field("id").Label("guest_id"), + guests.Field("secgrp_id").Label("secgroup_id")) + q1 = q1.Filter(sqlchemy.In(guests.Field("id"), guestIds)) + q2 := guestsecgroups.Query(guestsecgroups.Field("guest_id"), guestsecgroups.Field("secgroup_id")) + q2 = q2.Filter(sqlchemy.In(guestsecgroups.Field("guest_id"), guestIds)) + uq := sqlchemy.Union(q1, q2).SubQuery() + q := uq.Query(uq.Field("guest_id"), uq.Field("secgroup_id"), secgroups.Field("name").Label("secgroup_name")) + q = q.Join(secgroups, sqlchemy.Equals(uq.Field("secgroup_id"), secgroups.Field("id"))) + + type sGuestSecgroupInfo struct { + SecgroupId string + SecgroupName string + GuestId string + } + + gsgs := make([]sGuestSecgroupInfo, 0) + err := q.All(&gsgs) + if err != nil { + return nil + } + + ret := make(map[string][]sSecgroupInfo) + for i := range gsgs { + gsg, ok := ret[gsgs[i].GuestId] + if !ok { + gsg = make([]sSecgroupInfo, 0) + } + gsg = append(gsg, sSecgroupInfo{ + Id: gsgs[i].SecgroupId, + Name: gsgs[i].SecgroupName, + }) + ret[gsgs[i].GuestId] = gsg + } + + return ret +} + +type sEipInfo struct { + IpAddr string + Mode string + GuestId string +} + +func fetchGuestEips(guestIds []string) map[string]sEipInfo { + eips := ElasticipManager.Query().SubQuery() + + q := eips.Query(eips.Field("ip_addr"), eips.Field("mode"), eips.Field("associate_id").Label("guest_id")) + q = q.Equals("associate_type", "server") + q = q.In("associate_id", guestIds) + + geips := make([]sEipInfo, 0) + err := q.All(&geips) + if err != nil { + return nil + } + ret := make(map[string]sEipInfo) + for i := range geips { + ret[geips[i].GuestId] = geips[i] + } + return ret +} + +type sGuestKeypair struct { + GuestId string + Keypair string +} + +func fetchGuestKeypairs(guestIds []string) map[string]sGuestKeypair { + keypairs := KeypairManager.Query().SubQuery() + guests := GuestManager.Query().SubQuery() + + q := guests.Query(guests.Field("id").Label("guest_id"), keypairs.Field("name").Label("keypair")) + q = q.Join(keypairs, sqlchemy.Equals(guests.Field("keypair_id"), keypairs.Field("id"))) + q = q.Filter(sqlchemy.In(guests.Field("id"), guestIds)) + + gkps := make([]sGuestKeypair, 0) + err := q.All(&gkps) + if err != nil { + return nil + } + + ret := make(map[string]sGuestKeypair) + for i := range gkps { + ret[gkps[i].GuestId] = gkps[i] + } + return ret +} + +func fetchGuestIsolatedDevices(guestIds []string) map[string][]SIsolatedDevice { + q := IsolatedDeviceManager.Query().In("guest_id", guestIds) + devs := make([]SIsolatedDevice, 0) + err := q.All(&devs) + if err != nil { + return nil + } + ret := make(map[string][]SIsolatedDevice) + for i := range devs { + gdevs, ok := ret[devs[i].GuestId] + if !ok { + gdevs = make([]SIsolatedDevice, 0) + } + gdevs = append(gdevs, devs[i]) + ret[devs[i].GuestId] = gdevs + } + return ret +} + +func getIsolatedDeviceDetails(devs []SIsolatedDevice) string { + var buf strings.Builder + for _, dev := range devs { + buf.WriteString(dev.getDetailedString()) + buf.WriteString("\n") + } + return buf.String() +} + +func fetchGuestCdroms(guestIds []string) map[string]SGuestcdrom { + q := GuestcdromManager.Query().In("id", guestIds) + gcds := make([]SGuestcdrom, 0) + err := q.All(&gcds) + if err != nil { + return nil + } + ret := make(map[string]SGuestcdrom) + for i := range gcds { + ret[gcds[i].Id] = gcds[i] + } + return ret +} diff --git a/pkg/compute/models/guestdisks.go b/pkg/compute/models/guestdisks.go index 404098ea8b..46983f9fa2 100644 --- a/pkg/compute/models/guestdisks.go +++ b/pkg/compute/models/guestdisks.go @@ -45,7 +45,9 @@ func init() { DiskManager, ), } + GuestdiskManager.TableSpec().AddIndex(true, "disk_id", "guest_id") }) + } type SGuestdisk struct { diff --git a/pkg/compute/models/guestjoints.go b/pkg/compute/models/guestjoints.go index 74eec91f25..be5b07c29d 100644 --- a/pkg/compute/models/guestjoints.go +++ b/pkg/compute/models/guestjoints.go @@ -36,7 +36,7 @@ func NewGuestJointsManager(dt interface{}, tableName string, keyword string, key type SGuestJointsBase struct { db.SVirtualJointResourceBase - GuestId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required"` // Column(VARCHAR(36, charset='ascii'), nullable=False) + GuestId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"` // Column(VARCHAR(36, charset='ascii'), nullable=False) } func (self *SGuestJointsBase) getGuest() *SGuest { diff --git a/pkg/compute/models/guestnetworks.go b/pkg/compute/models/guestnetworks.go index 9dbd3bf567..f79f8fd1ca 100644 --- a/pkg/compute/models/guestnetworks.go +++ b/pkg/compute/models/guestnetworks.go @@ -60,6 +60,7 @@ func init() { NetworkManager, ), } + GuestnetworkManager.TableSpec().AddIndex(true, "ip_addr", "guest_id") }) } diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 31bcdc2af0..3672a4d53f 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -54,6 +54,7 @@ import ( "yunion.io/x/onecloud/pkg/util/logclient" "yunion.io/x/onecloud/pkg/util/netutils2" "yunion.io/x/onecloud/pkg/util/seclib2" + "yunion.io/x/onecloud/pkg/util/stringutils2" ) const ( @@ -1307,109 +1308,53 @@ func (self *SGuest) getExtBandwidth() int { func (self *SGuest) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { extra := self.SVirtualResourceBase.GetCustomizeColumns(ctx, userCred, query) - - if db.IsAdminAllowGet(userCred, self) { - host := self.GetHost() - if host != nil { - extra.Add(jsonutils.NewString(host.Name), "host") - } - } - extra.Add(jsonutils.NewString(strings.Join(self.getRealIPs(), ",")), "ips") - eip, _ := self.GetEip() - if eip != nil { - extra.Add(jsonutils.NewString(eip.IpAddr), "eip") - extra.Add(jsonutils.NewString(eip.Mode), "eip_mode") - } - extra.Add(jsonutils.NewInt(int64(self.getDiskSize())), "disk") - // flavor?? - // extra.Add(jsonutils.NewString(self.getFlavorName()), "flavor") - extra.Add(jsonutils.NewString(self.getKeypairName()), "keypair") - extra.Add(jsonutils.NewInt(int64(self.getExtBandwidth())), "ext_bw") - - extra.Add(jsonutils.NewString(self.GetSecgroupName()), "secgroup") - - if secgroups := self.getSecgroupJson(); len(secgroups) > 0 { - extra.Add(jsonutils.NewArray(secgroups...), "secgroups") - } - - if self.PendingDeleted { - pendingDeletedAt := self.PendingDeletedAt.Add(time.Second * time.Duration(options.Options.PendingDeleteExpireSeconds)) - extra.Add(jsonutils.NewString(timeutils.FullIsoTime(pendingDeletedAt)), "auto_delete_at") - } - - isGpu := jsonutils.JSONFalse - if self.isGpu() { - isGpu = jsonutils.JSONTrue - } - extra.Add(isGpu, "is_gpu") - - extra.Add(jsonutils.JSONNull, "cdrom") - if cdrom := self.getCdrom(); cdrom != nil { - extra.Set("cdrom", jsonutils.NewString(cdrom.GetDetails())) - } - - return self.moreExtraInfo(extra) + fields := stringutils2.NewSortedStrings(jsonutils.GetQueryStringArray(query, "field")) + return self.moreExtraInfo(extra, fields) } -func (self *SGuest) moreExtraInfo(extra *jsonutils.JSONDict) *jsonutils.JSONDict { - /*zone := self.getZone() - if zone != nil { - extra.Add(jsonutils.NewString(zone.GetId()), "zone_id") - extra.Add(jsonutils.NewString(zone.GetName()), "zone") - if len(zone.ExternalId) > 0 { - extra.Add(jsonutils.NewString(zone.ExternalId), "zone_external_id") +func (self *SGuest) moreExtraInfo(extra *jsonutils.JSONDict, fields stringutils2.SSortedStrings) *jsonutils.JSONDict { + // extra.Add(jsonutils.NewInt(int64(self.getExtBandwidth())), "ext_bw") + + if len(self.BackupHostId) > 0 && (len(fields) == 0 || fields.Contains("backup_host_name") || fields.Contains("backup_host_status")) { + backupHost := HostManager.FetchHostById(self.BackupHostId) + if len(fields) == 0 || fields.Contains("backup_host_name") { + extra.Set("backup_host_name", jsonutils.NewString(backupHost.Name)) } - - region := zone.GetRegion() - if region != nil { - extra.Add(jsonutils.NewString(region.Id), "region_id") - extra.Add(jsonutils.NewString(region.Name), "region") - - if len(region.ExternalId) > 0 { - extra.Add(jsonutils.NewString(region.ExternalId), "region_external_id") - } + if len(fields) == 0 || fields.Contains("backup_host_status") { + extra.Set("backup_host_status", jsonutils.NewString(backupHost.HostStatus)) } + } + if len(fields) == 0 || fields.Contains("host") || fields.ContainsAny(providerInfoFields...) { host := self.GetHost() if host != nil { - provider := host.GetCloudprovider() - if provider != nil { - extra.Add(jsonutils.NewString(host.ManagerId), "manager_id") - extra.Add(jsonutils.NewString(provider.GetName()), "manager") + if len(fields) == 0 || fields.Contains("host") { + extra.Add(jsonutils.NewString(host.Name), "host") + } + if len(fields) == 0 || fields.ContainsAny(providerInfoFields...) { + info := host.getCloudProviderInfo() + if len(fields) == 0 { + extra.Update(jsonutils.Marshal(&info)) + } else { + extra.Update(jsonutils.Marshal(&info).(*jsonutils.JSONDict).CopyIncludes([]string(fields)...)) + } } } - }*/ - - extra.Add(self.getDisksInfoDetails(), "disks_info") - extra.Add(jsonutils.NewString(self.getIsolatedDeviceDetails()), "isolated_devices") - - if len(self.BackupHostId) > 0 { - backupHost := HostManager.FetchHostById(self.BackupHostId) - extra.Set("backup_host_name", jsonutils.NewString(backupHost.Name)) - extra.Set("backup_host_status", jsonutils.NewString(backupHost.HostStatus)) } - host := self.GetHost() - if host != nil { - info := host.getCloudProviderInfo() - extra.Update(jsonutils.Marshal(&info)) + if len(fields) == 0 || fields.Contains("can_recycle") { + err := self.CanPerformPrepaidRecycle() + if err != nil { + extra.Add(jsonutils.JSONFalse, "can_recycle") + } else { + extra.Add(jsonutils.JSONTrue, "can_recycle") + } } - err := self.CanPerformPrepaidRecycle() - if err != nil { - extra.Add(jsonutils.JSONFalse, "can_recycle") - } else { - extra.Add(jsonutils.JSONTrue, "can_recycle") - } - - guestnetworks, _ := self.GetNetworks("") - if len(guestnetworks) > 0 { - guestnetwork := guestnetworks[0] - network := guestnetwork.GetNetwork() - if network != nil { - vpc := network.GetVpc() - extra.Set("vpc_id", jsonutils.NewString(vpc.Id)) - extra.Set("vpc", jsonutils.NewString(vpc.Name)) + if len(fields) == 0 || fields.Contains("auto_delete_at") { + if self.PendingDeleted { + pendingDeletedAt := self.PendingDeletedAt.Add(time.Second * time.Duration(options.Options.PendingDeleteExpireSeconds)) + extra.Add(jsonutils.NewString(timeutils.FullIsoTime(pendingDeletedAt)), "auto_delete_at") } } @@ -1424,21 +1369,11 @@ func (self *SGuest) GetExtraDetails(ctx context.Context, userCred mcclient.Token extra.Add(jsonutils.NewString(self.getNetworksDetails()), "networks") extra.Add(jsonutils.NewString(self.getDisksDetails()), "disks") - extra.Add(jsonutils.NewInt(int64(self.getDiskSize())), "disk") - cdrom := self.getCdrom() - if cdrom != nil { - extra.Add(jsonutils.NewString(cdrom.GetDetails()), "cdrom") - } - // extra.Add(jsonutils.NewString(self.getFlavorName()), "flavor") - extra.Add(jsonutils.NewString(self.getKeypairName()), "keypair") - extra.Add(jsonutils.NewString(self.GetSecgroupName()), "secgroup") + extra.Add(self.getDisksInfoDetails(), "disks_info") - if secgroups := self.getSecgroupJson(); len(secgroups) > 0 { - extra.Add(jsonutils.NewArray(secgroups...), "secgroups") - } - - extra.Add(jsonutils.NewString(strings.Join(self.getIPs(), ",")), "ips") + extra.Add(jsonutils.NewString(strings.Join(self.getVirtualIPs(), ",")), "virtual_ips") extra.Add(jsonutils.NewString(self.getSecurityGroupsRules()), "security_rules") + osName := self.GetOS() if len(osName) > 0 { extra.Add(jsonutils.NewString(osName), "os_name") @@ -1446,27 +1381,14 @@ func (self *SGuest) GetExtraDetails(ctx context.Context, userCred mcclient.Token extra.Add(jsonutils.NewString(osName), "os_type") } } + if metaData, err := self.GetAllMetadata(userCred); err == nil { extra.Add(jsonutils.Marshal(metaData), "metadata") } + if db.IsAdminAllowGet(userCred, self) { - host := self.GetHost() - if host != nil { - extra.Add(jsonutils.NewString(host.GetName()), "host") - } extra.Add(jsonutils.NewString(self.getAdminSecurityRules()), "admin_security_rules") } - eip, _ := self.GetEip() - if eip != nil { - extra.Add(jsonutils.NewString(eip.IpAddr), "eip") - extra.Add(jsonutils.NewString(eip.Mode), "eip_mode") - } - - isGpu := jsonutils.JSONFalse - if self.isGpu() { - isGpu = jsonutils.JSONTrue - } - extra.Add(isGpu, "is_gpu") if self.IsPrepaidRecycle() { extra.Add(jsonutils.JSONTrue, "is_prepaid_recycle") @@ -1474,7 +1396,7 @@ func (self *SGuest) GetExtraDetails(ctx context.Context, userCred mcclient.Token extra.Add(jsonutils.JSONFalse, "is_prepaid_recycle") } - return self.moreExtraInfo(extra), nil + return self.moreExtraInfo(extra, nil), nil } func (manager *SGuestManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { @@ -1600,41 +1522,28 @@ func (self *SGuest) getDisksInfoDetails() *jsonutils.JSONArray { return details } -func (self *SGuest) getIsolatedDeviceDetails() string { - var buf bytes.Buffer - for _, dev := range self.GetIsolatedDevices() { - buf.WriteString(dev.getDetailedString()) - buf.WriteString("\n") - } - return buf.String() -} - -func (self *SGuest) getDiskSize() int { - size := 0 - for _, disk := range self.GetDisks() { - size += disk.GetDisk().DiskSize - } - return size -} - func (self *SGuest) GetCdrom() *SGuestcdrom { - return self.getCdrom() + return self.getCdrom(false) } -func (self *SGuest) getCdrom() *SGuestcdrom { +func (self *SGuest) getCdrom(create bool) *SGuestcdrom { cdrom := SGuestcdrom{} cdrom.SetModelManager(GuestcdromManager) err := GuestcdromManager.Query().Equals("id", self.Id).First(&cdrom) if err != nil { if err == sql.ErrNoRows { - cdrom.Id = self.Id - err = GuestcdromManager.TableSpec().Insert(&cdrom) - if err != nil { - log.Errorf("insert cdrom fail %s", err) + if create { + cdrom.Id = self.Id + err = GuestcdromManager.TableSpec().Insert(&cdrom) + if err != nil { + log.Errorf("insert cdrom fail %s", err) + return nil + } + return &cdrom + } else { return nil } - return &cdrom } else { log.Errorf("getCdrom query fail %s", err) return nil @@ -1663,7 +1572,7 @@ func (self *SGuest) getKeypairName() string { } func (self *SGuest) getNotifyIps() string { - ips := self.getRealIPs() + ips := self.GetRealIPs() vips := self.getVirtualIPs() if vips != nil { ips = append(ips, vips...) @@ -1671,7 +1580,8 @@ func (self *SGuest) getNotifyIps() string { return strings.Join(ips, ",") } -func (self *SGuest) getRealIPs() []string { +/* +func (self *SGuest) GetRealIPs() []string { guestnets, err := self.GetNetworks("") if err != nil { return nil @@ -1684,9 +1594,10 @@ func (self *SGuest) getRealIPs() []string { } return ips } +*/ func (self *SGuest) IsExitOnly() bool { - for _, ip := range self.getRealIPs() { + for _, ip := range self.GetRealIPs() { addr, _ := netutils.NewIPV4Addr(ip) if !netutils.IsExitAddress(addr) { return false @@ -1711,7 +1622,7 @@ func (self *SGuest) getVirtualIPs() []string { } func (self *SGuest) getIPs() []string { - ips := self.getRealIPs() + ips := self.GetRealIPs() vips := self.getVirtualIPs() ips = append(ips, vips...) /*eip, _ := self.GetEip() @@ -2846,8 +2757,8 @@ func (self *SGuest) DetachAllNetworks(ctx context.Context, userCred mcclient.Tok } func (self *SGuest) EjectIso(userCred mcclient.TokenCredential) bool { - cdrom := self.getCdrom() - if len(cdrom.ImageId) > 0 { + cdrom := self.getCdrom(false) + if cdrom != nil && len(cdrom.ImageId) > 0 { imageId := cdrom.ImageId if cdrom.ejectIso() { db.OpsLog.LogEvent(self, db.ACT_ISO_DETACH, imageId, userCred) @@ -3154,9 +3065,12 @@ func (self *SGuest) GetJsonDescAtHypervisor(ctx context.Context, host *SHost) *j desc.Add(jsonutils.NewArray(jsonDisks...), "disks") // cdrom - cdDesc := self.getCdrom().getJsonDesc() - if cdDesc != nil { - desc.Add(cdDesc, "cdrom") + cdrom := self.getCdrom(false) + if cdrom != nil { + cdDesc := cdrom.getJsonDesc() + if cdDesc != nil { + desc.Add(cdDesc, "cdrom") + } } // tenant @@ -3495,7 +3409,7 @@ func (self *SGuest) GetShortDesc(ctx context.Context) *jsonutils.JSONDict { desc.Set("mem", jsonutils.NewInt(int64(self.VmemSize))) desc.Set("cpu", jsonutils.NewInt(int64(self.VcpuCount))) - address := jsonutils.NewString(strings.Join(self.getRealIPs(), ",")) + address := jsonutils.NewString(strings.Join(self.GetRealIPs(), ",")) desc.Set("ip_addr", address) if len(self.OsType) > 0 { @@ -3770,10 +3684,6 @@ func (self *SGuest) GetEip() (*SElasticip, error) { return ElasticipManager.getEipForInstance("server", self.Id) } -func (self *SGuest) GetRealIps() []string { - return self.getRealIPs() -} - func (self *SGuest) SyncVMEip(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, extEip cloudprovider.ICloudEIP, projectId string) compare.SyncResult { result := compare.SyncResult{} @@ -4169,7 +4079,7 @@ func (self *SGuest) toCreateInput() *api.ServerCreateInput { r := new(api.ServerCreateInput) r.VmemSize = self.VmemSize r.VcpuCount = int(self.VcpuCount) - if guestCdrom := self.getCdrom(); guestCdrom != nil { + if guestCdrom := self.getCdrom(false); guestCdrom != nil { r.Cdrom = guestCdrom.ImageId } r.Vga = self.Vga diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 230f6d2cb3..dbc901a5d0 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -2182,7 +2182,7 @@ func (self *SHost) getMoreDetails(ctx context.Context, extra *jsonutils.JSONDict extra.Add(jsonutils.NewString(server.Id), "server_id") extra.Add(jsonutils.NewString(server.Name), "server") if self.HostType == HOST_TYPE_BAREMETAL { - extra.Add(jsonutils.NewString(strings.Join(server.getRealIPs(), ",")), "server_ips") + extra.Add(jsonutils.NewString(strings.Join(server.GetRealIPs(), ",")), "server_ips") } } netifs := self.GetNetInterfaces() diff --git a/pkg/compute/models/managedresource.go b/pkg/compute/models/managedresource.go index f4518b148e..fe82c2da21 100644 --- a/pkg/compute/models/managedresource.go +++ b/pkg/compute/models/managedresource.go @@ -241,6 +241,24 @@ type SCloudProviderInfo struct { ZoneExtId string `json:",omitempty"` } +var ( + providerInfoFields = []string{ + "provider", + "account", + "account_id", + "manager", + "manager_id", + "manager_project", + "manager_project_id", + "region", + "region_id", + "region_ext_id", + "zone", + "zone_id", + "zone_ext_id", + } +) + func fetchExternalId(extId string) string { pos := strings.LastIndexByte(extId, '/') if pos > 0 { diff --git a/pkg/mcclient/options/base.go b/pkg/mcclient/options/base.go index 529b3465c9..26b4497f43 100644 --- a/pkg/mcclient/options/base.go +++ b/pkg/mcclient/options/base.go @@ -187,7 +187,7 @@ type BaseListOptions struct { Order string `help:"List order" choices:"desc|asc"` Details *bool `help:"Show more details" default:"false"` Search string `help:"Filter results by a simple keyword search"` - Meta *bool `help:"Piggyback metadata information"` + Meta *bool `help:"Piggyback metadata information" json:"with_meta" token:"meta"` Filter []string `help:"Filters"` JointFilter []string `help:"Filters with joint table col; joint_tbl.related_key(origin_key).filter_col.filter_cond(filters)"` FilterAny *bool `help:"If true, match if any of the filters matches; otherwise, match if all of the filters match"` diff --git a/pkg/util/stringutils2/sortedstrings.go b/pkg/util/stringutils2/sortedstrings.go new file mode 100644 index 0000000000..17078577af --- /dev/null +++ b/pkg/util/stringutils2/sortedstrings.go @@ -0,0 +1,113 @@ +package stringutils2 + +import ( + "sort" +) + +type SSortedStrings []string + +func NewSortedStrings(strs []string) SSortedStrings { + if strs == nil { + return nil + } + sort.Strings(strs) + return SSortedStrings(strs) +} + +func (ss SSortedStrings) Index(needle string) (int, bool) { + i := 0 + j := len(ss) - 1 + for i <= j { + m := (i + j) / 2 + if ss[m] < needle { + i = m + 1 + } else if ss[m] > needle { + j = m - 1 + } else { + return m, true + } + } + return j + 1, false +} + +func (ss SSortedStrings) Contains(needle string) bool { + _, find := ss.Index(needle) + return find +} + +func (ss SSortedStrings) ContainsAny(needles ...string) bool { + for i := range needles { + _, find := ss.Index(needles[i]) + if find { + return true + } + } + return false +} + +func (ss SSortedStrings) ContainsAll(needles ...string) bool { + for i := range needles { + _, find := ss.Index(needles[i]) + if !find { + return false + } + } + return true +} + +func Split(a, b SSortedStrings) (aNoB SSortedStrings, aAndB SSortedStrings, bNoA SSortedStrings) { + a_b := make([]string, 0) + b_a := make([]string, 0) + anb := make([]string, 0) + i := 0 + j := 0 + for i < len(a) && j < len(b) { + if a[i] == b[j] { + anb = append(anb, a[i]) + i += 1 + j += 1 + } else if a[i] < b[j] { + a_b = append(a_b, a[i]) + i += 1 + } else if a[i] > b[j] { + b_a = append(b_a, b[j]) + j += 1 + } + } + if i < len(a) { + a_b = append(a_b, a[i:]...) + } + if j < len(b) { + b_a = append(b_a, b[j:]...) + } + aNoB = SSortedStrings(a_b) + aAndB = SSortedStrings(anb) + bNoA = SSortedStrings(b_a) + return +} + +func Merge(a, b SSortedStrings) SSortedStrings { + ret := make([]string, 0) + i := 0 + j := 0 + for i < len(a) && j < len(b) { + if a[i] == b[j] { + ret = append(ret, a[i]) + i += 1 + j += 1 + } else if a[i] < b[j] { + ret = append(ret, a[i]) + i += 1 + } else if a[i] > b[j] { + ret = append(ret, b[j]) + j += 1 + } + } + if i < len(a) { + ret = append(ret, a[i:]...) + } + if j < len(b) { + ret = append(ret, b[j:]...) + } + return SSortedStrings(ret) +} diff --git a/pkg/util/stringutils2/sortedstrings_test.go b/pkg/util/stringutils2/sortedstrings_test.go new file mode 100644 index 0000000000..28b1b803ac --- /dev/null +++ b/pkg/util/stringutils2/sortedstrings_test.go @@ -0,0 +1,62 @@ +package stringutils2 + +import ( + "testing" +) + +func TestSortedString(t *testing.T) { + input := []string{"Go", "Bravo", "Gopher", "Alpha", "Grin", "Delta"} + // Alpha Bravo Delta Go Gopher Grin + // 0 1 2 3 4 5 + ss := NewSortedStrings(input) + cases := []struct { + Needle string + Index int + Want bool + }{ + {"Go", 3, true}, + {"Bravo", 1, true}, + {"Gopher", 4, true}, + {"Alpha", 0, true}, + {"Grin", 5, true}, + {"Delta", 2, true}, + {"Go1", 4, false}, + {"G", 3, false}, + {"A", 0, false}, + {"T", 6, false}, + } + for _, c := range cases { + idx, find := ss.Index(c.Needle) + if idx != c.Index || find != c.Want { + t.Errorf("%s: want: %d %v got: %d %v", c.Needle, c.Index, c.Want, idx, find) + } + } +} + +func TestSplitStrings(t *testing.T) { + input := []string{"Go", "Bravo", "Gopher", "Alpha", "Grin", "Delta"} + input2 := []string{"Go2", "Bravo", "Gopher", "Alpha1", "Grin", "Delt"} + + ss1 := NewSortedStrings(input) + ss2 := NewSortedStrings(input2) + + a, b, c := Split(ss1, ss2) + t.Logf("A: %s", ss1) + t.Logf("B: %s", ss2) + t.Logf("A-B: %s", a) + t.Logf("AnB: %s", b) + t.Logf("B-A: %s", c) +} + +func TestMergeStrings(t *testing.T) { + input := []string{"Go", "Bravo", "Gopher", "Alpha", "Grin", "Delta"} + input2 := []string{"Go2", "Bravo", "Gopher", "Alpha1", "Grin", "Delt"} + + ss1 := NewSortedStrings(input) + ss2 := NewSortedStrings(input2) + + m := Merge(ss1, ss2) + t.Logf("A: %s", ss1) + t.Logf("B: %s", ss2) + t.Logf("%s", m) +} diff --git a/vendor/yunion.io/x/sqlchemy/query.go b/vendor/yunion.io/x/sqlchemy/query.go index 0aa5349444..3bebe434b0 100644 --- a/vendor/yunion.io/x/sqlchemy/query.go +++ b/vendor/yunion.io/x/sqlchemy/query.go @@ -10,6 +10,14 @@ import ( "yunion.io/x/pkg/util/reflectutils" ) +type IQuery interface { + String() string + QueryFields() []IQueryField + Variables() []interface{} + SubQuery() *SSubQuery + Field(name string) IQueryField +} + type IQuerySource interface { Expression() string Alias() string @@ -67,7 +75,7 @@ type SQuery struct { } type SSubQuery struct { - query *SQuery + query IQuery alias string } @@ -117,7 +125,7 @@ func (sq *SSubQuery) Variables() []interface{} { } func (sq *SSubQuery) Field(id string, alias ...string) IQueryField { - for _, f := range sq.query.fields { + for _, f := range sq.query.QueryFields() { if f.Name() == id { sqf := SSubQueryField{query: sq, field: f} if len(alias) > 0 { @@ -131,7 +139,7 @@ func (sq *SSubQuery) Field(id string, alias ...string) IQueryField { func (sq *SSubQuery) Fields() []IQueryField { ret := make([]IQueryField, 0) - for _, f := range sq.query.fields { + for _, f := range sq.query.QueryFields() { sqf := SSubQueryField{query: sq, field: f} ret = append(ret, &sqf) } @@ -184,34 +192,44 @@ func (tq *SQuery) _orderBy(order QueryOrderType, fields []IQueryField) *SQuery { return tq } -func (tq *SQuery) convertQueryField(fields []interface{}) []IQueryField { +func (tq *SQuery) Asc(fields ...interface{}) *SQuery { + return tq._orderBy(SQL_ORDER_ASC, convertQueryField(tq, fields)) +} + +func (tq *SQuery) Desc(fields ...interface{}) *SQuery { + return tq._orderBy(SQL_ORDER_DESC, convertQueryField(tq, fields)) +} + +func convertQueryField(tq IQuery, fields []interface{}) []IQueryField { nFields := make([]IQueryField, 0) for _, f := range fields { - // log.Debugf("%s %s", f, reflect.TypeOf(f)) - if fName, ok := f.(string); ok { - nFields = append(nFields, tq.Field(fName)) - } else if fq, ok := f.(IQueryField); ok { - nFields = append(nFields, fq) - } else { + switch ff := f.(type) { + case string: + nFields = append(nFields, tq.Field(ff)) + case IQueryField: + find := false + for _, f := range tq.QueryFields() { + if f.Name() == ff.Name() { + find = true + nFields = append(nFields, ff) + break + } + } + if !find { + log.Errorf("Fail to find query field %s in query", f) + } + default: log.Errorf("Invalid query field %s neither string nor IQueryField", f) } } return nFields } -func (tq *SQuery) Asc(fields ...interface{}) *SQuery { - return tq._orderBy(SQL_ORDER_ASC, tq.convertQueryField(fields)) -} - -func (tq *SQuery) Desc(fields ...interface{}) *SQuery { - return tq._orderBy(SQL_ORDER_DESC, tq.convertQueryField(fields)) -} - func (tq *SQuery) GroupBy(f ...interface{}) *SQuery { if tq.groupBy == nil { tq.groupBy = make([]IQueryField, 0) } - qfs := tq.convertQueryField(f) + qfs := convertQueryField(tq, f) tq.groupBy = append(tq.groupBy, qfs...) return tq } @@ -226,6 +244,10 @@ func (tq *SQuery) Offset(offset int) *SQuery { return tq } +func (tq *SQuery) QueryFields() []IQueryField { + return tq.fields +} + func (tq *SQuery) String() string { sql := queryString(tq) // log.Debugf("Query: %s", sql) diff --git a/vendor/yunion.io/x/sqlchemy/union.go b/vendor/yunion.io/x/sqlchemy/union.go new file mode 100644 index 0000000000..64fe31387d --- /dev/null +++ b/vendor/yunion.io/x/sqlchemy/union.go @@ -0,0 +1,149 @@ +package sqlchemy + +import ( + "fmt" + "strings" + + "yunion.io/x/log" +) + +type SUnionQueryField struct { + name string +} + +func (sqf *SUnionQueryField) Expression() string { + return sqf.name +} + +func (sqf *SUnionQueryField) Name() string { + return sqf.name +} + +func (sqf *SUnionQueryField) Reference() string { + return sqf.name +} + +func (sqf *SUnionQueryField) Label(label string) IQueryField { + return sqf +} + +type SUnionQuery struct { + queries []IQuery + fields []IQueryField + orderBy []SQueryOrder + limit int + offset int +} + +func (uq *SUnionQuery) String() string { + var buf strings.Builder + for i := range uq.queries { + if i != 0 { + buf.WriteString(" UNION ") + } + buf.WriteByte('(') + buf.WriteString(uq.queries[i].String()) + buf.WriteByte(')') + } + if uq.orderBy != nil && len(uq.orderBy) > 0 { + buf.WriteString(" ORDER BY ") + for i, f := range uq.orderBy { + if i > 0 { + buf.WriteString(", ") + } + buf.WriteString(fmt.Sprintf("%s %s", f.field.Reference(), f.order)) + } + } + if uq.limit > 0 { + buf.WriteString(fmt.Sprintf(" LIMIT %d", uq.limit)) + } + if uq.offset > 0 { + buf.WriteString(fmt.Sprintf(" OFFSET %d", uq.offset)) + } + return buf.String() +} + +func (tq *SUnionQuery) _orderBy(order QueryOrderType, fields []IQueryField) *SUnionQuery { + if tq.orderBy == nil { + tq.orderBy = make([]SQueryOrder, 0) + } + for _, f := range fields { + tq.orderBy = append(tq.orderBy, SQueryOrder{field: f, order: order}) + } + return tq +} + +func (tq *SUnionQuery) Asc(fields ...interface{}) *SUnionQuery { + return tq._orderBy(SQL_ORDER_ASC, convertQueryField(tq, fields)) +} + +func (tq *SUnionQuery) Desc(fields ...interface{}) *SUnionQuery { + return tq._orderBy(SQL_ORDER_DESC, convertQueryField(tq, fields)) +} + +func (uq *SUnionQuery) Limit(limit int) *SUnionQuery { + uq.limit = limit + return uq +} + +func (uq *SUnionQuery) Offset(offset int) *SUnionQuery { + uq.offset = offset + return uq +} + +func (uq *SUnionQuery) QueryFields() []IQueryField { + return uq.fields +} + +func (uq *SUnionQuery) Field(name string) IQueryField { + for i := range uq.fields { + if name == uq.fields[i].Name() { + return uq.fields[i] + } + } + return nil +} + +func (uq *SUnionQuery) Variables() []interface{} { + ret := make([]interface{}, 0) + for i := range uq.queries { + ret = append(ret, uq.queries[i].Variables()...) + } + return ret +} + +func (uq *SUnionQuery) SubQuery() *SSubQuery { + sq := SSubQuery{query: uq, alias: getTableAliasName()} + return &sq +} + +func Union(query ...IQuery) *SUnionQuery { + fieldNames := make([]string, 0) + for _, f := range query[0].QueryFields() { + fieldNames = append(fieldNames, f.Name()) + } + + for i := 1; i < len(query); i += 1 { + qfields := query[i].QueryFields() + if len(fieldNames) != len(qfields) { + log.Fatalf("cannot union, number of fields not match!") + } + for i := range qfields { + if fieldNames[i] != qfields[i].Name() { + log.Fatalf("cannot union, name of fields not match!") + } + } + } + + fields := make([]IQueryField, len(fieldNames)) + for i := range fieldNames { + fields[i] = &SUnionQueryField{name: fieldNames[i]} + } + + uq := &SUnionQuery{ + queries: query, + fields: fields, + } + + return uq +}