Merge branch 'master' into bugfix/zxc-monitor

build/docker/Dockerfile.ansibleserver

@@ -1,7 +1,7 @@
 FROM registry.cn-beijing.aliyuncs.com/yunionio/ansibleserver-base:v1.0.3
 
 # install playbook and telegraf install pkg
-COPY --from=registry.cn-beijing.aliyuncs.com/yunionio/file-repo:v0.1.1 /opt/yunion/playbook /opt/yunion/playbook
-COPY --from=registry.cn-beijing.aliyuncs.com/yunionio/file-repo:v0.1.1 /opt/yunion/ansible-install-pkg /opt/yunion/ansible-install-pkg
+COPY --from=registry.cn-beijing.aliyuncs.com/yunionio/file-repo:v0.1.2 /opt/yunion/playbook /opt/yunion/playbook
+COPY --from=registry.cn-beijing.aliyuncs.com/yunionio/file-repo:v0.1.2 /opt/yunion/ansible-install-pkg /opt/yunion/ansible-install-pkg
 
 ADD ./_output/alpine-build/bin/ansibleserver /opt/yunion/bin/ansibleserver

cmd/climc/shell/compute/hosts.go

@@ -60,6 +60,8 @@ func init() {
 
 		Sn string `help:"find host by sn"`
 
+		OrderByServerCount string `help:"Order by server count" choices:"desc|asc"`
+
 		options.BaseListOptions
 	}
 	R(&HostListOptions{}, "host-list", "List hosts", func(s *mcclient.ClientSession, opts *HostListOptions) error {

pkg/apis/compute/host.go

@@ -124,6 +124,10 @@ type HostListInput struct {
 	ServerIdForNetwork string `json:"server_id_for_network"`
 	// 宿主机 cpu 架构
 	CpuArchitecture string `json:"cpu_architecture"`
+
+	// 按虚拟机数量排序
+	// enum: asc,desc
+	OrderByServerCount string `json:"order_by_server_count"`
 }
 
 type HostDetails struct {

pkg/apis/metadata.go

@@ -14,6 +14,10 @@
 
 package apis
 
+const (
+	IS_SYSTEM = "IsSystem"
+)
+
 type MetadataListInput struct {
 	ModelBaseListInput
 

pkg/cloudcommon/db/db_dispatcher.go

@@ -898,6 +898,10 @@ func getModelItemDetails(manager IModelManager, item IModel, ctx context.Context
 	}
 }
 
+func GetItemDetails(manager IModelManager, item IModel, ctx context.Context, userCred mcclient.TokenCredential) (jsonutils.JSONObject, error) {
+	return getItemDetails(manager, item, ctx, userCred, nil)
+}
+
 func getItemDetails(manager IModelManager, item IModel, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	metaFields, excludeFields := GetDetailFields(manager, userCred)
 	fieldFilter := jsonutils.GetQueryStringArray(query, "field")

pkg/cloudcommon/db/virtualresource.go

@@ -113,6 +113,14 @@ func (manager *SVirtualResourceBaseManager) FilterByHiddenSystemAttributes(q *sq
 	return q
 }
 
+func (model *SVirtualResourceBase) SetSystemInfo(isSystem bool) error {
+	_, err := Update(model, func() error {
+		model.IsSystem = isSystem
+		return nil
+	})
+	return err
+}
+
 func (model *SVirtualResourceBase) SetProjectInfo(ctx context.Context, userCred mcclient.TokenCredential, projectId, domainId string) error {
 	_, err := Update(model, func() error {
 		model.ProjectId = projectId

pkg/cloudproxy/agent/ssh/client.go

@@ -186,10 +186,17 @@ func (c *Client) runClientState(ctx context.Context, sshClientC chan *ssh.Client
 		default:
 		}
 
+		cc := c.cc
 		tmoCtx, _ := context.WithTimeout(ctx, 31*time.Second)
-		sshc, err := c.cc.ConnectContext(tmoCtx)
+		sshc, err := cc.ConnectContext(tmoCtx)
 		if err != nil {
-			log.Errorf("ssh connect: %v", err)
+			log.Errorf("ssh connect: %s@%s, port %d: %v", cc.Username, cc.Host, cc.Port, err)
+			waitTmo := time.NewTimer(13 * time.Second)
+			select {
+			case <-ctx.Done():
+				return
+			case <-waitTmo.C:
+			}
 			continue
 		}
 

pkg/cloudproxy/models/forwards.go

@@ -256,7 +256,11 @@ func (man *SForwardManager) PerformCreateFromServer(ctx context.Context, userCre
 		data, err = man.validatePortReq(ctx, typ, -1, agentId, epId, data)
 	}
 
-	forward := &SForward{}
+	forwardObj, err := db.NewModelObject(man)
+	if err != nil {
+		return nil, httperrors.NewGeneralError(err)
+	}
+	forward := forwardObj.(*SForward)
 	if err := data.Unmarshal(forward); err != nil {
 		return nil, httperrors.NewServerError("unmarshal create params: %v", err)
 	}
@@ -267,7 +271,7 @@ func (man *SForwardManager) PerformCreateFromServer(ctx context.Context, userCre
 		return nil, httperrors.NewServerError("database insertion error: %v", err)
 	}
 
-	return jsonutils.Marshal(forward), nil
+	return db.GetItemDetails(man, forward, ctx, userCred)
 }
 
 func (man *SForwardManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {

pkg/cloudproxy/models/proxy_endpoints.go

@@ -110,8 +110,8 @@ func (man *SProxyEndpointManager) PerformCreateFromServer(ctx context.Context, u
 		IntranetIpAddr: nic.IpAddr,
 	}
 	proxyendpoint.Name = name
-	proxyendpoint.DomainId = userCred.GetProjectDomainId()
-	proxyendpoint.ProjectId = userCred.GetProjectId()
+	proxyendpoint.DomainId = serverInfo.Server.DomainId
+	proxyendpoint.ProjectId = serverInfo.Server.ProjectId
 	if err := man.TableSpec().Insert(ctx, proxyendpoint); err != nil {
 		return nil, httperrors.NewServerError("database insertion error: %v", err)
 	}

pkg/compute/models/guest_sshable.go

@@ -207,9 +207,22 @@ func (guest *SGuest) sshableTryEach(
 		res, err := cloudproxy_module.Forwards.PerformClassAction(sess, "create-from-server", fwdCreateParams)
 		if err == nil {
 			var fwd cloudproxy_api.ForwardDetails
-			if err := res.Unmarshal(&fwd); err == nil {
-				if ok := guest.sshableTryForward(ctx, tryData, &fwd); ok {
-					return nil
+			if err := res.Unmarshal(&fwd); err != nil {
+				log.Errorf("unmarshal fwd details: %q", res.String())
+			}
+
+			tmo := time.NewTimer(13 * time.Second)
+			tick := time.NewTicker(3 * time.Second)
+			for {
+				select {
+				case <-tmo.C:
+					break
+				case <-tick.C:
+					if ok := guest.sshableTryForward(ctx, tryData, &fwd); ok {
+						return nil
+					}
+				case <-ctx.Done():
+					break
 				}
 			}
 		} else {

pkg/compute/models/hosts.go

@@ -490,6 +490,17 @@ func (manager *SHostManager) OrderByExtraFields(
 	if err != nil {
 		return nil, errors.Wrap(err, "SZoneResourceBaseManager.OrderByExtraFields")
 	}
+
+	if db.NeedOrderQuery([]string{query.OrderByServerCount}) {
+		guests := GuestManager.Query().SubQuery()
+		guestCounts := guests.Query(
+			guests.Field("host_id"),
+			sqlchemy.COUNT("id").Label("guest_count"),
+		).GroupBy("host_id").SubQuery()
+		q = q.LeftJoin(guestCounts, sqlchemy.Equals(q.Field("id"), guestCounts.Field("host_id")))
+		db.OrderByFields(q, []string{query.OrderByServerCount}, []sqlchemy.IQueryField{guestCounts.Field("guest_count")})
+	}
+
 	return q, nil
 }
 

pkg/compute/models/syncutils.go

@@ -19,6 +19,7 @@ import (
 
 	"yunion.io/x/log"
 
+	"yunion.io/x/onecloud/pkg/apis"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -32,6 +33,11 @@ type IMetadataSetter interface {
 	GetCloudproviderId() string
 }
 
+type IVirtualResourceMetadataSetter interface {
+	IMetadataSetter
+	SetSystemInfo(isSystem bool) error
+}
+
 func syncMetadata(ctx context.Context, userCred mcclient.TokenCredential, model IMetadataSetter, remote cloudprovider.ICloudResource) error {
 	sysTags := remote.GetSysTags()
 	sysStore := make(map[string]interface{}, 0)
@@ -51,10 +57,13 @@ func syncMetadata(ctx context.Context, userCred mcclient.TokenCredential, model
 	return nil
 }
 
-func syncVirtualResourceMetadata(ctx context.Context, userCred mcclient.TokenCredential, model IMetadataSetter, remote cloudprovider.IVirtualResource) error {
+func syncVirtualResourceMetadata(ctx context.Context, userCred mcclient.TokenCredential, model IVirtualResourceMetadataSetter, remote cloudprovider.IVirtualResource) error {
 	sysTags := remote.GetSysTags()
 	sysStore := make(map[string]interface{}, 0)
 	for key, value := range sysTags {
+		if key == apis.IS_SYSTEM && value == "true" {
+			model.SetSystemInfo(true)
+		}
 		sysStore[db.SYS_CLOUD_TAG_PREFIX+key] = value
 	}
 	extProjectId := remote.GetProjectId()
@@ -84,6 +93,6 @@ func SyncMetadata(ctx context.Context, userCred mcclient.TokenCredential, model
 	return syncMetadata(ctx, userCred, model, remote)
 }
 
-func SyncVirtualResourceMetadata(ctx context.Context, userCred mcclient.TokenCredential, model IMetadataSetter, remote cloudprovider.IVirtualResource) error {
+func SyncVirtualResourceMetadata(ctx context.Context, userCred mcclient.TokenCredential, model IVirtualResourceMetadataSetter, remote cloudprovider.IVirtualResource) error {
 	return syncVirtualResourceMetadata(ctx, userCred, model, remote)
 }

pkg/compute/sshkeys/handler.go

@@ -41,7 +41,7 @@ func AddSshKeysHandler(prefix string, app *appsrv.Application) {
 func adminSshKeysHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	publicOnly := false
 	userCred := auth.FetchUserCredential(ctx, policy.FilterPolicyCredential)
-	if !userCred.IsAllow(rbacutils.ScopeSystem, consts.GetServiceType(), "sshkeypairs", policy.PolicyActionGet) {
+	if !userCred.IsAllow(rbacutils.ScopeDomain, consts.GetServiceType(), "sshkeypairs", policy.PolicyActionGet) {
 		publicOnly = true
 	}
 	params := appctx.AppContextParams(ctx)

pkg/devtool/tasks/apply_script_task.go

@@ -72,7 +72,11 @@ func (self *ApplyScriptTask) taskFailed(ctx context.Context, sa *models.SScriptA
 			log.Errorf("unable to StartApply script %s to server %s", sa.ScriptId, sa.GuestId)
 		}
 	}
-	self.SetStageFailed(ctx, jsonutils.NewString(err.Error()))
+	var errMsg string
+	if err != nil {
+		errMsg = err.Error()
+	}
+	self.SetStageFailed(ctx, jsonutils.NewString(errMsg))
 }
 
 func (self *ApplyScriptTask) taskSuccess(ctx context.Context, sa *models.SScriptApply, sar *models.SScriptApplyRecord) {
@@ -325,7 +329,7 @@ func (self *ApplyScriptTask) OnAnsiblePlaybookComplete(ctx context.Context, obj
 	sarId, _ := self.Params.GetString("script_apply_record_id")
 	osar, err := models.ScriptApplyRecordManager.FetchById(sarId)
 	if err != nil {
-		log.Errorf("unable to fetch script apply record %s", sarId)
+		log.Errorf("unable to fetch script apply record %s: %v", sarId, err)
 		self.taskSuccess(ctx, sa, nil)
 	}
 	self.taskSuccess(ctx, sa, osar.(*models.SScriptApplyRecord))
@@ -337,13 +341,13 @@ func (self *ApplyScriptTask) OnAnsiblePlaybookCompleteFailed(ctx context.Context
 	forwardId, _ := self.Params.GetString("proxy_forward_id")
 	_, err := cloudproxy.Forwards.Delete(session, forwardId, nil)
 	if err != nil {
-		log.Errorf("unable to delete proxy forward %s", forwardId)
+		log.Errorf("unable to delete proxy forward %s: %v", forwardId, err)
 	}
 	sa := obj.(*models.SScriptApply)
 	sarId, _ := self.Params.GetString("script_apply_record_id")
 	osar, err := models.ScriptApplyRecordManager.FetchById(sarId)
 	if err != nil {
-		log.Errorf("unable to fetch script apply record %s", sarId)
+		log.Errorf("unable to fetch script apply record %s: %v", sarId, err)
 		self.taskSuccess(ctx, sa, nil)
 	}
 	self.taskFailed(ctx, sa, osar.(*models.SScriptApplyRecord), errors.Error(body.String()))

pkg/devtool/utils/influxdb_url.go

@@ -65,6 +65,7 @@ func proxyEndpoints(ctx context.Context, proxyEndpointId string, info sServerInf
 	for _, netId := range info.NetworkIds {
 		filter := jsonutils.NewDict()
 		filter.Set("network_id", jsonutils.NewString(netId))
+		filter.Set("scope", jsonutils.NewString("system"))
 		lr, err := cloudproxy.ProxyEndpoints.List(session, filter)
 		if err != nil {
 			return nil, errors.Wrapf(err, "unable to list proxy endpoint in network %q", netId)
@@ -81,6 +82,7 @@ func proxyEndpoints(ctx context.Context, proxyEndpointId string, info sServerInf
 	}
 	filter := jsonutils.NewDict()
 	filter.Set("vpc_id", jsonutils.NewString(info.VpcId))
+	filter.Set("scope", jsonutils.NewString("system"))
 	lr, err := cloudproxy.ProxyEndpoints.List(session, filter)
 	if err != nil {
 		return nil, errors.Wrapf(err, "unable to list proxy endpoint in vpc %q", info.VpcId)
@@ -138,8 +140,12 @@ func convertInfluxdbUrl(ctx context.Context, pUrl string, endpointId string) (po
 	if err != nil {
 		return 0, nil, errors.Wrap(err, "failed to list forward")
 	}
+	var forwardId string
+	var lastSeen string
 	if len(lr.Data) > 0 {
 		port, _ = lr.Data[0].Int("bind_port")
+		forwardId, _ = lr.Data[0].GetString("id")
+		lastSeen, _ = lr.Data[0].GetString("last_seen")
 	} else {
 		var rUrl *url.URL
 		rUrl, err = url.Parse(pUrl)
@@ -161,13 +167,32 @@ func convertInfluxdbUrl(ctx context.Context, pUrl string, endpointId string) (po
 			err = errors.Wrapf(err, "unable to create forward with create params %s", createP.String())
 			return
 		}
-		forwardId, _ := forward.GetString("id")
+		forwardId, _ = forward.GetString("id")
+		lastSeen, _ = forward.GetString("last_seen")
 		recycle = func() error {
 			_, err := cloudproxy.Forwards.Delete(session, forwardId, nil)
 			return err
 		}
 		port, _ = forward.Int("bind_port")
 	}
+	// wait forward last seen not empty
+	times, waitTime := 0, time.Second
+	var data jsonutils.JSONObject
+	for lastSeen == "" && times < 10 {
+		time.Sleep(waitTime)
+		times += 1
+		waitTime += time.Second * time.Duration(times)
+		data, err = cloudproxy.Forwards.GetSpecific(session, forwardId, "last_seen", nil)
+		if err != nil {
+			err = errors.Wrapf(err, "unable to check last_seen for forward %s", forwardId)
+			return
+		}
+		log.Infof("data of last seen: %s", data)
+		lastSeen, _ = data.GetString("last_seen")
+	}
+	if lastSeen == "" {
+		err = errors.Wrapf(err, "last_seen of forward %s always is empty, something wrong", forwardId)
+	}
 	return
 }
 
@@ -274,7 +299,7 @@ func checkUrl(ctx context.Context, url string, host *ansible_api.AnsibleHost) (b
 		}
 	}()
 	times, waitTimes := 0, time.Second
-	for times < 5 {
+	for times < 10 {
 		time.Sleep(waitTimes)
 		times++
 		waitTimes += time.Second * time.Duration(times)

pkg/mcclient/models/base.go

@@ -77,6 +77,7 @@ type EnabledStatusStandaloneResource struct {
 type VirtualResource struct {
 	StatusStandaloneResource
 
+	DomainId         string
 	ProjectId        string
 	IsSystem         bool
 	PendingDeletedAt time.Time

pkg/mcclient/modulebase/base.go

@@ -112,7 +112,7 @@ func (this *BaseManager) rawBaseUrlRequest(s *mcclient.ClientSession,
 }
 
 type ListResult struct {
-	Data   []jsonutils.JSONObject
+	Data   []jsonutils.JSONObject `json:"data,allowempty"`
 	Total  int
 	Limit  int
 	Offset int

pkg/multicloud/azure/instance.go

@@ -26,6 +26,7 @@ import (
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/osprofile"
 
+	"yunion.io/x/onecloud/pkg/apis"
 	billing_api "yunion.io/x/onecloud/pkg/apis/billing"
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -237,9 +238,12 @@ func (self *SInstance) GetSysTags() map[string]string {
 	if loginKey := self.Properties.OsProfile.AdminPassword; len(loginKey) > 0 {
 		data["login_key"] = loginKey
 	}
-	data["zone_ext_id"] = self.host.zone.GetGlobalId()
-	priceKey := fmt.Sprintf("%s::%s", self.Properties.HardwareProfile.VMSize, self.host.zone.region.Name)
-	data["price_key"] = priceKey
+	for _, res := range self.Resources {
+		if strings.HasSuffix(strings.ToLower(res.Id), "databricksbootstrap") {
+			data[apis.IS_SYSTEM] = "true"
+			break
+		}
+	}
 	return data
 }