Files
cloudpods/pkg/monitor/models/monitor_resource.go
2021-11-25 17:42:01 +08:00

578 lines
18 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import (
"context"
"fmt"
"reflect"
"sync"
"time"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/utils"
"yunion.io/x/sqlchemy"
"yunion.io/x/onecloud/pkg/apihelper"
"yunion.io/x/onecloud/pkg/apis/monitor"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/mcclient/auth"
"yunion.io/x/onecloud/pkg/util/rbacutils"
"yunion.io/x/onecloud/pkg/util/stringutils2"
)
var (
MonitorResourceManager *SMonitorResourceManager
)
type IMonitorResourceCache interface {
Get(resId string) (jsonutils.JSONObject, bool)
}
type sMonitorResourceCache struct {
length int
sync.Map
}
func (c *sMonitorResourceCache) set(resId string, obj jsonutils.JSONObject) {
c.Store(resId, obj)
c.length++
}
func (c *sMonitorResourceCache) remove(resId string) {
c.Delete(resId)
c.length--
}
func (c *sMonitorResourceCache) Get(resId string) (jsonutils.JSONObject, bool) {
obj, ok := c.Load(resId)
if !ok {
return nil, false
}
return obj.(jsonutils.JSONObject), true
}
func init() {
MonitorResourceManager = &SMonitorResourceManager{
SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager(
&SMonitorResource{},
"monitor_resource_tbl",
"monitorresource",
"monitorresources",
),
monitorResModelSets: NewModelSets(),
}
MonitorResourceManager.SetVirtualObject(MonitorResourceManager)
RegistryResourceSync(NewGuestResourceSync())
RegistryResourceSync(NewHostResourceSync())
RegistryResourceSync(NewRdsResourceSync())
RegistryResourceSync(NewRedisResourceSync())
RegistryResourceSync(NewOssResourceSync())
RegistryResourceSync(NewAccountResourceSync())
RegistryResourceSync(NewStorageResourceSync())
}
func (manager *SMonitorResourceManager) GetModelSets() *MonitorResModelSets {
return manager.monitorResModelSets
}
type SMonitorResourceManager struct {
db.SVirtualResourceBaseManager
db.SEnabledResourceBaseManager
monitorResModelSets *MonitorResModelSets
}
type SMonitorResource struct {
db.SVirtualResourceBase
db.SEnabledResourceBase
AlertState string `width:"36" charset:"ascii" list:"user" default:"init" update:"user" json:"alert_state"`
ResId string `width:"256" charset:"ascii" index:"true" list:"user" update:"user" json:"res_id"`
ResType string `width:"36" charset:"ascii" list:"user" update:"user" json:"res_type"`
}
func (manager *SMonitorResourceManager) GetMonitorResources(input monitor.MonitorResourceListInput) ([]SMonitorResource, error) {
monitorResources := make([]SMonitorResource, 0)
query := manager.Query()
if input.OnlyResId {
query = query.AppendField(query.Field("id"), query.Field("res_id"))
}
query = manager.FieldListFilter(query, input)
err := db.FetchModelObjects(manager, query, &monitorResources)
if err != nil {
return nil, errors.Wrap(err, "SMonitorResourceManager FetchModelObjects err")
}
return monitorResources, nil
}
type SdeleteRes struct {
resType string
notIn []string
in []string
}
func (manager *SMonitorResourceManager) DeleteMonitorResources(ctx context.Context, userCred mcclient.TokenCredential, input SdeleteRes) error {
monitorResources := make([]SMonitorResource, 0)
errs := make([]error, 0)
query := manager.Query()
if len(input.notIn) != 0 {
query.NotIn("res_id", input.notIn)
}
if len(input.in) != 0 {
query.In("res_id", input.in)
}
if len(input.resType) != 0 {
query.Equals("res_type", input.resType)
}
err := db.FetchModelObjects(manager, query, &monitorResources)
if err != nil {
return errors.Wrap(err, "SMonitorResourceManager FetchModelObjects when DeleteMonitorResources err")
}
for _, res := range monitorResources {
err := (&res).RealDelete(ctx, userCred)
if err != nil {
errs = append(errs, errors.Wrapf(err, "delete monitorResource:%s err", res.GetId()))
}
}
if len(errs) != 0 {
return errors.NewAggregate(errs)
}
return nil
}
func (manager *SMonitorResourceManager) GetMonitorResourceById(id string) (*SMonitorResource, error) {
iModel, err := db.FetchById(manager, id)
if err != nil {
return nil, errors.Wrapf(err, fmt.Sprintf("GetMonitorResourceById:%s err", id))
}
return iModel.(*SMonitorResource), nil
}
func (manager *SMonitorResourceManager) ListItemFilter(
ctx context.Context, q *sqlchemy.SQuery,
userCred mcclient.TokenCredential,
query monitor.MonitorResourceListInput,
) (*sqlchemy.SQuery, error) {
var err error
q, err = manager.SVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query.VirtualResourceListInput)
if err != nil {
return nil, errors.Wrap(err, "SVirtualResourceBaseManager.ListItemFilter")
}
q, err = manager.SEnabledResourceBaseManager.ListItemFilter(ctx, q, userCred, query.EnabledResourceBaseListInput)
if err != nil {
return nil, errors.Wrap(err, "SEnabledResourceBaseManager.ListItemFilter")
}
q = manager.FieldListFilter(q, query)
return q, nil
}
func (manager *SMonitorResourceManager) FieldListFilter(q *sqlchemy.SQuery, query monitor.MonitorResourceListInput) *sqlchemy.SQuery {
if len(query.ResType) != 0 {
q.Equals("res_type", query.ResType)
}
if len(query.ResId) != 0 {
q.In("res_id", query.ResId)
}
if len(query.ResName) != 0 {
q.Contains("name", query.ResName)
}
return q
}
func (man *SMonitorResourceManager) OrderByExtraFields(
ctx context.Context,
q *sqlchemy.SQuery,
userCred mcclient.TokenCredential,
input monitor.SuggestSysAlertListInput,
) (*sqlchemy.SQuery, error) {
var err error
q, err = man.SVirtualResourceBaseManager.OrderByExtraFields(ctx, q, userCred, input.VirtualResourceListInput)
if err != nil {
return nil, errors.Wrap(err, "SVirtualResourceBaseManager.OrderByExtraFields")
}
return q, nil
}
func (man *SMonitorResourceManager) HasName() bool {
return false
}
func (man *SMonitorResourceManager) ValidateCreateData(
ctx context.Context, userCred mcclient.TokenCredential,
ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject,
data monitor.MonitorResourceCreateInput) (monitor.MonitorResourceCreateInput, error) {
//rule 查询到资源信息后没有将资源id进行转换
if len(data.ResId) == 0 {
return data, httperrors.NewInputParameterError("not found res_id %q", data.ResId)
}
if len(data.ResType) == 0 {
return data, httperrors.NewInputParameterError("not found res_type %q", data.ResType)
}
return data, nil
}
func (self *SMonitorResource) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential,
ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
return nil
}
func (man *SMonitorResourceManager) FetchCustomizeColumns(
ctx context.Context,
userCred mcclient.TokenCredential,
query jsonutils.JSONObject,
objs []interface{},
fields stringutils2.SSortedStrings,
isList bool,
) []monitor.MonitorResourceDetails {
rows := make([]monitor.MonitorResourceDetails, len(objs))
virtRows := man.SVirtualResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
for i := range rows {
rows[i] = monitor.MonitorResourceDetails{
VirtualResourceDetails: virtRows[i],
}
rows[i] = objs[i].(*SMonitorResource).getMoreDetails(rows[i])
}
return rows
}
func (self *SMonitorResource) AttachAlert(ctx context.Context, userCred mcclient.TokenCredential, alertId string) error {
iModel, _ := db.NewModelObject(MonitorResourceAlertManager)
input := monitor.MonitorResourceJointCreateInput{
MonitorResourceId: self.ResId,
AlertId: alertId,
AlertState: monitor.MONITOR_RESOURCE_ALERT_STATUS_ATTACH,
}
data := input.JSON(&input)
err := data.Unmarshal(iModel)
if err != nil {
return errors.Wrap(err, "MonitorResourceJointCreateInput unmarshal to joint err")
}
if err := MonitorResourceAlertManager.TableSpec().Insert(ctx, iModel); err != nil {
return errors.Wrap(err, "insert MonitorResourceJoint model err")
}
return nil
}
func (self *SMonitorResource) UpdateAlertState() error {
joints, _ := MonitorResourceAlertManager.GetJoinsByListInput(monitor.MonitorResourceJointListInput{MonitorResourceId: self.ResId})
jointState := monitor.MONITOR_RESOURCE_ALERT_STATUS_ATTACH
if len(joints) == 0 {
jointState = monitor.MONITOR_RESOURCE_ALERT_STATUS_INIT
}
for _, joint := range joints {
if joint.AlertState == monitor.MONITOR_RESOURCE_ALERT_STATUS_ALERTING && time.Now().Sub(joint.
TriggerTime) < time.Minute*30 {
jointState = monitor.MONITOR_RESOURCE_ALERT_STATUS_ALERTING
}
}
_, err := db.Update(self, func() error {
self.AlertState = jointState
return nil
})
if err != nil {
return errors.Wrapf(err, "SMonitorResource:%s UpdateAlertState err", self.Name)
}
return nil
}
func (self *SMonitorResource) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
err := self.DetachJoint(ctx, userCred)
if err != nil {
return err
}
return self.SVirtualResourceBase.Delete(ctx, nil)
}
func (self *SMonitorResource) DetachJoint(ctx context.Context, userCred mcclient.TokenCredential) error {
err := MonitorResourceAlertManager.DetachJoint(ctx, userCred,
monitor.MonitorResourceJointListInput{MonitorResourceId: self.ResId})
if err != nil {
return errors.Wrap(err, "SMonitorResource DetachJoint err")
}
return nil
}
func (self *SMonitorResource) getMoreDetails(out monitor.MonitorResourceDetails) monitor.MonitorResourceDetails {
joints, err := MonitorResourceAlertManager.GetJoinsByListInput(monitor.
MonitorResourceJointListInput{MonitorResourceId: self.ResId})
if err != nil {
log.Errorf("getMoreDetails err:%v", err)
}
_, object := MonitorResourceManager.GetResourceObj(self.ResId)
if object != nil {
object.Unmarshal(&out)
}
out.AttachAlertCount = int64(len(joints))
return out
}
func (manager *SMonitorResourceManager) AllowGetPropertyAlert(ctx context.Context, userCred mcclient.TokenCredential,
query jsonutils.JSONObject) bool {
return true
}
type AlertStatusCount struct {
CountId int64
AlertState string
}
func (manager *SMonitorResourceManager) GetPropertyAlert(ctx context.Context, userCred mcclient.TokenCredential,
data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
scope, _ := data.GetString("scope")
if len(scope) == 0 {
scope = "system"
}
result := jsonutils.NewDict()
for resType, _ := range GetResourceSyncMap() {
query := manager.Query("alert_state")
owner, _ := manager.FetchOwnerId(ctx, data)
if owner == nil {
owner = userCred
}
manager.FilterByOwner(query, owner, rbacutils.TRbacScope(scope))
query.AppendField(sqlchemy.COUNT("count_id", query.Field("id")))
input := monitor.MonitorResourceListInput{ResType: resType}
query = manager.FieldListFilter(query, input)
query.GroupBy(query.Field("alert_state"))
log.Errorf("query:%s", query.String())
rows, err := query.Rows()
if err != nil {
return nil, errors.Wrap(err, "getMonitorResourceAlert query err")
}
defer rows.Close()
total := int64(0)
resTypeDict := jsonutils.NewDict()
for rows.Next() {
row := new(AlertStatusCount)
err := query.Row2Struct(rows, row)
if err != nil {
return nil, errors.Wrap(err, "MonitorResource Row2Struct err")
}
resTypeDict.Add(jsonutils.NewInt(row.CountId), row.AlertState)
total += row.CountId
}
resTypeDict.Add(jsonutils.NewInt(total), "total")
result.Add(resTypeDict, resType)
}
return result, nil
}
func (manager *SMonitorResourceManager) UpdateMonitorResourceAttachJoint(ctx context.Context,
userCred mcclient.TokenCredential, alertRecord *SAlertRecord) error {
//if !utils.IsInStringArray(alertRecord.ResType, []string{monitor.METRIC_RES_TYPE_HOST,
// monitor.METRIC_RES_TYPE_GUEST, monitor.METRIC_RES_TYPE_AGENT}) {
// return nil
//}
resType := alertRecord.ResType
if resType == monitor.METRIC_RES_TYPE_AGENT {
resType = monitor.METRIC_RES_TYPE_GUEST
}
matches, _ := alertRecord.GetEvalData()
errs := make([]error, 0)
matchResourceIds := make([]string, 0)
for _, matche := range matches {
resId := matche.Tags[monitor.MEASUREMENT_TAG_ID[alertRecord.ResType]]
if len(resId) == 0 {
continue
}
matchResourceIds = append(matchResourceIds, resId)
monitorResources, err := manager.GetMonitorResources(monitor.MonitorResourceListInput{ResType: resType, ResId: []string{resId}})
if err != nil {
errs = append(errs, errors.Wrapf(err, "SMonitorResourceManager GetMonitorResources by resId:%s err", resId))
continue
}
for _, res := range monitorResources {
err := res.UpdateAttachJoint(alertRecord, matche)
if err != nil {
errs = append(errs, err)
}
}
}
resourceAlerts, err := MonitorResourceAlertManager.GetJoinsByListInput(monitor.MonitorResourceJointListInput{AlertId: alertRecord.AlertId})
if err != nil {
return errors.Wrapf(err, "get monitor_resource_joint by alertId:%s err", alertRecord.AlertId)
}
deleteJointIds := make([]int64, 0)
for _, joint := range resourceAlerts {
if utils.IsInStringArray(joint.MonitorResourceId, matchResourceIds) {
continue
}
deleteJointIds = append(deleteJointIds, joint.RowId)
}
if len(deleteJointIds) != 0 {
err = MonitorResourceAlertManager.DetachJoint(ctx, userCred, monitor.MonitorResourceJointListInput{JointId: deleteJointIds})
if err != nil {
return errors.Wrapf(err, "DetachJoint by alertId:%s err", alertRecord.AlertId)
}
}
return errors.NewAggregate(errs)
}
func (self *SMonitorResource) UpdateAttachJoint(alertRecord *SAlertRecord, match monitor.EvalMatch) error {
joints, err := MonitorResourceAlertManager.GetJoinsByListInput(monitor.MonitorResourceJointListInput{MonitorResourceId: self.
ResId, AlertId: alertRecord.AlertId})
if err != nil {
return errors.Wrapf(err, "SMonitorResource:%s UpdateAttachJoint err", self.Name)
}
errs := make([]error, 0)
// 报警时发现没有进行关联增加attach
if len(joints) == 0 {
self.AttachAlert(context.Background(), nil, alertRecord.AlertId)
joints, _ = MonitorResourceAlertManager.GetJoinsByListInput(monitor.
MonitorResourceJointListInput{MonitorResourceId: self.
ResId, AlertId: alertRecord.AlertId})
}
for _, joint := range joints {
err := joint.UpdateAlertRecordData(alertRecord, &match)
if err != nil {
errs = append(errs, errors.Wrapf(err, "joint %s:%s %s:%s UpdateAlertRecordData err",
MonitorResourceAlertManager.GetMasterFieldName(), self.ResId,
MonitorResourceAlertManager.GetSlaveFieldName(), alertRecord.AlertId))
}
}
self.UpdateAlertState()
return errors.NewAggregate(errs)
}
func (manager *SMonitorResourceManager) GetResourceObj(id string) (bool, jsonutils.JSONObject) {
for _, set := range manager.GetModelSets().ModelSetList() {
setRv := reflect.ValueOf(set)
for _, kRv := range setRv.MapKeys() {
if id == kRv.String() {
mRv := setRv.MapIndex(kRv)
if mRv.IsValid() {
return true, jsonutils.Marshal(mRv.Interface())
}
}
}
}
return false, nil
}
func (manager *SMonitorResourceManager) GetResourceObjByResType(typ string) (bool, []jsonutils.JSONObject) {
manager.GetModelSets()
for _, set := range manager.GetModelSets().ModelSetList() {
if _, ok := set.(IMonitorResModelSet); !ok {
continue
}
if set.(IMonitorResModelSet).GetResType() != typ {
continue
}
setRv := reflect.ValueOf(set)
objects := make([]jsonutils.JSONObject, 0)
for _, kRv := range setRv.MapKeys() {
mRv := setRv.MapIndex(kRv)
objects = append(objects, jsonutils.Marshal(mRv.Interface()))
}
return true, objects
}
return false, nil
}
func (manager *SMonitorResourceManager) SyncResources(ctx context.Context, mss *MonitorResModelSets) error {
userCred := auth.AdminCredential()
errs := make([]error, 0)
log.Infoln("start sync")
aliveIds := make([]string, 0)
for _, set := range mss.ModelSetList() {
setRv := reflect.ValueOf(set)
needSync, typ := manager.GetSetType(set)
log.Infof("Type: %s,length: %d", typ, len(setRv.MapKeys()))
if !needSync {
log.Infof("Type: %s don't sync", typ)
continue
}
for _, kRv := range setRv.MapKeys() {
mRv := setRv.MapIndex(kRv)
//log.Errorf("resID:%s", kRv.String())
input := monitor.MonitorResourceListInput{
ResId: []string{kRv.String()},
}
res, err := MonitorResourceManager.GetMonitorResources(input)
if err != nil {
return errors.Wrap(err, "GetMonitorResources err")
}
if mRv.IsValid() {
aliveIds = append(aliveIds, kRv.String())
obj := jsonutils.Marshal(mRv.Interface())
if len(res) == 0 {
// no find to create
createData := newMonitorResourceCreateInput(obj, typ)
_, err = db.DoCreate(MonitorResourceManager, ctx, userCred, nil, createData,
userCred)
if err != nil {
name, _ := createData.GetString("name")
errs = append(errs, errors.Wrapf(err, "monitorResource:%s resType:%s DoCreate err", name, typ))
}
continue
}
_, err = db.Update(&res[0], func() error {
obj.(*jsonutils.JSONDict).Remove("id")
(&res[0]).ResType = typ
obj.Unmarshal(&res[0])
return nil
})
if err != nil {
errs = append(errs, errors.Wrapf(err, "monitorResource:%s Update err", res[0].Name))
continue
}
continue
}
if len(res) != 0 {
log.Infof("delete monitor resource,resId: %s,resType: %s", res[0].ResId, res[0].ResType)
err := (&res[0]).RealDelete(ctx, userCred)
if err != nil {
errs = append(errs, errors.Wrapf(err, "delete monitorResource:%s err", res[0].GetId()))
}
}
}
err := manager.DeleteMonitorResources(ctx, userCred, SdeleteRes{notIn: aliveIds, resType: typ})
if err != nil {
return err
}
}
log.Infoln("SMonitorResourceManager SyncResources End")
err := CommonAlertManager.Run(ctx)
if err != nil {
log.Errorf("CommonAlertManager UpdateMonitorResourceJoint err:%v", err)
}
return errors.NewAggregate(errs)
}
func (manager *SMonitorResourceManager) GetSetType(set apihelper.IModelSet) (bool, string) {
if iset, ok := set.(IMonitorResModelSet); ok {
return iset.NeedSync(), iset.GetResType()
}
return false, "NONE"
}
func newMonitorResourceCreateInput(input jsonutils.JSONObject, typ string) jsonutils.JSONObject {
monitorResource := jsonutils.DeepCopy(input).(*jsonutils.JSONDict)
id, _ := monitorResource.GetString("id")
monitorResource.Add(jsonutils.NewString(id), "res_id")
monitorResource.Remove("id")
monitorResource.Add(jsonutils.NewString(typ), "res_type")
return monitorResource
}