fix(notify): 对服务异常添加消息通知

build/notify/root/opt/yunion/share/local-templates/content@cn/SERVICE_ABNORMAL.tmpl

@@ -0,0 +1,8 @@
+{{- $d := .resource_details -}}
+服务: {{ $d.service_name }} 异常。
+方法: {{ $d.method }}
+路径: {{ $d.path }} 
+{{- if $d.body }}
+请求: {{ $d.body -}}
+{{ end }}
+错误: {{ $d.error }}

build/notify/root/opt/yunion/share/local-templates/content@en/SERVICE_ABNORMAL.tmpl

@@ -0,0 +1,8 @@
+{{- $d := .resource_details -}}
+Service: {{ $d.service_name }} abnormal
+Method: {{ $d.method }}
+Path: {{ $d.path }} 
+{{- if $d.body }}
+Body: {{ $d.body -}}
+{{ end }}
+Error: {{ $d.error }}

build/notify/root/opt/yunion/share/local-templates/title@cn/SERVICE_ABNORMAL.tmpl

@@ -0,0 +1,2 @@
+{{- $d := .resource_details -}}
+服务{{ $d.service_name }}异常

build/notify/root/opt/yunion/share/local-templates/title@en/SERVICE_ABNORMAL.tmpl

@@ -0,0 +1,2 @@
+{{- $d := .resource_details -}}
+Server {{ $d.service_name }} abnormal

pkg/apis/notify/const.go

@@ -129,6 +129,7 @@ const (
 	TOPIC_RESOURCE_ACCOUNT_STATUS           = "account"
 	TOPIC_RESOURCE_WORKER                   = "worker"
 	TOPIC_RESOURCE_NET                      = "net"
+	TOPIC_RESOURCE_SERVICE                  = "service"
 
 	SUBSCRIBER_TYPE_ROLE     = "role"
 	SUBSCRIBER_TYPE_ROBOT    = "robot"

pkg/apis/notify/event.go

@@ -58,6 +58,7 @@ var (
 	ActionWorkerBlock        SAction = "woker_block"
 	ActionNetOutOfSync       SAction = "net_out_of_sync"
 	ActionMysqlOutOfSync     SAction = "mysql_out_of_sync"
+	ActionServiceAbnormal    SAction = "service_abnormal"
 
 	ResultFailed  SResult = "failed"
 	ResultSucceed SResult = "succeed"

pkg/cloudcommon/notifyclient/events.go

@@ -58,6 +58,7 @@ var (
 	ActionSyncStatus         = api.ActionSyncStatus
 	ActionNetOutOfSync       = api.ActionNetOutOfSync
 	ActionMysqlOutOfSync     = api.ActionMysqlOutOfSync
+	ActionServiceAbnormal    = api.ActionServiceAbnormal
 
 	ActionPendingDelete = api.ActionPendingDelete
 

pkg/cloudcommon/notifyclient/notify.go

@@ -320,6 +320,29 @@ func EventNotify(ctx context.Context, userCred mcclient.TokenCredential, ep SEve
 	notifyClientWorkerMan.Run(&t, nil, nil)
 }
 
+func EventNotifyServiceAbnormal(ctx context.Context, userCred mcclient.TokenCredential, service, method, path string, body jsonutils.JSONObject, err error) {
+	event := api.Event.WithAction(api.ActionServiceAbnormal).WithResourceType(api.TOPIC_RESOURCE_SERVICE)
+	obj := jsonutils.NewDict()
+	if body != nil {
+		obj.Set("body", jsonutils.NewString(body.PrettyString()))
+	}
+	obj.Set("method", jsonutils.NewString(method))
+	obj.Set("path", jsonutils.NewString(path))
+	obj.Set("error", jsonutils.NewString(err.Error()))
+	obj.Set("service_name", jsonutils.NewString(service))
+	params := api.NotificationManagerEventNotifyInput{
+		ReceiverIds:     []string{userCred.GetUserId()},
+		ResourceDetails: obj,
+		Event:           event.String(),
+		AdvanceDays:     0,
+		Priority:        string(npk.NotifyPriorityNormal),
+	}
+	t := eventTask{
+		params: params,
+	}
+	notifyClientWorkerMan.Run(&t, nil, nil)
+}
+
 func systemEventNotify(ctx context.Context, action api.SAction, resType string, result api.SResult, priority string, obj *jsonutils.JSONDict) {
 	event := api.Event.WithAction(action).WithResourceType(resType).WithResult(result)
 	params := api.NotificationManagerEventNotifyInput{

pkg/cloudid/service/service.go

@@ -15,24 +15,29 @@
 package service
 
 import (
+	"context"
 	"os"
 	"time"
 
 	_ "yunion.io/x/cloudmux/pkg/multicloud/loader"
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	_ "yunion.io/x/sqlchemy/backends"
 
 	api "yunion.io/x/onecloud/pkg/apis/cloudid"
 	"yunion.io/x/onecloud/pkg/cloudcommon"
-	common_app "yunion.io/x/onecloud/pkg/cloudcommon/app"
+	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/cronman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 	"yunion.io/x/onecloud/pkg/cloudid/models"
 	"yunion.io/x/onecloud/pkg/cloudid/options"
 	_ "yunion.io/x/onecloud/pkg/cloudid/policy"
 	"yunion.io/x/onecloud/pkg/cloudid/saml"
 	_ "yunion.io/x/onecloud/pkg/cloudid/tasks"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
 )
 
 func StartService() {
@@ -42,12 +47,17 @@ func StartService() {
 	commonOpts := &opts.CommonOptions
 	common_options.ParseOptions(opts, os.Args, "cloudid.conf", api.SERVICE_TYPE)
 
-	common_app.InitAuth(commonOpts, func() {
+	app_common.InitAuth(commonOpts, func() {
 		log.Infof("Auth complete!!")
 	})
 	common_options.StartOptionManager(opts, opts.ConfigSyncPeriodSeconds, api.SERVICE_TYPE, api.SERVICE_VERSION, options.OnOptionsChange)
 
-	app := common_app.InitApp(baseOpts, false)
+	app := app_common.InitApp(baseOpts, true).
+		OnException(func(method, path string, body jsonutils.JSONObject, err error) {
+			ctx := context.Background()
+			session := auth.GetAdminSession(ctx, commonOpts.Region)
+			notifyclient.EventNotifyServiceAbnormal(ctx, session.GetToken(), consts.GetServiceType(), method, path, body, err)
+		})
 
 	cloudcommon.InitDB(dbOpts)
 
@@ -76,5 +86,5 @@ func StartService() {
 		defer cron.Stop()
 	}
 
-	common_app.ServeForever(app, baseOpts)
+	app_common.ServeForever(app, baseOpts)
 }

pkg/cloudmon/service/service.go

@@ -15,19 +15,24 @@
 package service
 
 import (
+	"context"
 	"os"
 	"time"
 
 	_ "yunion.io/x/cloudmux/pkg/multicloud/loader"
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 
 	"yunion.io/x/onecloud/pkg/apis"
 	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/cronman"
+	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 	"yunion.io/x/onecloud/pkg/cloudmon/misc"
 	"yunion.io/x/onecloud/pkg/cloudmon/options"
 	"yunion.io/x/onecloud/pkg/cloudmon/resources"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
 )
 
 func StartService() {
@@ -59,6 +64,11 @@ func StartService() {
 		go cron.Start()
 	}
 
-	app := app_common.InitApp(baseOpts, true)
+	app := app_common.InitApp(baseOpts, true).
+		OnException(func(method, path string, body jsonutils.JSONObject, err error) {
+			ctx := context.Background()
+			session := auth.GetAdminSession(ctx, commonOpts.Region)
+			notifyclient.EventNotifyServiceAbnormal(ctx, session.GetToken(), consts.GetServiceType(), method, path, body, err)
+		})
 	app_common.ServeForever(app, baseOpts)
 }

pkg/cloudnet/service/service.go

@@ -15,17 +15,22 @@
 package service
 
 import (
+	"context"
 	"os"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	_ "yunion.io/x/sqlchemy/backends"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon"
-	common_app "yunion.io/x/onecloud/pkg/cloudcommon/app"
+	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 	"yunion.io/x/onecloud/pkg/cloudnet/models"
 	"yunion.io/x/onecloud/pkg/cloudnet/options"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
 )
 
 func StartService() {
@@ -33,14 +38,19 @@ func StartService() {
 	common_options.ParseOptions(opts, os.Args, "cloudnet.conf", "cloudnet")
 
 	commonOpts := &opts.CommonOptions
-	common_app.InitAuth(commonOpts, func() {
+	app_common.InitAuth(commonOpts, func() {
 		log.Infof("Auth complete")
 	})
 
 	dbOpts := &opts.DBOptions
 	baseOpts := &opts.BaseOptions
 
-	app := common_app.InitApp(baseOpts, false)
+	app := app_common.InitApp(baseOpts, true).
+		OnException(func(method, path string, body jsonutils.JSONObject, err error) {
+			ctx := context.Background()
+			session := auth.GetAdminSession(ctx, commonOpts.Region)
+			notifyclient.EventNotifyServiceAbnormal(ctx, session.GetToken(), consts.GetServiceType(), method, path, body, err)
+		})
 
 	cloudcommon.InitDB(dbOpts)
 
@@ -49,5 +59,5 @@ func StartService() {
 	db.EnsureAppSyncDB(app, dbOpts, models.InitDB)
 	defer cloudcommon.CloseDB()
 
-	common_app.ServeForever(app, baseOpts)
+	app_common.ServeForever(app, baseOpts)
 }

pkg/cloudproxy/service/service.go

@@ -15,13 +15,19 @@
 package service
 
 import (
+	"context"
+
+	"yunion.io/x/jsonutils"
 	_ "yunion.io/x/sqlchemy/backends"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon"
-	common_app "yunion.io/x/onecloud/pkg/cloudcommon/app"
+	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
 	"yunion.io/x/onecloud/pkg/cloudproxy/models"
 	"yunion.io/x/onecloud/pkg/cloudproxy/options"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
 )
 
 func StartService() {
@@ -31,7 +37,12 @@ func StartService() {
 		baseOpts = &opts.BaseOptions
 	)
 
-	app := common_app.InitApp(baseOpts, false)
+	app := app_common.InitApp(baseOpts, true).
+		OnException(func(method, path string, body jsonutils.JSONObject, err error) {
+			ctx := context.Background()
+			session := auth.GetAdminSession(ctx, baseOpts.Region)
+			notifyclient.EventNotifyServiceAbnormal(ctx, session.GetToken(), consts.GetServiceType(), method, path, body, err)
+		})
 
 	cloudcommon.InitDB(dbOpts)
 
@@ -40,5 +51,5 @@ func StartService() {
 	db.EnsureAppSyncDB(app, dbOpts, models.InitDB)
 	defer cloudcommon.CloseDB()
 
-	common_app.ServeForever(app, baseOpts)
+	app_common.ServeForever(app, baseOpts)
 }

pkg/cloutpost/service/service.go

@@ -15,16 +15,21 @@
 package service
 
 import (
+	"context"
 	"os"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 
 	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/etcd"
 	"yunion.io/x/onecloud/pkg/cloudcommon/etcd/models"
+	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 	"yunion.io/x/onecloud/pkg/cloutpost/options"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
 )
 
 const (
@@ -47,7 +52,12 @@ func StartService() {
 	}
 	defer etcd.CloseDefaultEtcdClient()
 
-	app := app_common.InitApp(baseOpts, false)
+	app := app_common.InitApp(baseOpts, true).
+		OnException(func(method, path string, body jsonutils.JSONObject, err error) {
+			ctx := context.Background()
+			session := auth.GetAdminSession(ctx, baseOpts.Region)
+			notifyclient.EventNotifyServiceAbnormal(ctx, session.GetToken(), consts.GetServiceType(), method, path, body, err)
+		})
 	db.AppDBInit(app)
 	initHandlers(app)
 

pkg/compute/service/service.go

@@ -30,12 +30,14 @@ import (
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/cloudcommon"
-	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
+	common_app "yunion.io/x/onecloud/pkg/cloudcommon/app"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/cronman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/elect"
 	"yunion.io/x/onecloud/pkg/cloudcommon/etcd"
+	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 	_ "yunion.io/x/onecloud/pkg/compute/guestdrivers"
 	_ "yunion.io/x/onecloud/pkg/compute/hostdrivers"
@@ -62,7 +64,7 @@ func StartService() {
 		commonOpts.Port = opts.PortV2
 	}
 
-	app_common.InitAuth(commonOpts, func() {
+	common_app.InitAuth(commonOpts, func() {
 		log.Infof("Auth complete!!")
 	})
 	common_options.StartOptionManager(opts, opts.ConfigSyncPeriodSeconds, api.SERVICE_TYPE, api.SERVICE_VERSION, options.OnOptionsChange)
@@ -88,9 +90,11 @@ func StartService() {
 		log.Errorf("try to init etcd options error: %v", err)
 	}
 
-	app := app_common.InitApp(baseOpts, true).
+	app := common_app.InitApp(baseOpts, true).
 		OnException(func(method, path string, body jsonutils.JSONObject, err error) {
-			// send notify exception
+			ctx := context.Background()
+			session := auth.GetAdminSession(ctx, commonOpts.Region)
+			notifyclient.EventNotifyServiceAbnormal(ctx, session.GetToken(), consts.GetServiceType(), method, path, body, err)
 		})
 
 	cloudcommon.InitDB(dbOpts)
@@ -193,7 +197,7 @@ func StartService() {
 		autoscaling.ASController.Init(options.Options.SASControllerOptions, cron)
 	}
 
-	app_common.ServeForever(app, baseOpts)
+	common_app.ServeForever(app, baseOpts)
 }
 
 func initDefaultEtcdClient(opts *common_options.DBOptions) error {
@@ -228,7 +232,7 @@ func initDefaultEtcdClient(opts *common_options.DBOptions) error {
 }
 
 func initEtcdLockOpts(opts *options.ComputeOptions) error {
-	etcdEndpoint, err := app_common.FetchEtcdServiceInfo()
+	etcdEndpoint, err := common_app.FetchEtcdServiceInfo()
 	if err != nil {
 		if errors.Cause(err) == httperrors.ErrNotFound {
 			return nil

pkg/keystone/service/service.go

@@ -15,14 +15,17 @@
 package service
 
 import (
+	"context"
 	"os"
 	"time"
 
+	"yunion.io/x/jsonutils"
 	_ "yunion.io/x/sqlchemy/backends"
 
 	api "yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/cloudcommon"
-	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
+	common_app "yunion.io/x/onecloud/pkg/cloudcommon/app"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/cronman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
@@ -70,7 +73,12 @@ func StartService() {
 	}
 	*/
 
-	app := app_common.InitApp(&opts.BaseOptions, true)
+	app := common_app.InitApp(&opts.BaseOptions, true).
+		OnException(func(method, path string, body jsonutils.JSONObject, err error) {
+			ctx := context.Background()
+			session := auth.GetAdminSession(ctx, opts.Region)
+			notifyclient.EventNotifyServiceAbnormal(ctx, session.GetToken(), consts.GetServiceType(), method, path, body, err)
+		})
 
 	cloudcommon.InitDB(&opts.DBOptions)
 
@@ -78,7 +86,7 @@ func StartService() {
 
 	db.EnsureAppSyncDB(app, &opts.DBOptions, models.InitDB)
 
-	app_common.InitBaseAuth(&opts.BaseOptions)
+	common_app.InitBaseAuth(&opts.BaseOptions)
 
 	common_options.StartOptionManagerWithSessionDriver(opts, opts.ConfigSyncPeriodSeconds, api.SERVICE_TYPE, "", options.OnOptionsChange, models.NewServiceConfigSession())
 
@@ -105,10 +113,10 @@ func StartService() {
 	}
 
 	go func() {
-		app_common.ServeForeverExtended(app, &opts.BaseOptions, options.Options.AdminPort, nil, false)
+		common_app.ServeForeverExtended(app, &opts.BaseOptions, options.Options.AdminPort, nil, false)
 	}()
 
-	app_common.ServeForeverWithCleanup(app, &opts.BaseOptions, func() {
+	common_app.ServeForeverWithCleanup(app, &opts.BaseOptions, func() {
 		cloudcommon.CloseDB()
 		// cron.Stop()
 	})

pkg/monitor/service/service.go

@@ -21,13 +21,16 @@ import (
 
 	"golang.org/x/sync/errgroup"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	_ "yunion.io/x/sqlchemy/backends"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon"
 	common_app "yunion.io/x/onecloud/pkg/cloudcommon/app"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/cronman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	_ "yunion.io/x/onecloud/pkg/monitor/alerting"
@@ -56,7 +59,12 @@ func StartService() {
 	dbOpts := &opts.DBOptions
 	baseOpts := &opts.BaseOptions
 
-	app := common_app.InitApp(baseOpts, false)
+	app := common_app.InitApp(baseOpts, true).
+		OnException(func(method, path string, body jsonutils.JSONObject, err error) {
+			ctx := context.Background()
+			session := auth.GetAdminSession(ctx, commonOpts.Region)
+			notifyclient.EventNotifyServiceAbnormal(ctx, session.GetToken(), consts.GetServiceType(), method, path, body, err)
+		})
 
 	cloudcommon.InitDB(dbOpts)
 

pkg/notify/models/event_template.go

@@ -17,6 +17,7 @@ package models
 import (
 	"context"
 	"fmt"
+	"html"
 	"html/template"
 	"io/ioutil"
 	"os"
@@ -162,8 +163,8 @@ func (lt *SLocalTemplateManager) FillWithTemplate(ctx context.Context, lang stri
 		}
 	}
 
-	out.Title = title
-	out.Message = content
+	out.Title = html.UnescapeString(title)
+	out.Message = html.UnescapeString(content)
 	return out, nil
 }
 

pkg/notify/models/topic.go

@@ -102,6 +102,7 @@ const (
 	DefaultPasswordExpireDue7Day   = "password expire due 7 day"
 	DefaultNetOutOfSync            = "net out of sync"
 	DefaultMysqlOutOfSync          = "mysql out of sync"
+	DefaultServiceAbnormal         = "service abnormal"
 )
 
 func (sm *STopicManager) InitializeData() error {
@@ -126,6 +127,7 @@ func (sm *STopicManager) InitializeData() error {
 		DefaultPasswordExpireDue7Day,
 		DefaultNetOutOfSync,
 		DefaultMysqlOutOfSync,
+		DefaultServiceAbnormal,
 	)
 	q := sm.Query()
 	topics := make([]STopic, 0, initSNames.Len())
@@ -402,6 +404,15 @@ func (sm *STopicManager) InitializeData() error {
 			t.Type = notify.TOPIC_TYPE_AUTOMATED_PROCESS
 			t.AdvanceDays = 0
 			t.Results = tristate.True
+		case DefaultServiceAbnormal:
+			t.addResources(
+				notify.TOPIC_RESOURCE_SERVICE,
+			)
+			t.addAction(
+				notify.ActionServiceAbnormal,
+			)
+			t.Results = tristate.True
+			t.Type = notify.TOPIC_TYPE_AUTOMATED_PROCESS
 		}
 		if topic == nil {
 			err := sm.TableSpec().Insert(ctx, t)
@@ -641,6 +652,7 @@ func init() {
 			notify.TOPIC_RESOURCE_ACTION_LOG:               37,
 			notify.TOPIC_RESOURCE_ACCOUNT_STATUS:           38,
 			notify.TOPIC_RESOURCE_NET:                      39,
+			notify.TOPIC_RESOURCE_SERVICE:                  40,
 		},
 	)
 	converter.registerAction(
@@ -673,6 +685,7 @@ func init() {
 			notify.ActionPasswordExpireSoon: 25,
 			notify.ActionNetOutOfSync:       26,
 			notify.ActionMysqlOutOfSync:     27,
+			notify.ActionServiceAbnormal:    28,
 		},
 	)
 }

pkg/vpcagent/service/service.go

@@ -21,11 +21,15 @@ import (
 	"sync"
 	"syscall"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/appctx"
 
 	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
+	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/vpcagent/options"
 	"yunion.io/x/onecloud/pkg/vpcagent/worker"
 )
@@ -43,7 +47,12 @@ func StartService() {
 		log.Fatalf("opts validate: %s", err)
 	}
 
-	app := app_common.InitApp(&opts.BaseOptions, false)
+	app := app_common.InitApp(&opts.BaseOptions, true).
+		OnException(func(method, path string, body jsonutils.JSONObject, err error) {
+			ctx := context.Background()
+			session := auth.GetAdminSession(ctx, commonOpts.Region)
+			notifyclient.EventNotifyServiceAbnormal(ctx, session.GetToken(), consts.GetServiceType(), method, path, body, err)
+		})
 
 	w := worker.NewWorker(opts)
 	if w == nil {

pkg/yunionconf/service/service.go

@@ -15,16 +15,21 @@
 package service
 
 import (
+	"context"
 	"os"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	_ "yunion.io/x/sqlchemy/backends"
 
 	api "yunion.io/x/onecloud/pkg/apis/yunionconf"
 	"yunion.io/x/onecloud/pkg/cloudcommon"
 	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/yunionconf/models"
 	"yunion.io/x/onecloud/pkg/yunionconf/options"
 	_ "yunion.io/x/onecloud/pkg/yunionconf/policy"
@@ -43,7 +48,12 @@ func StartService() {
 
 	cloudcommon.InitDB(dbOpts)
 
-	app := app_common.InitApp(baseOpts, true)
+	app := app_common.InitApp(baseOpts, true).
+		OnException(func(method, path string, body jsonutils.JSONObject, err error) {
+			ctx := context.Background()
+			session := auth.GetAdminSession(ctx, baseOpts.Region)
+			notifyclient.EventNotifyServiceAbnormal(ctx, session.GetToken(), consts.GetServiceType(), method, path, body, err)
+		})
 	InitHandlers(app)
 	db.AppDBInit(app)