From 4036d346565dfd5079df08399c2ffb74e75a297b Mon Sep 17 00:00:00 2001 From: ioito Date: Tue, 22 Oct 2019 11:23:32 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BA=91=E4=B8=8A=E6=93=8D=E4=BD=9C=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/cloudevent/main.go | 23 ++ pkg/apis/cloudevent/cloudevent_const.go | 47 +++ pkg/apis/cloudevent/doc.go | 1 + pkg/cloudevent/models/cloudevents.go | 158 +++++++++ pkg/cloudevent/models/cloudproviders.go | 303 ++++++++++++++++++ pkg/cloudevent/models/doc.go | 1 + pkg/cloudevent/models/initdb.go | 40 +++ pkg/cloudevent/options/doc.go | 1 + pkg/cloudevent/options/options.go | 32 ++ pkg/cloudevent/service/doc.go | 1 + pkg/cloudevent/service/handlers.go | 64 ++++ pkg/cloudevent/service/service.go | 64 ++++ pkg/cloudevent/tasks/cloudevent_sync_task.go | 107 +++++++ pkg/cloudevent/tasks/doc.go | 1 + pkg/cloudprovider/cloudprovider.go | 15 + pkg/cloudprovider/resources.go | 15 + pkg/multicloud/aliyun/aliyun.go | 10 + pkg/multicloud/aliyun/events.go | 177 ++++++++++ pkg/multicloud/aliyun/provider/provider.go | 4 + pkg/multicloud/aliyun/shell/events.go | 43 +++ pkg/multicloud/azure/azure.go | 72 +++-- pkg/multicloud/azure/event.go | 164 ++++++++++ pkg/multicloud/azure/provider/provider.go | 12 + pkg/multicloud/azure/shell/event.go | 37 +++ pkg/multicloud/huawei/client/client.go | 2 + .../huawei/client/modules/manager_resource.go | 1 + .../huawei/client/modules/mod_traces.go | 37 +++ pkg/multicloud/huawei/provider/provider.go | 12 + pkg/multicloud/huawei/shell/events.go | 37 +++ pkg/multicloud/huawei/traces.go | 130 ++++++++ pkg/multicloud/qcloud/events.go | 158 +++++++++ pkg/multicloud/qcloud/provider/provider.go | 12 + pkg/multicloud/qcloud/qcloud.go | 17 + pkg/multicloud/qcloud/region.go | 5 + pkg/multicloud/qcloud/shell/events.go | 40 +++ pkg/multicloud/region_base.go | 5 + 36 files changed, 1821 insertions(+), 27 deletions(-) create mode 100644 cmd/cloudevent/main.go create mode 100644 pkg/apis/cloudevent/cloudevent_const.go create mode 100644 pkg/apis/cloudevent/doc.go create mode 100644 pkg/cloudevent/models/cloudevents.go create mode 100644 pkg/cloudevent/models/cloudproviders.go create mode 100644 pkg/cloudevent/models/doc.go create mode 100644 pkg/cloudevent/models/initdb.go create mode 100644 pkg/cloudevent/options/doc.go create mode 100644 pkg/cloudevent/options/options.go create mode 100644 pkg/cloudevent/service/doc.go create mode 100644 pkg/cloudevent/service/handlers.go create mode 100644 pkg/cloudevent/service/service.go create mode 100644 pkg/cloudevent/tasks/cloudevent_sync_task.go create mode 100644 pkg/cloudevent/tasks/doc.go create mode 100644 pkg/multicloud/aliyun/events.go create mode 100644 pkg/multicloud/aliyun/shell/events.go create mode 100644 pkg/multicloud/azure/event.go create mode 100644 pkg/multicloud/azure/shell/event.go create mode 100644 pkg/multicloud/huawei/client/modules/mod_traces.go create mode 100644 pkg/multicloud/huawei/shell/events.go create mode 100644 pkg/multicloud/huawei/traces.go create mode 100644 pkg/multicloud/qcloud/events.go create mode 100644 pkg/multicloud/qcloud/shell/events.go diff --git a/cmd/cloudevent/main.go b/cmd/cloudevent/main.go new file mode 100644 index 0000000000..62d2ac0bc7 --- /dev/null +++ b/cmd/cloudevent/main.go @@ -0,0 +1,23 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "yunion.io/x/onecloud/pkg/cloudevent/service" +) + +func main() { + service.StartService() +} diff --git a/pkg/apis/cloudevent/cloudevent_const.go b/pkg/apis/cloudevent/cloudevent_const.go new file mode 100644 index 0000000000..aae9f8e95c --- /dev/null +++ b/pkg/apis/cloudevent/cloudevent_const.go @@ -0,0 +1,47 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloudevent + +const ( + CLOUD_EVENT_SERVICE_COMPUTE = "compute" + CLOUD_EVENT_SERVICE_UNKNOWN = "unknown" + + CLOUD_EVENT_RESOURCE_TYPE_SERVER = "server" + CLOUD_EVENT_RESOURCE_TYPE_BUCKET = "bucket" + CLOUD_EVENT_RESOURCE_TYPE_DBINSTANCEACCOUNT = "dbinstanceaccount" + CLOUD_EVENT_RESOURCE_TYPE_DBINSTANCEBACKUP = "dbinstancebackup" + CLOUD_EVENT_RESOURCE_TYPE_DBINSTANCEDATABASE = "dbinstancedatabase" + CLOUD_EVENT_RESOURCE_TYPE_DBINSTANCEPARAMETER = "dbinstanceparameter" + CLOUD_EVENT_RESOURCE_TYPE_DBINSTANCEPRIVILEGE = "dbinstanceprivilege" + CLOUD_EVENT_RESOURCE_TYPE_DBINSTANCE = "dbinstance" + CLOUD_EVENT_RESOURCE_TYPE_DISK = "disk" + CLOUD_EVENT_RESOURCE_TYPE_EIP = "eip" + CLOUD_EVENT_RESOURCE_TYPE_HOST = "host" + CLOUD_EVENT_RESOURCE_TYPE_IMAGE = "image" + CLOUD_EVENT_RESOURCE_TYPE_LOADBALANCERBACKENDGROUP = "loadbalancerbackendgroup" + CLOUD_EVENT_RESOURCE_TYPE_LOADBALANCERBACKEND = "loadbalancerbackend" + CLOUD_EVENT_RESOURCE_TYPE_LOADBALANCERLISTENERRULE = "loadbalancerlistenerrule" + CLOUD_EVENT_RESOURCE_TYPE_LOADBALANCERLISTENER = "loadbalancerlistener" + CLOUD_EVENT_RESOURCE_TYPE_LOADBALANCER = "loadbalancer" + CLOUD_EVENT_RESOURCE_TYPE_NATDENTRY = "natdentry" + CLOUD_EVENT_RESOURCE_TYPE_NATGATEWAY = "natgateway" + CLOUD_EVENT_RESOURCE_TYPE_NATSENTRY = "natsentry" + CLOUD_EVENT_RESOURCE_TYPE_NETWORKINTERFACE = "networkinterface" + CLOUD_EVENT_RESOURCE_TYPE_NETWORK = "network" + CLOUD_EVENT_RESOURCE_TYPE_SECGROUPCACHE = "secgroupcache" + CLOUD_EVENT_RESOURCE_TYPE_SNAPSHOTPOLICY = "snapshotpolicy" + CLOUD_EVENT_RESOURCE_TYPE_SNAPSHOT = "snapshot" + CLOUD_EVENT_RESOURCE_TYPE_VPC = "vpc" +) diff --git a/pkg/apis/cloudevent/doc.go b/pkg/apis/cloudevent/doc.go new file mode 100644 index 0000000000..55353b4c9d --- /dev/null +++ b/pkg/apis/cloudevent/doc.go @@ -0,0 +1 @@ +package cloudevent // import "yunion.io/x/onecloud/pkg/apis/cloudevent" diff --git a/pkg/cloudevent/models/cloudevents.go b/pkg/cloudevent/models/cloudevents.go new file mode 100644 index 0000000000..25fadde2dc --- /dev/null +++ b/pkg/cloudevent/models/cloudevents.go @@ -0,0 +1,158 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudevent/options" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modulebase" +) + +type SCloudeventManager struct { + db.SVirtualResourceBaseManager +} + +var CloudeventManager *SCloudeventManager +var mods map[string]modulebase.Manager + +func init() { + CloudeventManager = &SCloudeventManager{ + SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager( + SCloudevent{}, + "cloudevents_tbl", + "cloudevent", + "cloudevents", + ), + } + CloudeventManager.SetVirtualObject(CloudeventManager) +} + +type SCloudevent struct { + db.SVirtualResourceBase + + Service string `width:"64" charset:"utf8" nullable:"true" list:"user"` + ResourceType string `width:"64" charset:"utf8" nullable:"true" list:"user"` + Action string `width:"64" charset:"utf8" nullable:"true" list:"user"` + RequestId string `width:"128" charset:"utf8" nullable:"true" list:"user"` + Request jsonutils.JSONObject `charset:"utf8" nullable:"true" list:"user"` + Account string `width:"64" charset:"utf8" nullable:"true" list:"user"` + Success bool `nullable:"false" list:"user"` + + CloudproviderId string `width:"64" charset:"utf8" nullable:"true" list:"user"` +} + +func (self *SCloudeventManager) AllowCreateItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return false +} + +func (self *SCloudevent) AllowDeleteItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return false +} + +func (self *SCloudevent) AllowUpdateItem(ctx context.Context, userCred mcclient.TokenCredential) bool { + return false +} + +func (self *SCloudeventManager) fetchMods(ctx context.Context, userCred mcclient.TokenCredential) { + if len(mods) > 0 { + return + } + s := auth.GetAdminSession(ctx, options.Options.Region, "v2") + mods = map[string]modulebase.Manager{} + rms, _ := modulebase.GetRegisterdModules() + for _, _mods := range rms { + for _, _mod := range _mods { + if _, ok := mods[_mod]; !ok { + mod, err := modulebase.GetModule(s, _mod) + if err != nil { + log.Errorf("failed to get mod %s error: %v", _mod, err) + continue + } + mods[mod.GetKeyword()] = mod + } + } + } + return +} + +func (manager *SCloudeventManager) setEventInfo(session *mcclient.ClientSession, mod modulebase.Manager, event *SCloudevent) error { + params := jsonutils.NewDict() + params.Add(jsonutils.NewString(fmt.Sprintf("external_id.equals(%s)", event.Name)), "filter") + result, err := mod.List(session, params) + if err != nil { + return errors.Wrapf(err, "mod.List for %s by externalId: %s", mod.KeyString(), event.Name) + } + if len(result.Data) != 1 { + return errors.Wrapf(err, "found %d %s by externalId: %s", len(result.Data), mod.KeyString(), event.Name) + } + + data := struct { + Name string + TenantId string + }{} + err = result.Data[0].Unmarshal(&data) + if err != nil { + return errors.Wrapf(err, "result.Data[0].Unmarshal %s", result.Data[0]) + } + if len(data.Name) > 0 { + event.Name = event.Name + } + if len(data.TenantId) > 0 { + event.ProjectId = data.TenantId + } + return nil +} + +func (manager *SCloudeventManager) SyncCloudevent(ctx context.Context, userCred mcclient.TokenCredential, cloudprovider *SCloudprovider, iEvents []cloudprovider.ICloudEvent) int { + count := 0 + for _, iEvent := range iEvents { + event := &SCloudevent{ + Service: iEvent.GetService(), + ResourceType: iEvent.GetResourceType(), + Action: iEvent.GetAction(), + Account: iEvent.GetAccount(), + RequestId: iEvent.GetRequestId(), + Request: iEvent.GetRequest(), + Success: iEvent.IsSuccess(), + CloudproviderId: cloudprovider.Id, + } + + event.Name = iEvent.GetName() + event.Status = "ready" + event.ProjectId = userCred.GetProjectId() + event.CreatedAt = iEvent.GetCreatedAt() + event.ProjectId = userCred.GetProjectId() + event.DomainId = userCred.GetDomainId() + + event.SetModelManager(manager, event) + err := manager.TableSpec().Insert(event) + if err != nil { + log.Errorf("failed to insert event: %s for cloudprovider: %s(%s) error: %v", jsonutils.Marshal(event).PrettyString(), cloudprovider.Name, cloudprovider.Id, err) + continue + } + count += 1 + } + return count +} diff --git a/pkg/cloudevent/models/cloudproviders.go b/pkg/cloudevent/models/cloudproviders.go new file mode 100644 index 0000000000..0219ee8098 --- /dev/null +++ b/pkg/cloudevent/models/cloudproviders.go @@ -0,0 +1,303 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/compare" + "yunion.io/x/pkg/util/timeutils" + "yunion.io/x/pkg/utils" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/cloudevent/options" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/s3gateway/session" +) + +type SCloudproviderManager struct { + db.SEnabledStatusStandaloneResourceBaseManager +} + +var CloudproviderManager *SCloudproviderManager + +func init() { + CloudproviderManager = &SCloudproviderManager{ + SEnabledStatusStandaloneResourceBaseManager: db.NewEnabledStatusStandaloneResourceBaseManager( + SCloudprovider{}, + "cloudproviders_tbl", + "cloudprovider", + "cloudproviders", + ), + } + CloudproviderManager.SetVirtualObject(CloudproviderManager) +} + +type SCloudprovider struct { + db.SEnabledStatusStandaloneResourceBase + + HealthStatus string + SyncStatus string + LastSync time.Time + LastSyncEndAt time.Time + + AccessUrl string + Account string + Secret string + + Provider string +} + +func (manager *SCloudproviderManager) GetRegionCloudproviders(ctx context.Context, userCred mcclient.TokenCredential) ([]SCloudprovider, error) { + s := session.GetSession(ctx, userCred) + params := jsonutils.NewDict() + params.Add(jsonutils.NewString("public"), "cloud_env") + result, err := modules.Cloudproviders.List(s, params) + if err != nil { + return nil, errors.Wrap(err, "modules.Cloudproviders.List") + } + providers := []SCloudprovider{} + for _, _provider := range result.Data { + provider := SCloudprovider{} + provider.SetModelManager(manager, &provider) + err = _provider.Unmarshal(&provider) + if err != nil { + return nil, errors.Wrap(err, "_provider.Unmarshal") + } + providers = append(providers, provider) + } + return providers, nil +} + +func (manager *SCloudproviderManager) GetLocalCloudproviders() ([]SCloudprovider, error) { + dbProviders := []SCloudprovider{} + q := manager.Query() + err := db.FetchModelObjects(manager, q, &dbProviders) + if err != nil { + return nil, err + } + return dbProviders, nil +} + +func (manager *SCloudproviderManager) SyncCloudproviders(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) { + providers, err := manager.GetRegionCloudproviders(ctx, userCred) + if err != nil { + log.Errorf("failed to get region cloudproviders: %v", err) + return + } + + dbProviders, err := manager.GetLocalCloudproviders() + if err != nil { + log.Errorf("failed to get local cloudproviders: %v", err) + return + } + + removed := make([]SCloudprovider, 0) + commondb := make([]SCloudprovider, 0) + commonext := make([]SCloudprovider, 0) + added := make([]SCloudprovider, 0) + + err = compare.CompareSets(dbProviders, providers, &removed, &commondb, &commonext, &added) + if err != nil { + log.Errorf("compare.CompareSets: %v", err) + return + } + + for i := 0; i < len(removed); i++ { + err = removed[i].Delete(ctx, userCred) + if err != nil { + log.Errorf("failed to remove cloudprovider %s(%s) error: %v", removed[i].Name, removed[i].Id, err) + } + } + + for i := 0; i < len(commondb); i++ { + err = commondb[i].syncWithRegionProvider(ctx, userCred, commonext[i]) + if err != nil { + log.Errorf("failed to sync cloudprovider %s(%s) error: %v", commondb[i].Name, commondb[i].Id, err) + } + } + + for i := 0; i < len(added); i++ { + err = manager.newFromRegionProvider(ctx, userCred, added[i]) + if err != nil { + log.Errorf("failed to add cloudprovider %s(%s) error: %v", added[i].Name, added[i].Id, err) + } + } +} + +func (provider *SCloudprovider) syncWithRegionProvider(ctx context.Context, userCred mcclient.TokenCredential, cloudprovider SCloudprovider) error { + _, err := db.Update(provider, func() error { + provider.Status = cloudprovider.Status + provider.Secret = cloudprovider.Secret + provider.Enabled = cloudprovider.Enabled + return nil + }) + return err +} + +func (self *SCloudprovider) MarkSyncing(userCred mcclient.TokenCredential) error { + _, err := db.Update(self, func() error { + self.SyncStatus = api.CLOUD_PROVIDER_SYNC_STATUS_SYNCING + self.LastSync = timeutils.UtcNow() + self.LastSyncEndAt = time.Time{} + return nil + }) + if err != nil { + log.Errorf("Failed to MarkSyncing error: %v", err) + return err + } + return nil +} + +func (self *SCloudprovider) MarkEndSync(userCred mcclient.TokenCredential) error { + _, err := db.Update(self, func() error { + self.SyncStatus = api.CLOUD_PROVIDER_SYNC_STATUS_IDLE + self.LastSyncEndAt = timeutils.UtcNow() + return nil + }) + if err != nil { + log.Errorf("Failed to markEndSync error: %v", err) + return err + } + return nil +} + +func (manager *SCloudproviderManager) newFromRegionProvider(ctx context.Context, userCred mcclient.TokenCredential, cloudprovider SCloudprovider) error { + cloudprovider.SyncStatus = api.CLOUD_PROVIDER_SYNC_STATUS_IDLE + return manager.TableSpec().Insert(&cloudprovider) +} + +func (manager *SCloudproviderManager) SyncCloudeventTask(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) { + cloudproviders := []SCloudprovider{} + q := manager.Query().IsTrue("enabled").Equals("status", api.CLOUD_PROVIDER_CONNECTED).Equals("sync_status", api.CLOUD_PROVIDER_SYNC_STATUS_IDLE) + err := db.FetchModelObjects(manager, q, &cloudproviders) + if err != nil { + log.Errorf("failed to fetch cloudproviders") + return + } + for _, provider := range cloudproviders { + err = provider.StartCloudeventSyncTask(ctx, userCred) + if err != nil { + log.Errorf("Failed start cloudevent sync task error: %v", err) + } + } + return +} + +func (provider *SCloudprovider) StartCloudeventSyncTask(ctx context.Context, userCred mcclient.TokenCredential) error { + params := jsonutils.NewDict() + provider.MarkSyncing(userCred) + task, err := taskman.TaskManager.NewTask(ctx, "CloudeventSyncTask", provider, userCred, params, "", "", nil) + if err != nil { + return errors.Wrap(err, "NewTask") + } + task.ScheduleRun(nil) + return nil +} + +func (self *SCloudprovider) GetNextTimeRange() (time.Time, time.Time, error) { + start, end := time.Time{}, time.Time{} + factory, err := self.GetProviderFactory() + if err != nil { + return start, end, errors.Wrap(err, "self.GetProviderFactory") + } + q := CloudeventManager.Query().Equals("cloudprovider_id", self.Id).Desc("created_at") + count, err := q.CountWithError() + if err != nil { + return start, end, errors.Wrap(err, "q.CountWithError") + } + if count == 0 { + start = time.Now().AddDate(0, 0, -1*factory.GetMaxCloudEventKeepDays()) + } else { + provider := SCloudprovider{} + err = q.First(&provider) + if err != nil { + return start, end, errors.Wrap(err, "q.First") + } + start = provider.CreatedAt + if start.Before(time.Now().AddDate(0, 0, factory.GetMaxCloudEventKeepDays()*-1)) { + start = time.Now().AddDate(0, 0, factory.GetMaxCloudEventKeepDays()*-1) + } + } + if options.Options.OneSyncForHours > factory.GetMaxCloudEventSyncDays()*24 { + end = start.Add(time.Duration(factory.GetMaxCloudEventSyncDays()*24) * time.Hour) + } else { + end = start.Add(time.Duration(options.Options.OneSyncForHours) * time.Hour) + } + start = start.Add(time.Second) + if end.After(time.Now()) { + end = time.Now() + } + return start, end, nil +} + +func (provider *SCloudprovider) getPassword() (string, error) { + return utils.DescryptAESBase64(provider.Id, provider.Secret) +} + +func (provider *SCloudprovider) getAccessUrl() string { + return provider.AccessUrl +} + +func (provider *SCloudprovider) GetProviderFactory() (cloudprovider.ICloudProviderFactory, error) { + return cloudprovider.GetProviderFactory(provider.Provider) +} + +func (provider SCloudprovider) GetGlobalId() string { + return provider.Id +} + +func (provider SCloudprovider) GetExternalId() string { + return provider.Id +} + +func (provider *SCloudprovider) GetProvider() (cloudprovider.ICloudProvider, error) { + if !provider.Enabled { + return nil, errors.Error("Cloud provider is not enabled") + } + accessUrl := provider.getAccessUrl() + passwd, err := provider.getPassword() + if err != nil { + return nil, err + } + return cloudprovider.GetProvider(provider.Id, provider.Name, accessUrl, provider.Account, passwd, provider.Provider) +} + +func (manager *SCloudproviderManager) InitializeData() error { + providers := []SCloudprovider{} + q := manager.Query().NotEquals("sync_status", api.CLOUD_PROVIDER_SYNC_STATUS_IDLE) + err := db.FetchModelObjects(manager, q, &providers) + if err != nil { + return err + } + for i := range providers { + _, err = db.Update(&providers[i], func() error { + providers[i].SyncStatus = api.CLOUD_PROVIDER_SYNC_STATUS_IDLE + return nil + }) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/cloudevent/models/doc.go b/pkg/cloudevent/models/doc.go new file mode 100644 index 0000000000..3f5a8f7a44 --- /dev/null +++ b/pkg/cloudevent/models/doc.go @@ -0,0 +1 @@ +package models // import "yunion.io/x/onecloud/pkg/cloudevent/models" diff --git a/pkg/cloudevent/models/initdb.go b/pkg/cloudevent/models/initdb.go new file mode 100644 index 0000000000..5eda91b02b --- /dev/null +++ b/pkg/cloudevent/models/initdb.go @@ -0,0 +1,40 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" +) + +func InitDB() error { + for _, manager := range []db.IModelManager{ + /* + * Important!!! + * initialization order matters, do not change the order + */ + db.TenantCacheManager, + + CloudproviderManager, + } { + err := manager.InitializeData() + if err != nil { + log.Errorf("Manager %s initializeData fail %s", manager.Keyword(), err) + // return err skip error table + } + } + return nil +} diff --git a/pkg/cloudevent/options/doc.go b/pkg/cloudevent/options/doc.go new file mode 100644 index 0000000000..c3aebfcb46 --- /dev/null +++ b/pkg/cloudevent/options/doc.go @@ -0,0 +1 @@ +package options // import "yunion.io/x/onecloud/pkg/cloudevent/options" diff --git a/pkg/cloudevent/options/options.go b/pkg/cloudevent/options/options.go new file mode 100644 index 0000000000..f19bbad356 --- /dev/null +++ b/pkg/cloudevent/options/options.go @@ -0,0 +1,32 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package options + +import common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" + +type CloudeventOptions struct { + common_options.CommonOptions + common_options.DBOptions + + CloudproviderSyncIntervalMinutes int `help:"frequency to sync region cloudprovider task" default:"15"` + CloudeventSyncIntervalHours int `help:"frequency to sync cloud event task" default:"1"` + + SyncWithReadEvent bool `help:"sync read operation events" default:"false"` + OneSyncForHours int `help:"Onece sync for hours" default:"1"` +} + +var ( + Options CloudeventOptions +) diff --git a/pkg/cloudevent/service/doc.go b/pkg/cloudevent/service/doc.go new file mode 100644 index 0000000000..afa75a4c8f --- /dev/null +++ b/pkg/cloudevent/service/doc.go @@ -0,0 +1 @@ +package service // import "yunion.io/x/onecloud/pkg/cloudevent/service" diff --git a/pkg/cloudevent/service/handlers.go b/pkg/cloudevent/service/handlers.go new file mode 100644 index 0000000000..30befdd12f --- /dev/null +++ b/pkg/cloudevent/service/handlers.go @@ -0,0 +1,64 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service + +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import ( + "yunion.io/x/onecloud/pkg/appsrv" + "yunion.io/x/onecloud/pkg/appsrv/dispatcher" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/cloudevent/models" +) + +func InitHandlers(app *appsrv.Application) { + db.InitAllManagers() + + taskman.AddTaskHandler("v1", app) + + for _, manager := range []db.IModelManager{ + taskman.TaskManager, + taskman.SubTaskManager, + taskman.TaskObjectManager, + db.UserCacheManager, + db.TenantCacheManager, + db.UserCacheManager, + models.CloudproviderManager, + } { + db.RegisterModelManager(manager) + } + + for _, manager := range []db.IModelManager{ + db.OpsLog, + models.CloudeventManager, + } { + db.RegisterModelManager(manager) + handler := db.NewModelHandler(manager) + dispatcher.AddModelDispatcher("", app, handler) + } +} diff --git a/pkg/cloudevent/service/service.go b/pkg/cloudevent/service/service.go new file mode 100644 index 0000000000..8691f7d7e3 --- /dev/null +++ b/pkg/cloudevent/service/service.go @@ -0,0 +1,64 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service + +import ( + "os" + "time" + + _ "github.com/go-sql-driver/mysql" + + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudcommon" + common_app "yunion.io/x/onecloud/pkg/cloudcommon/app" + "yunion.io/x/onecloud/pkg/cloudcommon/cronman" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" + "yunion.io/x/onecloud/pkg/cloudevent/models" + "yunion.io/x/onecloud/pkg/cloudevent/options" + _ "yunion.io/x/onecloud/pkg/cloudevent/tasks" + _ "yunion.io/x/onecloud/pkg/mcclient/modules" + _ "yunion.io/x/onecloud/pkg/multicloud/loader" +) + +func StartService() { + opts := &options.Options + common_options.ParseOptions(opts, os.Args, "yunionevent.conf", "yunionevent") + + commonOpts := &opts.CommonOptions + common_app.InitAuth(commonOpts, func() { + log.Infof("Auth complete") + }) + + dbOpts := &opts.DBOptions + baseOpts := &opts.BaseOptions + + app := common_app.InitApp(baseOpts, false) + InitHandlers(app) + + db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB) + defer cloudcommon.CloseDB() + + if !opts.IsSlaveNode { + cron := cronman.InitCronJobManager(true, options.Options.CronJobWorkerCount) + cron.AddJobAtIntervalsWithStartRun("SyncCloudprovider", time.Duration(opts.CloudproviderSyncIntervalMinutes)*time.Minute, models.CloudproviderManager.SyncCloudproviders, true) + cron.AddJobAtIntervalsWithStartRun("CloudeventSyncTask", time.Duration(opts.CloudeventSyncIntervalHours)*time.Hour, models.CloudproviderManager.SyncCloudeventTask, true) + cron.Start() + defer cron.Stop() + } + + common_app.ServeForever(app, baseOpts) +} diff --git a/pkg/cloudevent/tasks/cloudevent_sync_task.go b/pkg/cloudevent/tasks/cloudevent_sync_task.go new file mode 100644 index 0000000000..45fcb758e0 --- /dev/null +++ b/pkg/cloudevent/tasks/cloudevent_sync_task.go @@ -0,0 +1,107 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tasks + +import ( + "context" + "time" + + "github.com/pkg/errors" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/cloudevent/models" + "yunion.io/x/onecloud/pkg/cloudevent/options" + "yunion.io/x/onecloud/pkg/cloudprovider" +) + +type CloudeventSyncTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(CloudeventSyncTask{}) +} + +func (self *CloudeventSyncTask) taskFailed(ctx context.Context, cloudprovider *models.SCloudprovider, err error) { + cloudprovider.MarkEndSync(self.UserCred) + self.SetStageFailed(ctx, err.Error()) +} + +func (self *CloudeventSyncTask) taskComplete(ctx context.Context, cloudprovider *models.SCloudprovider) { + cloudprovider.MarkEndSync(self.UserCred) + self.SetStageComplete(ctx, nil) +} + +func (self *CloudeventSyncTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + provider := obj.(*models.SCloudprovider) + + factory, err := provider.GetProviderFactory() + if err != nil { + self.taskFailed(ctx, provider, errors.Wrap(err, "cloudprovider.GetProviderFactory")) + return + } + + iProvider, err := provider.GetProvider() + if err != nil { + self.taskFailed(ctx, provider, errors.Wrap(err, "cloudprovider.GetProvider")) + return + } + + start, end, err := provider.GetNextTimeRange() + if err != nil { + self.taskFailed(ctx, provider, errors.Wrap(err, "provider.GetNextTimeRange")) + return + } + + //小于1小时的暂时不同步 + duration := end.Sub(start) + time.Second + if duration < time.Hour { + self.taskComplete(ctx, provider) + return + } + + count := 0 + for { + events := []cloudprovider.ICloudEvent{} + regions := iProvider.GetIRegions() + for i := range regions { + if factory.IsCloudeventRegional() || i == 0 { + _events, err := regions[i].GetICloudEvents(start, end, options.Options.SyncWithReadEvent) + if err != nil { + if err == cloudprovider.ErrNotSupported { + continue + } + self.taskFailed(ctx, provider, errors.Wrapf(err, "regions[%d].GetICloudEvents", i)) + return + } + events = append(events, _events...) + } + } + _count := models.CloudeventManager.SyncCloudevent(ctx, self.UserCred, provider, events) + log.Infof("Sync %d events for %s(%s) from %s(%d) hours", _count, provider.Name, provider.Id, start.Format("2006-01-02T15:04:05Z"), duration/time.Hour) + count += _count + + if time.Now().Sub(end) < duration { + break + } + start = start.Add(duration) + end = end.Add(duration) + } + self.taskComplete(ctx, provider) +} diff --git a/pkg/cloudevent/tasks/doc.go b/pkg/cloudevent/tasks/doc.go new file mode 100644 index 0000000000..fb076b68cc --- /dev/null +++ b/pkg/cloudevent/tasks/doc.go @@ -0,0 +1 @@ +package tasks // import "yunion.io/x/onecloud/pkg/cloudevent/tasks" diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index fab3bf0c9c..05be311057 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -55,6 +55,9 @@ type ICloudProviderFactory interface { NeedSyncSkuFromCloud() bool IsSupportObjectStorage() bool + IsCloudeventRegional() bool + GetMaxCloudEventSyncDays() int + GetMaxCloudEventKeepDays() int } type ICloudProvider interface { @@ -219,6 +222,18 @@ func (factory *baseProviderFactory) IsOnPremise() bool { return false } +func (factory *baseProviderFactory) IsCloudeventRegional() bool { + return false +} + +func (factory *baseProviderFactory) GetMaxCloudEventSyncDays() int { + return 7 +} + +func (factory *baseProviderFactory) GetMaxCloudEventKeepDays() int { + return 7 +} + type SPremiseBaseProviderFactory struct { baseProviderFactory } diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index e66bc8c14a..ecc822f182 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -132,6 +132,8 @@ type ICloudRegion interface { CreateIElasticcaches(ec *SCloudElasticCacheInput) (ICloudElasticcache, error) GetProvider() string + + GetICloudEvents(start time.Time, end time.Time, withReadEvent bool) ([]ICloudEvent, error) //获取公有云操作日志接口 } type ICloudZone interface { @@ -913,3 +915,16 @@ type ICloudElasticcacheParameter interface { GetModifiable() bool GetForceRestart() bool } + +type ICloudEvent interface { + GetName() string + GetService() string + GetAction() string + GetResourceType() string + GetRequestId() string + GetRequest() jsonutils.JSONObject + GetAccount() string + IsSuccess() bool + + GetCreatedAt() time.Time +} diff --git a/pkg/multicloud/aliyun/aliyun.go b/pkg/multicloud/aliyun/aliyun.go index f510240f27..72078cbceb 100644 --- a/pkg/multicloud/aliyun/aliyun.go +++ b/pkg/multicloud/aliyun/aliyun.go @@ -44,6 +44,8 @@ const ( ALIYUN_API_VERSION_LB = "2014-05-15" ALIYUN_API_VERSION_KVS = "2015-01-01" + ALIYUN_API_VERSION_TRIAL = "2017-12-04" + ALIYUN_BSS_API_VERSION = "2017-12-14" ALIYUN_RAM_API_VERSION = "2015-05-01" @@ -178,6 +180,14 @@ func (self *SAliyunClient) ecsRequest(apiName string, params map[string]string) return jsonRequest(cli, "ecs.aliyuncs.com", ALIYUN_API_VERSION, apiName, params, self.Debug) } +func (self *SAliyunClient) trialRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { + cli, err := self.getDefaultClient() + if err != nil { + return nil, err + } + return jsonRequest(cli, "actiontrail.cn-hangzhou.aliyuncs.com", ALIYUN_API_VERSION_TRIAL, apiName, params, self.Debug) +} + func (self *SAliyunClient) fetchRegions() error { body, err := self.ecsRequest("DescribeRegions", map[string]string{"AcceptLanguage": "zh-CN"}) if err != nil { diff --git a/pkg/multicloud/aliyun/events.go b/pkg/multicloud/aliyun/events.go new file mode 100644 index 0000000000..1c910aed85 --- /dev/null +++ b/pkg/multicloud/aliyun/events.go @@ -0,0 +1,177 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aliyun + +import ( + "time" + + "github.com/pkg/errors" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/apis/cloudevent" + "yunion.io/x/onecloud/pkg/cloudprovider" +) + +type SAttributes struct { + CreationDate time.Time + MfaAuthenticated bool +} + +type SSessionContext struct { + Attributes SAttributes +} + +type SUserIdentity struct { + AccessKeyId string + AccountId string + PrincipalId string + SessionContext SSessionContext + Type string + UserName string +} + +type SEvent struct { + region *SRegion + AdditionalEventData map[string]string + ApiVersion string + EventId string + EventName string + EventSource string + EventTime time.Time + EventType string + EventVersion string + RequestId string + RequestParameters map[string]string + ServiceName string + SourceIpAddress string + UserAgent string + UserIdentity SUserIdentity + ResponseElements map[string]string +} + +func (event *SEvent) GetCreatedAt() time.Time { + return event.EventTime +} + +func (event *SEvent) GetName() string { + return event.EventName +} + +func (event *SEvent) GetAction() string { + return event.ServiceName +} + +func (event *SEvent) GetResourceType() string { + return "" +} + +func (event *SEvent) GetRequestId() string { + return event.RequestId +} + +func (event *SEvent) GetRequest() jsonutils.JSONObject { + return jsonutils.Marshal(event) +} + +func (event *SEvent) GetAccount() string { + if account, ok := event.AdditionalEventData["loginAccount"]; ok { + return account + } + if len(event.UserIdentity.AccessKeyId) > 0 { + return event.UserIdentity.AccessKeyId + } + return event.UserIdentity.UserName +} + +func (event *SEvent) GetService() string { + switch event.ServiceName { + case "Ecs": + return cloudevent.CLOUD_EVENT_SERVICE_COMPUTE + default: + return cloudevent.CLOUD_EVENT_SERVICE_UNKNOWN + } +} + +func (event *SEvent) IsSuccess() bool { + return true +} + +func (region *SRegion) GetICloudEvents(start time.Time, end time.Time, withReadEvent bool) ([]cloudprovider.ICloudEvent, error) { + var ( + events []SEvent + err error + token string + _events []SEvent + iEvents []cloudprovider.ICloudEvent + eventRW string + ) + + eventRW = "Write" + if withReadEvent { + eventRW = "All" + } + + for { + _events, token, err = region.GetEvents(start, end, token, eventRW, "") + if err != nil { + return nil, errors.Wrap(err, "region.GetEvents") + } + events = append(events, _events...) + if len(token) == 0 || len(_events) == 0 { + break + } + } + + for i := range events { + //if withReadEvent || !strings.HasPrefix(events[i].EventName, "Query") { + iEvents = append(iEvents, &events[i]) + //} + } + return iEvents, nil +} + +func (region *SRegion) GetEvents(start time.Time, end time.Time, token string, eventRW string, requestId string) ([]SEvent, string, error) { + params := map[string]string{ + "RegionId": region.RegionId, + } + if !start.IsZero() { + params["StartTime"] = start.Format("2006-01-02T15:04:05Z") + } + if !end.IsZero() { + params["EndTime"] = end.Format("2006-01-02T15:04:05Z") + } + if len(eventRW) > 0 { + params["EventRW"] = eventRW + } + if len(token) > 0 { + params["NextToken"] = token + } + if len(requestId) > 0 { + params["Request"] = requestId + } + resp, err := region.client.trialRequest("LookupEvents", params) + if err != nil { + return nil, "", err + } + + events := []SEvent{} + err = resp.Unmarshal(&events, "Events") + if err != nil { + return nil, "", err + } + nextToken, _ := resp.GetString("NextToken") + return events, nextToken, nil +} diff --git a/pkg/multicloud/aliyun/provider/provider.go b/pkg/multicloud/aliyun/provider/provider.go index 39142da0b6..e64f90e20a 100644 --- a/pkg/multicloud/aliyun/provider/provider.go +++ b/pkg/multicloud/aliyun/provider/provider.go @@ -38,6 +38,10 @@ func (self *SAliyunProviderFactory) GetName() string { return aliyun.CLOUD_PROVIDER_ALIYUN_CN } +func (self *SAliyunProviderFactory) IsCloudeventRegional() bool { + return true +} + func (self *SAliyunProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCreateInput) error { if len(input.AccessKeyId) == 0 { return httperrors.NewMissingParameterError("access_key_id") diff --git a/pkg/multicloud/aliyun/shell/events.go b/pkg/multicloud/aliyun/shell/events.go new file mode 100644 index 0000000000..689cb19e62 --- /dev/null +++ b/pkg/multicloud/aliyun/shell/events.go @@ -0,0 +1,43 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "fmt" + "time" + + "yunion.io/x/onecloud/pkg/multicloud/aliyun" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type EventListOptions struct { + Start time.Time + End time.Time + Token string + EventRW string `choices:"Read|Write|All"` + RequestId string + } + shellutils.R(&EventListOptions{}, "event-list", "List event", func(cli *aliyun.SRegion, args *EventListOptions) error { + events, token, err := cli.GetEvents(args.Start, args.End, args.Token, args.EventRW, args.RequestId) + if err != nil { + return err + } + fmt.Printf("token: %s", token) + printList(events, 0, 0, 0, []string{}) + return nil + }) + +} diff --git a/pkg/multicloud/azure/azure.go b/pkg/multicloud/azure/azure.go index bb696d1853..34c6c10247 100644 --- a/pkg/multicloud/azure/azure.go +++ b/pkg/multicloud/azure/azure.go @@ -66,27 +66,28 @@ type SAzureClient struct { var DEFAULT_API_VERSION = map[string]string{ "vmSizes": "2018-06-01", //2015-05-01-preview,2015-06-15,2016-03-30,2016-04-30-preview,2016-08-30,2017-03-30,2017-12-01,2018-04-01,2018-06-01,2018-10-01 - "Microsoft.Compute/virtualMachineScaleSets": "2017-12-01", - "Microsoft.Compute/virtualMachines": "2018-04-01", - "Microsoft.ClassicCompute/virtualMachines": "2017-04-01", - "Microsoft.Compute/operations": "2018-10-01", - "Microsoft.ClassicCompute/operations": "2017-04-01", - "Microsoft.Network/virtualNetworks": "2018-08-01", - "Microsoft.ClassicNetwork/virtualNetworks": "2017-11-15", //avaliable 2014-01-01,2014-06-01,2015-06-01,2015-12-01,2016-04-01,2016-11-01,2017-11-15 - "Microsoft.Compute/disks": "2018-06-01", //avaliable 2016-04-30-preview,2017-03-30,2018-04-01,2018-06-01 - "Microsoft.Storage/storageAccounts": "2016-12-01", //2018-03-01-preview,2018-02-01,2017-10-01,2017-06-01,2016-12-01,2016-05-01,2016-01-01,2015-06-15,2015-05-01-preview - "Microsoft.ClassicStorage/storageAccounts": "2016-04-01", //2014-01-01,2014-04-01,2014-04-01-beta,2014-06-01,2015-06-01,2015-12-01,2016-04-01,2016-11-01 - "Microsoft.Compute/snapshots": "2018-06-01", //2016-04-30-preview,2017-03-30,2018-04-01,2018-06-01 - "Microsoft.Compute/images": "2018-10-01", //2016-04-30-preview,2016-08-30,2017-03-30,2017-12-01,2018-04-01,2018-06-01,2018-10-01 - "Microsoft.Storage": "2016-12-01", //2018-03-01-preview,2018-02-01,2017-10-01,2017-06-01,2016-12-01,2016-05-01,2016-01-01,2015-06-15,2015-05-01-preview - "Microsoft.Network/publicIPAddresses": "2018-06-01", //2014-12-01-preview, 2015-05-01-preview, 2015-06-15, 2016-03-30, 2016-06-01, 2016-07-01, 2016-08-01, 2016-09-01, 2016-10-01, 2016-11-01, 2016-12-01, 2017-03-01, 2017-04-01, 2017-06-01, 2017-08-01, 2017-09-01, 2017-10-01, 2017-11-01, 2018-01-01, 2018-02-01, 2018-03-01, 2018-04-01, 2018-05-01, 2018-06-01, 2018-07-01, 2018-08-01 - "Microsoft.Network/networkSecurityGroups": "2018-06-01", - "Microsoft.Network/networkInterfaces": "2018-06-01", //2014-12-01-preview, 2015-05-01-preview, 2015-06-15, 2016-03-30, 2016-06-01, 2016-07-01, 2016-08-01, 2016-09-01, 2016-10-01, 2016-11-01, 2016-12-01, 2017-03-01, 2017-04-01, 2017-06-01, 2017-08-01, 2017-09-01, 2017-10-01, 2017-11-01, 2018-01-01, 2018-02-01, 2018-03-01, 2018-04-01, 2018-05-01, 2018-06-01, 2018-07-01, 2018-08-01 - "Microsoft.Network": "2018-06-01", - "Microsoft.ClassicNetwork/reservedIps": "2016-04-01", //2014-01-01,2014-06-01,2015-06-01,2015-12-01,2016-04-01,2016-11-01 - "Microsoft.ClassicNetwork/networkSecurityGroups": "2016-11-01", //2015-06-01,2015-12-01,2016-04-01,2016-11-01 - "Microsoft.ClassicCompute/domainNames": "2015-12-01", //2014-01-01, 2014-06-01, 2015-06-01, 2015-10-01, 2015-12-01, 2016-04-01, 2016-11-01, 2017-11-01, 2017-11-15 - "Microsoft.Compute/locations": "2018-06-01", + "Microsoft.Compute/virtualMachineScaleSets": "2017-12-01", + "Microsoft.Compute/virtualMachines": "2018-04-01", + "Microsoft.ClassicCompute/virtualMachines": "2017-04-01", + "Microsoft.Compute/operations": "2018-10-01", + "Microsoft.ClassicCompute/operations": "2017-04-01", + "Microsoft.Network/virtualNetworks": "2018-08-01", + "Microsoft.ClassicNetwork/virtualNetworks": "2017-11-15", //avaliable 2014-01-01,2014-06-01,2015-06-01,2015-12-01,2016-04-01,2016-11-01,2017-11-15 + "Microsoft.Compute/disks": "2018-06-01", //avaliable 2016-04-30-preview,2017-03-30,2018-04-01,2018-06-01 + "Microsoft.Storage/storageAccounts": "2016-12-01", //2018-03-01-preview,2018-02-01,2017-10-01,2017-06-01,2016-12-01,2016-05-01,2016-01-01,2015-06-15,2015-05-01-preview + "Microsoft.ClassicStorage/storageAccounts": "2016-04-01", //2014-01-01,2014-04-01,2014-04-01-beta,2014-06-01,2015-06-01,2015-12-01,2016-04-01,2016-11-01 + "Microsoft.Compute/snapshots": "2018-06-01", //2016-04-30-preview,2017-03-30,2018-04-01,2018-06-01 + "Microsoft.Compute/images": "2018-10-01", //2016-04-30-preview,2016-08-30,2017-03-30,2017-12-01,2018-04-01,2018-06-01,2018-10-01 + "Microsoft.Storage": "2016-12-01", //2018-03-01-preview,2018-02-01,2017-10-01,2017-06-01,2016-12-01,2016-05-01,2016-01-01,2015-06-15,2015-05-01-preview + "Microsoft.Network/publicIPAddresses": "2018-06-01", //2014-12-01-preview, 2015-05-01-preview, 2015-06-15, 2016-03-30, 2016-06-01, 2016-07-01, 2016-08-01, 2016-09-01, 2016-10-01, 2016-11-01, 2016-12-01, 2017-03-01, 2017-04-01, 2017-06-01, 2017-08-01, 2017-09-01, 2017-10-01, 2017-11-01, 2018-01-01, 2018-02-01, 2018-03-01, 2018-04-01, 2018-05-01, 2018-06-01, 2018-07-01, 2018-08-01 + "Microsoft.Network/networkSecurityGroups": "2018-06-01", + "Microsoft.Network/networkInterfaces": "2018-06-01", //2014-12-01-preview, 2015-05-01-preview, 2015-06-15, 2016-03-30, 2016-06-01, 2016-07-01, 2016-08-01, 2016-09-01, 2016-10-01, 2016-11-01, 2016-12-01, 2017-03-01, 2017-04-01, 2017-06-01, 2017-08-01, 2017-09-01, 2017-10-01, 2017-11-01, 2018-01-01, 2018-02-01, 2018-03-01, 2018-04-01, 2018-05-01, 2018-06-01, 2018-07-01, 2018-08-01 + "Microsoft.Network": "2018-06-01", + "Microsoft.ClassicNetwork/reservedIps": "2016-04-01", //2014-01-01,2014-06-01,2015-06-01,2015-12-01,2016-04-01,2016-11-01 + "Microsoft.ClassicNetwork/networkSecurityGroups": "2016-11-01", //2015-06-01,2015-12-01,2016-04-01,2016-11-01 + "Microsoft.ClassicCompute/domainNames": "2015-12-01", //2014-01-01, 2014-06-01, 2015-06-01, 2015-10-01, 2015-12-01, 2016-04-01, 2016-11-01, 2017-11-01, 2017-11-15 + "Microsoft.Compute/locations": "2018-06-01", + "microsoft.insights/eventtypes/management/values": "2017-03-01-preview", } func NewAzureClient(providerId string, providerName string, envName, tenantId, clientId, clientSecret, subscriptionId string, debug bool) (*SAzureClient, error) { @@ -226,10 +227,14 @@ func (self *SAzureClient) ListAll(resourceType string, retVal interface{}) error return self.ListResources(resourceType, retVal, []string{"value"}) } -func (self *SAzureClient) ListResources(resourceType string, retVal interface{}, keys []string) error { +func (self *SAzureClient) ListAllWithNextToken(resourceType string, retVal interface{}) (string, error) { + return self.ListResourcesWithNextLink(resourceType, retVal, []string{"value"}) +} + +func (self *SAzureClient) ListResourcesWithNextLink(resourceType string, retVal interface{}, keys []string) (string, error) { cli, err := self.getDefaultClient() if err != nil { - return err + return "", err } url := "/subscriptions" if len(self.subscriptionId) > 0 { @@ -240,13 +245,22 @@ func (self *SAzureClient) ListResources(resourceType string, retVal interface{}, } body, err := jsonRequest(cli, "GET", self.domain, url, self.subscriptionId, "") if err != nil { - return err + return "", err } // fmt.Printf("%s: %s\n", resourceType, body) if retVal != nil { - return body.Unmarshal(retVal, keys...) + err = body.Unmarshal(retVal, keys...) + if err != nil { + return "", err + } } - return nil + nextLink, _ := body.GetString("nextLink") + return nextLink, nil +} + +func (self *SAzureClient) ListResources(resourceType string, retVal interface{}, keys []string) error { + _, err := self.ListResourcesWithNextLink(resourceType, retVal, keys) + return err } func (self *SAzureClient) ListSubscriptions() (jsonutils.JSONObject, error) { @@ -631,7 +645,11 @@ func _jsonRequest(client *autorest.Client, method, domain, baseURL, body string) } url := fmt.Sprintf("%s%s?api-version=%s", domain, baseURL, version) if strings.Index(baseURL, "?") > 0 { - url = fmt.Sprintf("%s%s&api-version=%s", domain, baseURL, version) + if strings.Contains(baseURL, "api-version") { + url = domain + baseURL + } else { + url = fmt.Sprintf("%s%s&api-version=%s", domain, baseURL, version) + } } req := &http.Request{} if len(body) != 0 { diff --git a/pkg/multicloud/azure/event.go b/pkg/multicloud/azure/event.go new file mode 100644 index 0000000000..d55ac05513 --- /dev/null +++ b/pkg/multicloud/azure/event.go @@ -0,0 +1,164 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +import ( + "fmt" + "net/url" + "strings" + "time" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudprovider" +) + +type SAuthorization struct { + Action string + Scope string +} + +type SClaims struct { + Aud string + Iss string + Iat string + Nbf string + Exp string + Aio string + Appid string + Appidacr string + Uti string + Ver string +} + +type SLocalized struct { + Value string + LocalizedValue string +} + +type SEvent struct { + region *SRegion + + Authorization SAuthorization + Channels string + Claims SClaims + CorrelationId string + Description string + EventDataId string + EventName SLocalized + Category SLocalized + Level string + ResourceGroupName string + ResourceProviderName SLocalized + ResourceId string + ResourceType SLocalized + OperationId string + OperationName SLocalized + Properties string + Status SLocalized + SubStatus SLocalized + Caller string + EventTimestamp time.Time + SubmissionTimestamp time.Time + SubscriptionId string + TenantId string + ID string + Name string +} + +func (event *SEvent) GetName() string { + return event.ResourceId +} + +func (event *SEvent) GetService() string { + return event.ResourceProviderName.Value +} + +func (event *SEvent) GetAction() string { + return event.OperationName.Value +} + +func (event *SEvent) GetResourceType() string { + return event.ResourceType.Value +} + +func (event *SEvent) GetRequestId() string { + return event.CorrelationId +} + +func (event *SEvent) GetRequest() jsonutils.JSONObject { + return jsonutils.Marshal(event) +} + +func (event *SEvent) GetAccount() string { + return event.Claims.Appid +} + +func (event *SEvent) IsSuccess() bool { + return event.Status.Value != "Failed" +} + +func (event *SEvent) GetCreatedAt() time.Time { + return event.EventTimestamp +} +func (region *SRegion) GetICloudEvents(start time.Time, end time.Time, withReadEvent bool) ([]cloudprovider.ICloudEvent, error) { + events, err := region.GetEvents(start, end) + if err != nil { + return nil, err + } + iEvents := []cloudprovider.ICloudEvent{} + for i := range events { + read := false + for _, k := range []string{"read", "listKeys"} { + if strings.Contains(events[i].Authorization.Action, k) { + read = true + break + } + } + if withReadEvent || !read { + iEvents = append(iEvents, &events[i]) + } + } + return iEvents, nil +} + +func (region *SRegion) GetEvents(start time.Time, end time.Time) ([]SEvent, error) { + events := []SEvent{} + params := url.Values{} + if start.IsZero() { + start = time.Now().AddDate(0, 0, -7) + } + if end.IsZero() { + end = time.Now() + } + params.Set("$filter", fmt.Sprintf("eventTimestamp ge '%s' and eventTimestamp le '%s' and eventChannels eq 'Admin, Operation' and levels eq 'Critical,Error,Warning,Informational'", start.Format("2006-01-02T15:04:05Z"), end.Format("2006-01-02T15:04:05Z"))) + nextLink := fmt.Sprintf("microsoft.insights/eventtypes/management/values?%s", params.Encode()) + var err error + for { + _events := []SEvent{} + nextLink, err = region.client.ListAllWithNextToken(nextLink, &_events) + if err != nil { + return nil, err + } + events = append(events, _events...) + if len(nextLink) > 0 { + nextLink = nextLink[strings.Index(nextLink, "microsoft.insights"):] + } + if len(nextLink) == 0 || len(_events) == 0 { + break + } + } + return events, nil +} diff --git a/pkg/multicloud/azure/provider/provider.go b/pkg/multicloud/azure/provider/provider.go index 8b3325fd76..35e255f5dd 100644 --- a/pkg/multicloud/azure/provider/provider.go +++ b/pkg/multicloud/azure/provider/provider.go @@ -41,6 +41,18 @@ func (self *SAzureProviderFactory) GetName() string { return azure.CLOUD_PROVIDER_AZURE_CN } +func (self *SAzureProviderFactory) GetMaxCloudEventKeepDays() int { + return 90 +} + +func (self *SAzureProviderFactory) GetMaxCloudEventSyncDays() int { + return 7 +} + +func (self *SAzureProviderFactory) IsCloudeventRegional() bool { + return false +} + func (self *SAzureProviderFactory) ValidateChangeBandwidth(instanceId string, bandwidth int64) error { return fmt.Errorf("Changing %s bandwidth is not supported", azure.CLOUD_PROVIDER_AZURE) } diff --git a/pkg/multicloud/azure/shell/event.go b/pkg/multicloud/azure/shell/event.go new file mode 100644 index 0000000000..2a2c2bf18f --- /dev/null +++ b/pkg/multicloud/azure/shell/event.go @@ -0,0 +1,37 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "time" + + "yunion.io/x/onecloud/pkg/multicloud/azure" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type EventListOptions struct { + Start time.Time + End time.Time + } + shellutils.R(&EventListOptions{}, "event-list", "List events", func(cli *azure.SRegion, args *EventListOptions) error { + events, err := cli.GetEvents(args.Start, args.End) + if err != nil { + return err + } + printList(events, len(events), 0, 0, []string{}) + return nil + }) +} diff --git a/pkg/multicloud/huawei/client/client.go b/pkg/multicloud/huawei/client/client.go index 315cf7947e..a2b9ea6a52 100644 --- a/pkg/multicloud/huawei/client/client.go +++ b/pkg/multicloud/huawei/client/client.go @@ -77,6 +77,7 @@ type Client struct { DBInstanceBackup *modules.SDBInstanceBackupManager DBInstanceFlavor *modules.SDBInstanceFlavorManager DBInstanceJob *modules.SDBInstanceJobManager + Traces *modules.STraceManager } func (self *Client) Init() error { @@ -158,6 +159,7 @@ func (self *Client) initManagers() { self.DBInstanceBackup = modules.NewDBInstanceBackupManager(self.regionId, self.projectId, self.signer, self.debug) self.DBInstanceFlavor = modules.NewDBInstanceFlavorManager(self.regionId, self.projectId, self.signer, self.debug) self.DBInstanceJob = modules.NewDBInstanceJobManager(self.regionId, self.projectId, self.signer, self.debug) + self.Traces = modules.NewTraceManager(self.regionId, self.projectId, self.signer, self.debug) } self.init = true diff --git a/pkg/multicloud/huawei/client/modules/manager_resource.go b/pkg/multicloud/huawei/client/modules/manager_resource.go index 13713ebee0..dc335631c6 100644 --- a/pkg/multicloud/huawei/client/modules/manager_resource.go +++ b/pkg/multicloud/huawei/client/modules/manager_resource.go @@ -48,6 +48,7 @@ const ( ServiceNameNAT ServiceNameType = "nat" // Nat网关 NAT ServiceNameDCS ServiceNameType = "dcs" // 分布式缓存服务 ServiceNameRDS ServiceNameType = "rds" // 关系型数据库 RDS + ServiceNameCTS ServiceNameType = "cts" //云审计服务 ) type SManagerContext struct { diff --git a/pkg/multicloud/huawei/client/modules/mod_traces.go b/pkg/multicloud/huawei/client/modules/mod_traces.go new file mode 100644 index 0000000000..0f6f3f70ed --- /dev/null +++ b/pkg/multicloud/huawei/client/modules/mod_traces.go @@ -0,0 +1,37 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package modules + +import ( + "yunion.io/x/onecloud/pkg/multicloud/huawei/client/auth" +) + +type STraceManager struct { + SResourceManager +} + +func NewTraceManager(regionId string, projectId string, signer auth.Signer, debug bool) *STraceManager { + return &STraceManager{SResourceManager: SResourceManager{ + SBaseManager: NewBaseManager(signer, debug), + ServiceName: ServiceNameCTS, + Region: regionId, + ProjectId: projectId, + version: "v2.0", + Keyword: "trace", + KeywordPlural: "traces", + + ResourceKeyword: "system/trace", + }} +} diff --git a/pkg/multicloud/huawei/provider/provider.go b/pkg/multicloud/huawei/provider/provider.go index 3ac89ab5d8..0d0c3f523d 100644 --- a/pkg/multicloud/huawei/provider/provider.go +++ b/pkg/multicloud/huawei/provider/provider.go @@ -39,6 +39,18 @@ func (self *SHuaweiProviderFactory) GetName() string { return huawei.CLOUD_PROVIDER_HUAWEI_CN } +func (self *SHuaweiProviderFactory) IsCloudeventRegional() bool { + return true +} + +func (self *SHuaweiProviderFactory) GetMaxCloudEventSyncDays() int { + return 7 +} + +func (self *SHuaweiProviderFactory) GetMaxCloudEventKeepDays() int { + return 7 +} + func (self *SHuaweiProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCreateInput) error { if len(input.AccessKeyId) == 0 { return httperrors.NewMissingParameterError("access_key_id") diff --git a/pkg/multicloud/huawei/shell/events.go b/pkg/multicloud/huawei/shell/events.go new file mode 100644 index 0000000000..87b0004ad6 --- /dev/null +++ b/pkg/multicloud/huawei/shell/events.go @@ -0,0 +1,37 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "time" + + "yunion.io/x/onecloud/pkg/multicloud/huawei" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type EventListOptions struct { + Start time.Time + End time.Time + } + shellutils.R(&EventListOptions{}, "event-list", "List events", func(cli *huawei.SRegion, args *EventListOptions) error { + events, err := cli.GetEvents(args.Start, args.End) + if err != nil { + return err + } + printList(events, 0, 0, 0, []string{}) + return nil + }) +} diff --git a/pkg/multicloud/huawei/traces.go b/pkg/multicloud/huawei/traces.go new file mode 100644 index 0000000000..95116456d3 --- /dev/null +++ b/pkg/multicloud/huawei/traces.go @@ -0,0 +1,130 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package huawei + +import ( + "fmt" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/cloudprovider" +) + +type SUser struct { + Domain map[string]string + Name string + Id string +} + +type SEvent struct { + TraceId string + Code string + TraceName string + ResourceType string + ApiVersion string + SourceIp string + TraceType string + ServiceType string + EventType string + ProjectId string + Request string + Response string + TrackerName string + TraceStatus string + Time int64 + ResourceId string + ResourceName string + User SUser + RecordTime int64 +} + +func (event *SEvent) GetName() string { + if len(event.ResourceId) > 0 { + return event.ResourceId + } + if len(event.ResourceName) > 0 { + return event.ResourceName + } + return event.TraceName +} + +func (event *SEvent) GetService() string { + return event.ServiceType +} + +func (event *SEvent) GetAction() string { + return event.TraceName +} + +func (event *SEvent) GetResourceType() string { + return event.ResourceType +} + +func (event *SEvent) GetRequestId() string { + return event.TraceId +} + +func (event *SEvent) GetRequest() jsonutils.JSONObject { + return jsonutils.Marshal(event) +} + +func (event *SEvent) GetAccount() string { + return event.User.Name +} + +func (event *SEvent) IsSuccess() bool { + return strings.HasPrefix(event.Code, "2") +} + +func (event *SEvent) GetCreatedAt() time.Time { + return time.Unix(event.Time/1000, event.Time%1000) +} + +func (self *SRegion) GetICloudEvents(start time.Time, end time.Time, withReadEvent bool) ([]cloudprovider.ICloudEvent, error) { + if !self.client.isMainProject { + return nil, cloudprovider.ErrNotSupported + } + events, err := self.GetEvents(start, end) + if err != nil { + return nil, err + } + iEvents := []cloudprovider.ICloudEvent{} + for i := range events { + iEvents = append(iEvents, &events[i]) + } + return iEvents, nil +} + +func (self *SRegion) GetEvents(start time.Time, end time.Time) ([]SEvent, error) { + events := []SEvent{} + params := map[string]string{} + if start.IsZero() { + start = time.Now().AddDate(0, 0, -7) + } + if end.IsZero() { + end = time.Now() + } + params["from"] = fmt.Sprintf("%d000", start.Unix()) + params["to"] = fmt.Sprintf("%d000", end.Unix()) + + err := doListAllWithMarker(self.ecsClient.Traces.List, params, &events) + if err != nil { + return nil, errors.Wrap(err, "doListAllWithMarker") + } + return events, nil +} diff --git a/pkg/multicloud/qcloud/events.go b/pkg/multicloud/qcloud/events.go new file mode 100644 index 0000000000..cd56a02efa --- /dev/null +++ b/pkg/multicloud/qcloud/events.go @@ -0,0 +1,158 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package qcloud + +import ( + "fmt" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/cloudprovider" +) + +type SEvent struct { + region *SRegion + AccountID int64 + CloudAuditEvent string + ErrorCode int + EventId string + EventName string + EventNameCn string + EventRegion string + EventSource string + EventTime time.Time + RequestID string + ResourceRegion string + ResourceTypeCn string + Resources map[string]string + SecretId string + SourceIPAddress string + Username string +} + +func (event *SEvent) GetName() string { + if resourceName, ok := event.Resources["ResourceName"]; ok && len(resourceName) > 0 { + return resourceName + } + return event.EventName +} + +func (event *SEvent) GetService() string { + return event.ResourceTypeCn +} + +func (event *SEvent) GetAction() string { + return event.EventName +} + +func (event *SEvent) GetResourceType() string { + if resourceType, ok := event.Resources["ResourceType"]; ok { + return resourceType + } + return "" +} + +func (event *SEvent) GetRequest() jsonutils.JSONObject { + return jsonutils.Marshal(event) +} + +func (event *SEvent) GetRequestId() string { + return event.RequestID +} + +func (event *SEvent) GetAccount() string { + if len(event.SecretId) > 0 { + return event.SecretId + } + return event.Username +} + +func (event *SEvent) IsSuccess() bool { + return event.ErrorCode > 0 +} + +func (event *SEvent) GetCreatedAt() time.Time { + return event.EventTime +} + +func (region *SRegion) GetICloudEvents(start time.Time, end time.Time, withReadEvent bool) ([]cloudprovider.ICloudEvent, error) { + events, err := region.GetEvents(start, end) + if err != nil { + return nil, err + } + iEvents := []cloudprovider.ICloudEvent{} + for i := range events { + if withReadEvent || !strings.Contains(events[i].CloudAuditEvent, `"Read"`) { + iEvents = append(iEvents, &events[i]) + } + } + return iEvents, nil +} + +func (region *SRegion) GetEvents(start time.Time, end time.Time) ([]SEvent, error) { + var ( + events []SEvent + nextToken string + err error + _events []SEvent + ) + for { + _events, nextToken, err = region.getEvents(start, end, nextToken) + if err != nil { + return nil, err + } + if len(nextToken) == 0 { + break + } + events = append(events, _events...) + } + return events, nil +} + +func (region *SRegion) getEvents(start time.Time, end time.Time, nextToken string) ([]SEvent, string, error) { + params := map[string]string{} + if start.IsZero() { + start = time.Now().AddDate(0, 0, -7) + } + if end.IsZero() { + end = time.Now() + } + params["StartTime"] = fmt.Sprintf("%d", start.Unix()) + params["EndTime"] = fmt.Sprintf("%d", end.Unix()) + params["MaxResults"] = "50" + if len(nextToken) > 0 { + params["NextToken"] = nextToken + } + + body, err := region.auditRequest("LookUpEvents", params) + if err != nil { + return nil, "", err + } + + events := []SEvent{} + + err = body.Unmarshal(&events, "Events") + if err != nil { + return nil, "", errors.Wrap(err, "body.Unmarshal") + } + nextToken, _ = body.GetString("NextToken") + if over, _ := body.Bool("ListOver"); over { + nextToken = "" + } + return events, nextToken, nil +} diff --git a/pkg/multicloud/qcloud/provider/provider.go b/pkg/multicloud/qcloud/provider/provider.go index 2a28f5660e..4c83cec99c 100644 --- a/pkg/multicloud/qcloud/provider/provider.go +++ b/pkg/multicloud/qcloud/provider/provider.go @@ -40,6 +40,18 @@ func (self *SQcloudProviderFactory) GetName() string { return qcloud.CLOUD_PROVIDER_QCLOUD_CN } +func (self *SQcloudProviderFactory) IsCloudeventRegional() bool { + return true +} + +func (self *SQcloudProviderFactory) GetMaxCloudEventSyncDays() int { + return 7 +} + +func (self *SQcloudProviderFactory) GetMaxCloudEventKeepDays() int { + return 30 +} + func (self *SQcloudProviderFactory) ValidateChangeBandwidth(instanceId string, bandwidth int64) error { if len(instanceId) == 0 { return fmt.Errorf("Only changes to the binding machine's EIP bandwidth are supported") diff --git a/pkg/multicloud/qcloud/qcloud.go b/pkg/multicloud/qcloud/qcloud.go index 841981bbc0..d5f3f7e412 100644 --- a/pkg/multicloud/qcloud/qcloud.go +++ b/pkg/multicloud/qcloud/qcloud.go @@ -49,6 +49,7 @@ const ( QCLOUD_API_VERSION = "2017-03-12" QCLOUD_CLB_API_VERSION = "2018-03-17" QCLOUD_BILLING_API_VERSION = "2018-07-09" + QCLOUD_AUDIT_API_VERSION = "2019-03-19" ) type SQcloudClient struct { @@ -116,6 +117,11 @@ func vpcRequest(client *common.Client, apiName string, params map[string]string, return _jsonRequest(client, domain, QCLOUD_API_VERSION, apiName, params, debug, true) } +func auditRequest(client *common.Client, apiName string, params map[string]string, debug bool) (jsonutils.JSONObject, error) { + domain := apiDomain("cloudaudit", params) + return _jsonRequest(client, domain, QCLOUD_AUDIT_API_VERSION, apiName, params, debug, true) +} + func cbsRequest(client *common.Client, apiName string, params map[string]string, debug bool) (jsonutils.JSONObject, error) { domain := apiDomain("cbs", params) return _jsonRequest(client, domain, QCLOUD_API_VERSION, apiName, params, debug, true) @@ -361,6 +367,9 @@ func _baseJsonRequest(client *common.Client, req tchttp.Request, resp qcloudResp if strings.Contains(err.Error(), "Code=ResourceNotFound") { return nil, cloudprovider.ErrNotFound } + if strings.Contains(err.Error(), "Code=UnsupportedRegion") { + return nil, cloudprovider.ErrNotSupported + } if needRetry { log.Errorf("request url %s\nparams: %s\nerror: %v\ntry after %d seconds", req.GetDomain(), jsonutils.Marshal(req.GetParams()).PrettyString(), err, i*10) time.Sleep(time.Second * time.Duration(i*10)) @@ -400,6 +409,14 @@ func (client *SQcloudClient) vpcRequest(apiName string, params map[string]string return vpcRequest(cli, apiName, params, client.Debug) } +func (client *SQcloudClient) auditRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { + cli, err := client.getDefaultClient() + if err != nil { + return nil, err + } + return auditRequest(cli, apiName, params, client.Debug) +} + func (client *SQcloudClient) cbsRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { cli, err := client.getDefaultClient() if err != nil { diff --git a/pkg/multicloud/qcloud/region.go b/pkg/multicloud/qcloud/region.go index 3b721c8815..d2c18db9d0 100644 --- a/pkg/multicloud/qcloud/region.go +++ b/pkg/multicloud/qcloud/region.go @@ -632,6 +632,11 @@ func (self *SRegion) vpcRequest(apiName string, params map[string]string) (jsonu return self.client.vpcRequest(apiName, params) } +func (self *SRegion) auditRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { + params["Region"] = self.Region + return self.client.auditRequest(apiName, params) +} + func (self *SRegion) vpc2017Request(apiName string, params map[string]string) (jsonutils.JSONObject, error) { params["Region"] = self.Region return self.client.vpc2017Request(apiName, params) diff --git a/pkg/multicloud/qcloud/shell/events.go b/pkg/multicloud/qcloud/shell/events.go new file mode 100644 index 0000000000..8e19fad15e --- /dev/null +++ b/pkg/multicloud/qcloud/shell/events.go @@ -0,0 +1,40 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "time" + + "yunion.io/x/onecloud/pkg/multicloud/qcloud" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type EventListOptions struct { + Start time.Time + End time.Time + Offset int `help:"List offset"` + Limit int `help:"List limit"` + } + shellutils.R(&EventListOptions{}, "event-list", "List events", func(cli *qcloud.SRegion, args *EventListOptions) error { + events, err := cli.GetEvents(args.Start, args.End) + if err != nil { + return err + } + printList(events, 0, args.Offset, args.Limit, []string{}) + return nil + }) + +} diff --git a/pkg/multicloud/region_base.go b/pkg/multicloud/region_base.go index ca560e1d6d..95246a4c6f 100644 --- a/pkg/multicloud/region_base.go +++ b/pkg/multicloud/region_base.go @@ -16,6 +16,7 @@ package multicloud import ( "fmt" + "time" "yunion.io/x/onecloud/pkg/cloudprovider" ) @@ -93,3 +94,7 @@ func (self *SRegion) CreateIElasticcaches(ec *cloudprovider.SCloudElasticCacheIn func (self *SRegion) GetIElasticcacheById(id string) (cloudprovider.ICloudElasticcache, error) { return nil, fmt.Errorf("Not Implemented GetIElasticcacheById") } + +func (self *SRegion) GetICloudEvents(start time.Time, end time.Time, withReadEvent bool) ([]cloudprovider.ICloudEvent, error) { + return nil, fmt.Errorf("Not Implemented GetICloudEvnets") +}