diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index 2f524841d3..9121b32e05 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -43,6 +43,8 @@ type ICloudRegion interface { GetIHostById(id string) (ICloudHost, error) GetIStorageById(id string) (ICloudStorage, error) GetIStoragecacheById(id string) (ICloudStoragecache, error) + + DeleteSecurityGroup(vpcId, secgroupId string) error SyncSecurityGroup(secgroupId string, vpcId string, name string, desc string, rules []secrules.SecurityRule) (string, error) CreateIVpc(name string, desc string, cidr string) (ICloudVpc, error) @@ -164,10 +166,8 @@ type ICloudVM interface { GetBios() string GetMachine() string - RevokeSecurityGroup() error AssignSecurityGroup(secgroupId string) error - //SyncSecurityGroup(secgroupId string, name string, rules []secrules.SecurityRule) error GetHypervisor() string // GetSecurityGroup() ICloudSecurityGroup @@ -221,6 +221,7 @@ type ICloudSecurityGroup interface { ICloudResource GetDescription() string GetRules() ([]secrules.SecurityRule, error) + GetVpcId() string } type ICloudDisk interface { @@ -253,12 +254,10 @@ type ICloudDisk interface { type ICloudSnapshot interface { ICloudResource - GetManagerId() string GetSize() int32 GetDiskId() string GetDiskType() string Delete() error - GetRegionId() string } type ICloudVpc interface { @@ -276,8 +275,6 @@ type ICloudVpc interface { Delete() error GetIWireById(wireId string) (ICloudWire, error) - - //SyncSecurityGroup(secgroupId string, name string, rules []secrules.SecurityRule) (string, error) } type ICloudWire interface { diff --git a/pkg/compute/guestdrivers/aliyun.go b/pkg/compute/guestdrivers/aliyun.go index 190891966e..640669a294 100644 --- a/pkg/compute/guestdrivers/aliyun.go +++ b/pkg/compute/guestdrivers/aliyun.go @@ -202,19 +202,18 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu if action == "create" { taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - iregion, err := host.GetIRegion() - if err != nil { - log.Errorf("GetIRegion fail %s", err) - return nil, err - } nets := guest.GetNetworks() net := nets[0].GetNetwork() vpc := net.GetVpc() + iregion, err := host.GetIRegion() + if err != nil { + return nil, err + } - secgroupCache := models.SecurityGroupCacheManager.Register(ctx, task.GetUserCred(), desc.SecGroupId, host.GetRegion().GetId(), host.ManagerId) + secgroupCache := models.SecurityGroupCacheManager.Register(ctx, task.GetUserCred(), desc.SecGroupId, vpc.Id, vpc.CloudregionId, vpc.ManagerId) if secgroupCache == nil { - return nil, fmt.Errorf("failed to registor secgroupCache for secgroup: %s, regionId: %s, provider: %s", desc.SecGroupId, host.GetRegion().GetId(), host.ManagerId) + return nil, fmt.Errorf("failed to registor secgroupCache for secgroup: %s, vpc: %s", desc.SecGroupId, vpc.Name) } secgroupExtId, err := iregion.SyncSecurityGroup(secgroupCache.ExternalId, vpc.ExternalId, desc.SecGroupName, "", desc.SecRules) @@ -490,26 +489,3 @@ func (self *SAliyunGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Contex func (self *SAliyunGuestDriver) AllowReconfigGuest() bool { return true } - -func (self *SAliyunGuestDriver) RequestDiskSnapshot(ctx context.Context, guest *models.SGuest, task taskman.ITask, snapshotId, diskId string) error { - iDisk, _ := models.DiskManager.FetchById(diskId) - disk := iDisk.(*models.SDisk) - providerDisk, err := disk.GetIDisk() - if err != nil { - return err - } - iSnapshot, _ := models.SnapshotManager.FetchById(snapshotId) - snapshot := iSnapshot.(*models.SSnapshot) - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - cloudSnapshot, err := providerDisk.CreateISnapshot(snapshot.Name, "") - if err != nil { - return nil, err - } - res := jsonutils.NewDict() - res.Set("snapshot_id", jsonutils.NewString(cloudSnapshot.GetId())) - res.Set("manager_id", jsonutils.NewString(cloudSnapshot.GetManagerId())) - res.Set("cloudregion_id", jsonutils.NewString(cloudSnapshot.GetRegionId())) - return res, nil - }) - return nil -} diff --git a/pkg/compute/guestdrivers/azure.go b/pkg/compute/guestdrivers/azure.go index 0fb4555cc7..f219d0d426 100644 --- a/pkg/compute/guestdrivers/azure.go +++ b/pkg/compute/guestdrivers/azure.go @@ -3,6 +3,7 @@ package guestdrivers import ( "context" "fmt" + "strings" "time" "yunion.io/x/log" @@ -10,6 +11,7 @@ import ( "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/seclib2" + "yunion.io/x/pkg/util/compare" "yunion.io/x/pkg/utils" "yunion.io/x/jsonutils" @@ -138,22 +140,26 @@ func (self *SAzureGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gue passwd = seclib2.RandomPassword2(12) } - iregion, err := host.GetIRegion() - if err != nil { - log.Errorf("GetIRegion fail %s", err) - return nil, err - } - - secgroupCache := models.SecurityGroupCacheManager.Register(ctx, task.GetUserCred(), desc.SecGroupId, host.GetRegion().GetId(), host.ManagerId) - if secgroupCache == nil { - return nil, fmt.Errorf("failed to registor secgroupCache for secgroup: %s, regionId: %s, provider: %s", desc.SecGroupId, host.GetRegion().GetId(), host.ManagerId) - } - nets := guest.GetNetworks() net := nets[0].GetNetwork() vpc := net.GetVpc() - secgroupExtId, err := iregion.SyncSecurityGroup(secgroupCache.ExternalId, vpc.ExternalId, desc.SecGroupName, "", desc.SecRules) + iregion, err := host.GetIRegion() + if err != nil { + return nil, err + } + + vpcId := "normal" + if strings.HasSuffix(host.Name, "-classic") { + vpcId = "classic" + } + + secgroupCache := models.SecurityGroupCacheManager.Register(ctx, task.GetUserCred(), desc.SecGroupId, vpcId, vpc.CloudregionId, vpc.ManagerId) + if secgroupCache == nil { + return nil, fmt.Errorf("failed to registor secgroupCache for secgroup: %s, vpc: %s", desc.SecGroupId, vpc.Name) + } + + secgroupExtId, err := iregion.SyncSecurityGroup(secgroupCache.ExternalId, vpcId, desc.SecGroupName, "", desc.SecRules) if err != nil { log.Errorf("SyncSecurityGroup fail %s", err) return nil, err @@ -162,22 +168,21 @@ func (self *SAzureGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gue return nil, fmt.Errorf("failed to set externalId for secgroup %s externalId %s: error: %v", desc.SecGroupId, secgroupExtId, err) } - if iVM, err := ihost.CreateVM(desc.Name, desc.ExternalImageId, desc.SysDiskSize, desc.Cpu, desc.Memory, desc.ExternalNetworkId, - desc.IpAddr, desc.Description, passwd, desc.StorageType, desc.DataDisks, publicKey, secgroupExtId); err != nil { + iVM, err := ihost.CreateVM(desc.Name, desc.ExternalImageId, desc.SysDiskSize, desc.Cpu, desc.Memory, desc.ExternalNetworkId, + desc.IpAddr, desc.Description, passwd, desc.StorageType, desc.DataDisks, publicKey, secgroupExtId) + if err != nil { return nil, err - } else { - log.Debugf("VMcreated %s, wait status running ...", iVM.GetGlobalId()) - if err = cloudprovider.WaitStatus(iVM, models.VM_RUNNING, time.Second*5, time.Second*1800); err != nil { - return nil, err - } - if iVM, err = ihost.GetIVMById(iVM.GetGlobalId()); err != nil { - log.Errorf("cannot find vm %s", err) - return nil, err - } - - data := fetchIVMinfo(desc, iVM, guest.Id, DEFAULT_USER, passwd, action) - return data, nil } + log.Debugf("VMcreated %s, wait status running ...", iVM.GetGlobalId()) + if err = cloudprovider.WaitStatus(iVM, models.VM_RUNNING, time.Second*5, time.Second*1800); err != nil { + return nil, err + } + if iVM, err = ihost.GetIVMById(iVM.GetGlobalId()); err != nil { + log.Errorf("cannot find vm %s", err) + return nil, err + } + + return fetchIVMinfo(desc, iVM, guest.Id, DEFAULT_USER, passwd, action), nil }) } else if action == "deploy" { iVM, err := ihost.GetIVMById(guest.GetExternalId()) @@ -292,3 +297,70 @@ func (self *SAzureGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Context guest.SaveDeployInfo(ctx, task.GetUserCred(), data) return nil } + +func (self *SAzureGuestDriver) RequestSyncConfigOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + ihost, err := host.GetIHost() + if err != nil { + return nil, err + } + iVM, err := ihost.GetIVMById(guest.ExternalId) + if err != nil { + return nil, err + } + + if fwOnly, _ := task.GetParams().Bool("fw_only"); fwOnly { + vpcID := "normal" + if strings.HasSuffix(host.Name, "-classic") { + vpcID = "classic" + } + iregion, err := host.GetIRegion() + if err != nil { + return nil, err + } + secgroupCache := models.SecurityGroupCacheManager.Register(ctx, task.GetUserCred(), guest.SecgrpId, vpcID, host.GetRegion().Id, host.ManagerId) + if secgroupCache == nil { + return nil, fmt.Errorf("failed to registor secgroupCache for secgroup: %s vpc: %s", guest.SecgrpId, vpcID) + } + extID, err := iregion.SyncSecurityGroup(secgroupCache.ExternalId, vpcID, guest.GetSecgroupName(), "", guest.GetSecRules()) + if err != nil { + return nil, err + } + if err = secgroupCache.SetExternalId(extID); err != nil { + return nil, err + } + return nil, iVM.AssignSecurityGroup(extID) + } + + iDisks, err := iVM.GetIDisks() + if err != nil { + return nil, err + } + disks := make([]models.SDisk, 0) + for _, guestdisk := range guest.GetDisks() { + disk := guestdisk.GetDisk() + disks = append(disks, *disk) + } + + added := make([]models.SDisk, 0) + commondb := make([]models.SDisk, 0) + commonext := make([]cloudprovider.ICloudDisk, 0) + removed := make([]cloudprovider.ICloudDisk, 0) + + if err := compare.CompareSets(disks, iDisks, &added, &commondb, &commonext, &removed); err != nil { + return nil, err + } + for _, disk := range removed { + if err := iVM.DetachDisk(disk.GetId()); err != nil { + return nil, err + } + } + for _, disk := range added { + if err := iVM.AttachDisk(disk.ExternalId); err != nil { + return nil, err + } + } + return nil, nil + }) + return nil +} diff --git a/pkg/compute/guestdrivers/managedvirtual.go b/pkg/compute/guestdrivers/managedvirtual.go index 0b0d13c151..5c28bb5240 100644 --- a/pkg/compute/guestdrivers/managedvirtual.go +++ b/pkg/compute/guestdrivers/managedvirtual.go @@ -305,6 +305,27 @@ func (self *SManagedVirtualizedGuestDriver) RequestChangeVmConfig(ctx context.Co return nil } +func (self *SManagedVirtualizedGuestDriver) RequestDiskSnapshot(ctx context.Context, guest *models.SGuest, task taskman.ITask, snapshotId, diskId string) error { + iDisk, _ := models.DiskManager.FetchById(diskId) + disk := iDisk.(*models.SDisk) + providerDisk, err := disk.GetIDisk() + if err != nil { + return err + } + iSnapshot, _ := models.SnapshotManager.FetchById(snapshotId) + snapshot := iSnapshot.(*models.SSnapshot) + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + cloudSnapshot, err := providerDisk.CreateISnapshot(snapshot.Name, "") + if err != nil { + return nil, err + } + res := jsonutils.NewDict() + res.Set("snapshot_id", jsonutils.NewString(cloudSnapshot.GetId())) + return res, nil + }) + return nil +} + func (self *SManagedVirtualizedGuestDriver) RequestSyncConfigOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { ihost, err := host.GetIHost() @@ -315,20 +336,8 @@ func (self *SManagedVirtualizedGuestDriver) RequestSyncConfigOnHost(ctx context. if err != nil { return nil, err } - if fw_only, _ := task.GetParams().Bool("fw_only"); fw_only { - if len(guest.SecgrpId) == 0 { - return nil, iVM.RevokeSecurityGroup() - } - iregion, err := host.GetIRegion() - if err != nil { - return nil, err - } - region := host.GetRegion() - secgroupCache := models.SecurityGroupCacheManager.Register(ctx, task.GetUserCred(), guest.SecgrpId, region.GetId(), host.ManagerId) - if secgroupCache == nil { - return nil, fmt.Errorf("failed to registor secgroupCache for secgroup: %s region: %s, provider: %s", guest.SecgrpId, region.GetId(), host.ManagerId) - } + if fwOnly, _ := task.GetParams().Bool("fw_only"); fwOnly { vpcId := "" for _, network := range guest.GetNetworks() { if vpc := network.GetNetwork().GetVpc(); vpc != nil { @@ -336,16 +345,24 @@ func (self *SManagedVirtualizedGuestDriver) RequestSyncConfigOnHost(ctx context. break } } - - extId, err := iregion.SyncSecurityGroup(secgroupCache.ExternalId, vpcId, guest.GetSecgroupName(), "", guest.GetSecRules()) + iregion, err := host.GetIRegion() if err != nil { return nil, err } - if err = secgroupCache.SetExternalId(extId); err != nil { + secgroupCache := models.SecurityGroupCacheManager.Register(ctx, task.GetUserCred(), guest.SecgrpId, vpcId, host.GetRegion().Id, host.ManagerId) + if secgroupCache == nil { + return nil, fmt.Errorf("failed to registor secgroupCache for secgroup: %s vpc: %s", guest.SecgrpId, vpcId) + } + extID, err := iregion.SyncSecurityGroup(secgroupCache.ExternalId, vpcId, guest.GetSecgroupName(), "", guest.GetSecRules()) + if err != nil { return nil, err } - return nil, iVM.AssignSecurityGroup(extId) + if err = secgroupCache.SetExternalId(extID); err != nil { + return nil, err + } + return nil, iVM.AssignSecurityGroup(extID) } + iDisks, err := iVM.GetIDisks() if err != nil { return nil, err diff --git a/pkg/compute/models/secgroupcache.go b/pkg/compute/models/secgroupcache.go index 953f9b255f..41d785b68c 100644 --- a/pkg/compute/models/secgroupcache.go +++ b/pkg/compute/models/secgroupcache.go @@ -8,6 +8,7 @@ import ( "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/pkg/util/stringutils" @@ -24,8 +25,9 @@ type SSecurityGroupCache struct { Id string `width:"128" charset:"ascii" primary:"true" list:"user"` SecgroupId string `width:"128" charset:"ascii" create:"required"` + VpcId string `width:"128" charset:"ascii" create:"required"` + CloudregionId string `width:"128" charset:"ascii" create:"required"` ExternalId string `width:"256" charset:"utf8" index:"true" list:"admin" create:"admin_optional"` - CloudregionId string `width:"36" charset:"ascii" nullable:"true" list:"user"` } var SecurityGroupCacheManager *SSecurityGroupCacheManager @@ -69,25 +71,50 @@ func (manager *SSecurityGroupCacheManager) ListItemFilter(ctx context.Context, q if secgroup, _ := SecurityGroupManager.FetchByIdOrName(userCred.GetProjectId(), defsecgroup); secgroup != nil { sql = sql.Equals("secgroup_id", secgroup.GetId()) } else { - return nil, httperrors.NewNotFoundError(fmt.Sprintf("Security Group %s not found", defsecgroup)) + return nil, httperrors.NewNotFoundError("Security Group %s not found", defsecgroup) } } return sql, nil } +func (self *SSecurityGroupCache) GetIRegion() (cloudprovider.ICloudRegion, error) { + provider, err := self.GetDriver() + if err != nil { + return nil, err + } + if region := CloudregionManager.FetchRegionById(self.CloudregionId); region != nil { + return provider.GetIRegionById(region.ExternalId) + } + return nil, fmt.Errorf("failed to find iregion for secgroupcache %s vpc: %s externalId: %s", self.Id, self.VpcId, self.ExternalId) +} + +func (self *SSecurityGroupCache) DeleteCloudSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential) error { + if len(self.ExternalId) > 0 { + iregion, err := self.GetIRegion() + if err != nil { + return err + } + return iregion.DeleteSecurityGroup(self.VpcId, self.ExternalId) + } + return nil +} + func (self *SSecurityGroupCache) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + if err := self.DeleteCloudSecurityGroup(ctx, userCred); err != nil { + log.Errorf("delete secgroup cache %v error: %v", self, err) + } return db.DeleteModel(ctx, userCred, self) } -func (manager *SSecurityGroupCacheManager) GetSecgroupCache(ctx context.Context, userCred mcclient.TokenCredential, secgroupId, regionId, providerId string) *SSecurityGroupCache { +func (manager *SSecurityGroupCacheManager) GetSecgroupCache(ctx context.Context, userCred mcclient.TokenCredential, secgroupId, vpcId string, regionId string, providerId string) *SSecurityGroupCache { secgroupCache := SSecurityGroupCache{} query := manager.Query() - cond := sqlchemy.AND(sqlchemy.Equals(query.Field("secgroup_id"), secgroupId), sqlchemy.Equals(query.Field("cloudregion_id"), regionId), sqlchemy.Equals(query.Field("manager_id"), providerId)) + cond := sqlchemy.AND(sqlchemy.Equals(query.Field("secgroup_id"), secgroupId), sqlchemy.Equals(query.Field("vpc_id"), vpcId), sqlchemy.Equals(query.Field("cloudregion_id"), regionId), sqlchemy.Equals(query.Field("manager_id"), providerId)) query = query.Filter(cond) count := query.Count() if count > 1 { - log.Errorf("duplicate secgroupcache for secgroup: %s regionId: %s providerId: %s", secgroupId, regionId, providerId) + log.Errorf("duplicate secgroupcache for secgroup: %s vpcId: %s regionId: %s", secgroupId, vpcId, regionId) } else if count == 0 { return nil } @@ -96,21 +123,22 @@ func (manager *SSecurityGroupCacheManager) GetSecgroupCache(ctx context.Context, return &secgroupCache } -func (manager *SSecurityGroupCacheManager) Register(ctx context.Context, userCred mcclient.TokenCredential, secgroupId, regionId, providerId string) *SSecurityGroupCache { +func (manager *SSecurityGroupCacheManager) Register(ctx context.Context, userCred mcclient.TokenCredential, secgroupId, vpcId, regionId string, providerId string) *SSecurityGroupCache { lockman.LockClass(ctx, manager, userCred.GetProjectId()) defer lockman.ReleaseClass(ctx, manager, userCred.GetProjectId()) - secgroupCache := manager.GetSecgroupCache(ctx, userCred, secgroupId, regionId, providerId) + secgroupCache := manager.GetSecgroupCache(ctx, userCred, secgroupId, vpcId, regionId, providerId) if secgroupCache != nil { return secgroupCache } secgroupCache = &SSecurityGroupCache{ SecgroupId: secgroupId, + VpcId: vpcId, CloudregionId: regionId, } - secgroupCache.SetModelManager(manager) secgroupCache.ManagerId = providerId + secgroupCache.SetModelManager(manager) if err := manager.TableSpec().Insert(secgroupCache); err != nil { log.Errorf("insert secgroupcache error: %v", err) return nil diff --git a/pkg/compute/models/secgroups.go b/pkg/compute/models/secgroups.go index f570af9de5..835943b580 100644 --- a/pkg/compute/models/secgroups.go +++ b/pkg/compute/models/secgroups.go @@ -15,6 +15,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" @@ -194,64 +195,7 @@ func (manager *SSecurityGroupManager) getSecurityGroups() ([]SSecurityGroup, err } } -func (manager *SSecurityGroupManager) SyncSecgroups(ctx context.Context, userCred mcclient.TokenCredential, secgroups []cloudprovider.ICloudSecurityGroup, regionId, providerId string) ([]SSecurityGroup, []cloudprovider.ICloudSecurityGroup, compare.SyncResult) { - localSecgroups := make([]SSecurityGroup, 0) - remoteSecgroups := make([]cloudprovider.ICloudSecurityGroup, 0) - syncResult := compare.SyncResult{} - - if dbSecgroups, err := manager.getSecurityGroups(); err != nil { - syncResult.Error(err) - return nil, nil, syncResult - } else { - removed := make([]SSecurityGroup, 0) - commondb := make([]SSecurityGroup, 0) - commonext := make([]cloudprovider.ICloudSecurityGroup, 0) - added := make([]cloudprovider.ICloudSecurityGroup, 0) - if err := compare.CompareSets(dbSecgroups, secgroups, &removed, &commondb, &commonext, &added); err != nil { - syncResult.Error(err) - return nil, nil, syncResult - } - - for i := 0; i < len(commondb); i += 1 { - if rules, err := commonext[i].GetRules(); err != nil { - syncResult.Error(err) - } else if len(rules) > 0 { - if err = commondb[i].SyncWithCloudSecurityGroup(userCred, commonext[i]); err != nil { - syncResult.UpdateError(err) - } else { - localSecgroups = append(localSecgroups, commondb[i]) - remoteSecgroups = append(remoteSecgroups, commonext[i]) - SecurityGroupRuleManager.SyncRules(ctx, userCred, &commondb[i], rules) - syncResult.Update() - } - } - } - - for i := 0; i < len(added); i += 1 { - if metadata := added[i].GetMetadata(); metadata != nil && metadata.Contains("id") { - secgroupId, _ := metadata.GetString("id") - if secgrp, _ := manager.FetchById(secgroupId); secgrp != nil { - continue - } - } - if rules, err := added[i].GetRules(); err != nil { - syncResult.AddError(err) - } else if len(rules) > 0 { - if new, err := manager.newFromCloudVpc(userCred, added[i], regionId, providerId); err != nil { - syncResult.AddError(err) - } else if len(rules) > 0 { - localSecgroups = append(localSecgroups, *new) - remoteSecgroups = append(remoteSecgroups, added[i]) - SecurityGroupRuleManager.SyncRules(ctx, userCred, new, rules) - syncResult.Add() - } - } - } - } - return localSecgroups, remoteSecgroups, syncResult -} - -func (self *SSecurityGroup) SyncWithCloudSecurityGroup(userCred mcclient.TokenCredential, extSec cloudprovider.ICloudSecurityGroup) error { +func (self *SSecurityGroup) SyncWithCloudSecurityGroup(userCred mcclient.TokenCredential, extSec cloudprovider.ICloudSecurityGroup, vpc *SVpc) error { if _, err := self.GetModelManager().TableSpec().Update(self, func() error { extSec.Refresh() self.Name = extSec.GetName() @@ -262,10 +206,17 @@ func (self *SSecurityGroup) SyncWithCloudSecurityGroup(userCred mcclient.TokenCr log.Errorf("syncWithCloudSecurityGroup error %s", err) return err } + + if secgroupcache := SecurityGroupCacheManager.Register(context.Background(), userCred, self.Id, extSec.GetVpcId(), vpc.CloudregionId, vpc.ManagerId); secgroupcache != nil { + if err := secgroupcache.SetExternalId(self.ExternalId); err != nil { + log.Errorf("set secgroupcache %s externalId error: %v", secgroupcache.Id, err) + } + } + return nil } -func (manager *SSecurityGroupManager) newFromCloudVpc(userCred mcclient.TokenCredential, extSec cloudprovider.ICloudSecurityGroup, regionId, providerId string) (*SSecurityGroup, error) { +func (manager *SSecurityGroupManager) newFromCloudVpc(userCred mcclient.TokenCredential, extSec cloudprovider.ICloudSecurityGroup, vpc *SVpc) (*SSecurityGroup, error) { secgroup := SSecurityGroup{} secgroup.SetModelManager(manager) secgroup.Name = extSec.GetName() @@ -277,19 +228,69 @@ func (manager *SSecurityGroupManager) newFromCloudVpc(userCred mcclient.TokenCre return nil, err } - secgroupcache := SSecurityGroupCache{} - secgroupcache.ExternalId = secgroup.ExternalId - secgroupcache.SecgroupId = secgroup.Id - secgroupcache.CloudregionId = regionId - secgroupcache.ManagerId = providerId - - if err := SecurityGroupCacheManager.TableSpec().Insert(&secgroupcache); err != nil { - return nil, err + if secgroupcache := SecurityGroupCacheManager.Register(context.Background(), userCred, secgroup.Id, extSec.GetVpcId(), vpc.CloudregionId, vpc.ManagerId); secgroupcache != nil { + if err := secgroupcache.SetExternalId(secgroup.ExternalId); err != nil { + log.Errorf("set secgroupcache %s externalId error: %v", secgroupcache.Id, err) + } } return &secgroup, nil } +func (manager *SSecurityGroupManager) SyncSecgroups(ctx context.Context, userCred mcclient.TokenCredential, secgroups []cloudprovider.ICloudSecurityGroup, vpc *SVpc) ([]SSecurityGroup, []cloudprovider.ICloudSecurityGroup, compare.SyncResult) { + localSecgroups := make([]SSecurityGroup, 0) + remoteSecgroups := make([]cloudprovider.ICloudSecurityGroup, 0) + syncResult := compare.SyncResult{} + + dbSecgroups, err := manager.getSecurityGroups() + if err != nil { + syncResult.Error(err) + return nil, nil, syncResult + } + removed := make([]SSecurityGroup, 0) + commondb := make([]SSecurityGroup, 0) + commonext := make([]cloudprovider.ICloudSecurityGroup, 0) + added := make([]cloudprovider.ICloudSecurityGroup, 0) + if err := compare.CompareSets(dbSecgroups, secgroups, &removed, &commondb, &commonext, &added); err != nil { + syncResult.Error(err) + return nil, nil, syncResult + } + + for i := 0; i < len(commondb); i += 1 { + rules, err := commonext[i].GetRules() + if err != nil { + syncResult.Error(err) + continue + } + if err := commondb[i].SyncWithCloudSecurityGroup(userCred, commonext[i], vpc); err != nil { + syncResult.UpdateError(err) + continue + } + localSecgroups = append(localSecgroups, commondb[i]) + remoteSecgroups = append(remoteSecgroups, commonext[i]) + SecurityGroupRuleManager.SyncRules(ctx, userCred, &commondb[i], rules) + syncResult.Update() + } + + for i := 0; i < len(added); i += 1 { + rules, err := added[i].GetRules() + if err != nil { + syncResult.AddError(err) + continue + } + new, err := manager.newFromCloudVpc(userCred, added[i], vpc) + if err != nil { + syncResult.AddError(err) + continue + } + localSecgroups = append(localSecgroups, *new) + remoteSecgroups = append(remoteSecgroups, added[i]) + SecurityGroupRuleManager.SyncRules(ctx, userCred, new, rules) + syncResult.Add() + } + return localSecgroups, remoteSecgroups, syncResult +} + func (manager *SSecurityGroupManager) DelaySync(ctx context.Context, userCred mcclient.TokenCredential, idStr string) { if secgrp := manager.FetchSecgroupById(idStr); secgrp == nil { log.Errorf("DelaySync secgroup failed") @@ -391,3 +392,35 @@ func (self *SSecurityGroup) ValidateDeleteCondition(ctx context.Context) error { } return self.SSharableVirtualResourceBase.ValidateDeleteCondition(ctx) } + +func (self *SSecurityGroup) GetSecurityGroupCaches() []SSecurityGroupCache { + caches := []SSecurityGroupCache{} + q := SecurityGroupCacheManager.Query() + q = q.Filter(sqlchemy.Equals(q.Field("secgroup_id"), self.Id)) + if err := db.FetchModelObjects(SecurityGroupCacheManager, q, &caches); err != nil { + log.Errorf("get secgroupcache for secgroup %s error: %v", self.Name, err) + } + return caches +} + +func (self *SSecurityGroup) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { + return self.StartDeleteSecurityGroupTask(ctx, userCred, jsonutils.NewDict(), "") +} + +func (self *SSecurityGroup) StartDeleteSecurityGroupTask(ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, parentTaskId string) error { + task, err := taskman.TaskManager.NewTask(ctx, "SecurityGroupDeleteTask", self, userCred, params, parentTaskId, "", nil) + if err != nil { + return err + } + task.ScheduleRun(nil) + return nil +} + +func (self *SSecurityGroup) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + log.Infof("SecurityGroup delete do nothing") + return nil +} + +func (self *SSecurityGroup) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error { + return self.SVirtualResourceBase.Delete(ctx, userCred) +} diff --git a/pkg/compute/models/snapshots.go b/pkg/compute/models/snapshots.go index 491381fe77..367f05bb71 100644 --- a/pkg/compute/models/snapshots.go +++ b/pkg/compute/models/snapshots.go @@ -273,6 +273,7 @@ func (self *SSnapshotManager) CreateSnapshot(ctx context.Context, userCred mccli return nil, err } disk := iDisk.(*SDisk) + storage := disk.GetStorage() snapshot := &SSnapshot{} snapshot.SetModelManager(self) snapshot.ProjectId = userCred.GetProjectId() @@ -282,6 +283,8 @@ func (self *SSnapshotManager) CreateSnapshot(ctx context.Context, userCred mccli snapshot.DiskType = disk.DiskType snapshot.Location = location snapshot.CreatedBy = createdBy + snapshot.ManagerId = storage.ManagerId + snapshot.CloudregionId = storage.getZone().GetRegion().GetId() snapshot.Name = name snapshot.Status = SNAPSHOT_CREATING err = SnapshotManager.TableSpec().Insert(snapshot) @@ -437,10 +440,12 @@ func totalSnapshotCount(projectId string) int { return count } -func (self *SSnapshot) SyncWithCloudSnapshot(userCred mcclient.TokenCredential, ext cloudprovider.ICloudSnapshot) error { +func (self *SSnapshot) SyncWithCloudSnapshot(userCred mcclient.TokenCredential, ext cloudprovider.ICloudSnapshot, region *SCloudregion) error { _, err := self.GetModelManager().TableSpec().Update(self, func() error { + self.Name = ext.GetName() self.Status = ext.GetStatus() self.DiskType = ext.GetDiskType() + self.CloudregionId = region.Id return nil }) if err != nil { @@ -449,7 +454,7 @@ func (self *SSnapshot) SyncWithCloudSnapshot(userCred mcclient.TokenCredential, return err } -func (manager *SSnapshotManager) newFromCloudSnapshot(userCred mcclient.TokenCredential, extSnapshot cloudprovider.ICloudSnapshot, region *SCloudregion) (*SSnapshot, error) { +func (manager *SSnapshotManager) newFromCloudSnapshot(userCred mcclient.TokenCredential, extSnapshot cloudprovider.ICloudSnapshot, region *SCloudregion, provider *SCloudprovider) (*SSnapshot, error) { snapshot := SSnapshot{} snapshot.SetModelManager(manager) @@ -467,7 +472,7 @@ func (manager *SSnapshotManager) newFromCloudSnapshot(userCred mcclient.TokenCre snapshot.DiskType = extSnapshot.GetDiskType() snapshot.Size = int(extSnapshot.GetSize()) * 1024 - snapshot.ManagerId = extSnapshot.GetManagerId() + snapshot.ManagerId = provider.Id snapshot.CloudregionId = region.Id snapshot.ProjectId = userCred.GetProjectId() @@ -518,7 +523,7 @@ func (manager *SSnapshotManager) SyncSnapshots(ctx context.Context, userCred mcc } } for i := 0; i < len(commondb); i += 1 { - err = commondb[i].SyncWithCloudSnapshot(userCred, commonext[i]) + err = commondb[i].SyncWithCloudSnapshot(userCred, commonext[i], region) if err != nil { syncResult.UpdateError(err) } else { @@ -526,7 +531,7 @@ func (manager *SSnapshotManager) SyncSnapshots(ctx context.Context, userCred mcc } } for i := 0; i < len(added); i += 1 { - _, err := manager.newFromCloudSnapshot(userCred, added[i], region) + _, err := manager.newFromCloudSnapshot(userCred, added[i], region, provider) if err != nil { syncResult.AddError(err) } else { diff --git a/pkg/compute/models/vpcs.go b/pkg/compute/models/vpcs.go index 2ec8e1bd24..639cbd6f10 100644 --- a/pkg/compute/models/vpcs.go +++ b/pkg/compute/models/vpcs.go @@ -344,10 +344,6 @@ func (manager *SVpcManager) ValidateCreateData(ctx context.Context, userCred mcc return nil, httperrors.NewInputParameterError("Invalid cloudregion_id") } if region.isManaged() { - if region.GetVpcCount() >= MAX_VPC_PER_REGION { - return nil, httperrors.NewNotAcceptableError("Too many vpcs per region") - } - managerStr, _ := data.GetString("manager_id") if len(managerStr) == 0 { managerStr, _ = data.GetString("manager") diff --git a/pkg/compute/tasks/cloud_provider_sync_info_task.go b/pkg/compute/tasks/cloud_provider_sync_info_task.go index 89850eb49a..eb0db47ca0 100644 --- a/pkg/compute/tasks/cloud_provider_sync_info_task.go +++ b/pkg/compute/tasks/cloud_provider_sync_info_task.go @@ -219,7 +219,7 @@ func syncVpcSecGroup(ctx context.Context, provider *models.SCloudprovider, task logSyncFailed(provider, task, msg) return } else { - _, _, result := models.SecurityGroupManager.SyncSecgroups(ctx, task.UserCred, secgroups, localVpc.CloudregionId, provider.Id) + _, _, result := models.SecurityGroupManager.SyncSecgroups(ctx, task.UserCred, secgroups, localVpc) msg := result.Result() notes := fmt.Sprintf("SyncSecurityGroup for VPC %s result: %s", localVpc.Name, msg) log.Infof(notes) diff --git a/pkg/compute/tasks/guest_disk_snapshot_task.go b/pkg/compute/tasks/guest_disk_snapshot_task.go index 9c9e7aa7ea..4f5f230ef2 100644 --- a/pkg/compute/tasks/guest_disk_snapshot_task.go +++ b/pkg/compute/tasks/guest_disk_snapshot_task.go @@ -52,32 +52,25 @@ func (self *GuestDiskSnapshotTask) DoDiskSnapshot(ctx context.Context, guest *mo func (self *GuestDiskSnapshotTask) OnDiskSnapshotComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { res := data.(*jsonutils.JSONDict) + snapshotId, _ := self.Params.GetString("snapshot_id") + iSnapshot, _ := models.SnapshotManager.FetchById(snapshotId) + snapshot := iSnapshot.(*models.SSnapshot) if guest.Hypervisor == models.HYPERVISOR_KVM { location, err := res.GetString("location") if err != nil { log.Infof("OnDiskSnapshotComplete called with data no location") return } - snapshotId, _ := self.Params.GetString("snapshot_id") - iSnapshot, _ := models.SnapshotManager.FetchById(snapshotId) - snapshot := iSnapshot.(*models.SSnapshot) models.SnapshotManager.TableSpec().Update(snapshot, func() error { snapshot.Location = location snapshot.Status = models.SNAPSHOT_READY return nil }) } else { - snapshotId, _ := self.Params.GetString("snapshot_id") - iSnapshot, _ := models.SnapshotManager.FetchById(snapshotId) - snapshot := iSnapshot.(*models.SSnapshot) extSnapshotId, _ := data.GetString("snapshot_id") - cloudregionId, _ := data.GetString("cloudregion_id") - managerId, _ := data.GetString("manager_id") models.SnapshotManager.TableSpec().Update(snapshot, func() error { - snapshot.CloudregionId = cloudregionId snapshot.ExternalId = extSnapshotId snapshot.Status = models.SNAPSHOT_READY - snapshot.ManagerId = managerId return nil }) } diff --git a/pkg/compute/tasks/security_group_delete_task.go b/pkg/compute/tasks/security_group_delete_task.go new file mode 100644 index 0000000000..54fe0fc2e2 --- /dev/null +++ b/pkg/compute/tasks/security_group_delete_task.go @@ -0,0 +1,28 @@ +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" +) + +type SecurityGroupDeleteTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(SecurityGroupDeleteTask{}) +} + +func (self *SecurityGroupDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + secgroup := obj.(*models.SSecurityGroup) + secgroupCache := secgroup.GetSecurityGroupCaches() + for _, cache := range secgroupCache { + cache.Delete(ctx, self.GetUserCred()) + } + secgroup.RealDelete(ctx, self.GetUserCred()) + self.SetStageComplete(ctx, nil) +} diff --git a/pkg/util/aliyun/host.go b/pkg/util/aliyun/host.go index 8f116d6746..6a3aa4524a 100644 --- a/pkg/util/aliyun/host.go +++ b/pkg/util/aliyun/host.go @@ -2,6 +2,7 @@ package aliyun import ( "fmt" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -195,25 +196,6 @@ func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int } var err error - - if len(secgroupId) == 0 { - secgroups, err := net.wire.vpc.GetISecurityGroups() - if err != nil { - return "", fmt.Errorf("get security group error %s", err) - } - - if len(secgroups) == 0 { - secId, err := self.zone.region.createDefaultSecurityGroup(net.wire.vpc.VpcId) - if err != nil { - return "", fmt.Errorf("no secgroup for vpc and failed to create a default One!!") - } else { - secgroupId = secId - } - } else { - secgroupId = secgroups[0].GetId() - } - } - keypair := "" if len(publicKey) > 0 { keypair, err = self.zone.region.syncKeypair(publicKey) diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index ce83715db8..f8772f60ef 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -752,6 +752,10 @@ func (self *SInstance) GetIEIP() (cloudprovider.ICloudEIP, error) { } } +func (self *SInstance) AssignSecurityGroup(secgroupId string) error { + return self.host.zone.region.AssignSecurityGroup(secgroupId, self.InstanceId) +} + func (self *SInstance) GetBillingType() string { switch self.InstanceChargeType { case PrePaidInstanceChargeType: @@ -766,11 +770,3 @@ func (self *SInstance) GetBillingType() string { func (self *SInstance) GetExpiredAt() time.Time { return self.ExpiredTime } - -func (self *SInstance) AssignSecurityGroup(secgroupId string) error { - return cloudprovider.ErrNotImplemented -} - -func (self *SInstance) RevokeSecurityGroup() error { - return cloudprovider.ErrNotImplemented -} diff --git a/pkg/util/aliyun/region.go b/pkg/util/aliyun/region.go index b15757b9c6..346a8b3cad 100644 --- a/pkg/util/aliyun/region.go +++ b/pkg/util/aliyun/region.go @@ -637,12 +637,21 @@ func (self *SRegion) GetIEipById(eipId string) (cloudprovider.ICloudEIP, error) } func (region *SRegion) SyncSecurityGroup(secgroupId string, vpcId string, name string, desc string, rules []secrules.SecurityRule) (string, error) { - if len(secgroupId) == 0 { - extId, err := region.CreateSecurityGroup(vpcId, name, desc) + if len(secgroupId) > 0 { + _, total, err := region.GetSecurityGroups("", []string{secgroupId}, 0, 1) if err != nil { return "", err } - secgroupId = extId + if total == 0 { + secgroupId = "" + } } - return secgroupId, cloudprovider.ErrNotImplemented + if len(secgroupId) == 0 { + extID, err := region.CreateSecurityGroup(vpcId, name, desc) + if err != nil { + return "", err + } + secgroupId = extID + } + return secgroupId, region.syncSecgroupRules(secgroupId, rules) } diff --git a/pkg/util/aliyun/securitygroup.go b/pkg/util/aliyun/securitygroup.go index b24f976640..d49def0ec0 100644 --- a/pkg/util/aliyun/securitygroup.go +++ b/pkg/util/aliyun/securitygroup.go @@ -8,7 +8,6 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/pkg/util/secrules" "yunion.io/x/pkg/utils" ) @@ -87,6 +86,10 @@ func (v PermissionSet) Less(i, j int) bool { return false } +func (self *SSecurityGroup) GetVpcId() string { + return self.VpcId +} + func (self *SSecurityGroup) GetMetadata() *jsonutils.JSONDict { if len(self.Tags.Tag) == 0 { return nil @@ -155,7 +158,7 @@ func (self *SSecurityGroup) Refresh() error { } } -func (self *SRegion) GetSecurityGroups(vpcId string, offset int, limit int) ([]SSecurityGroup, int, error) { +func (self *SRegion) GetSecurityGroups(vpcId string, securityGroupIds []string, offset int, limit int) ([]SSecurityGroup, int, error) { if limit > 50 || limit <= 0 { limit = 50 } @@ -167,6 +170,10 @@ func (self *SRegion) GetSecurityGroups(vpcId string, offset int, limit int) ([]S params["VpcId"] = vpcId } + if securityGroupIds != nil && len(securityGroupIds) > 0 { + params["SecurityGroupIds"] = jsonutils.Marshal(securityGroupIds).String() + } + body, err := self.ecsRequest("DescribeSecurityGroups", params) if err != nil { log.Errorf("GetSecurityGroups fail %s", err) @@ -209,6 +216,11 @@ func (self *SRegion) CreateSecurityGroup(vpcId string, name string, desc string) if len(vpcId) > 0 { params["VpcId"] = vpcId } + + if name == "Default" { + name = "Default-copy" + } + if len(name) > 0 { params["SecurityGroupName"] = name } @@ -383,58 +395,6 @@ func (self *SRegion) delSecurityGroupRule(secGrpId string, rule *secrules.Securi } } -func (self *SRegion) createDefaultSecurityGroup(vpcId string) (string, error) { - secId, err := self.CreateSecurityGroup(vpcId, "", "") - if err != nil { - return "", err - } - inRule := secrules.SecurityRule{ - Priority: 1, - Action: secrules.SecurityRuleAllow, - Protocol: "", - Direction: secrules.SecurityRuleIngress, - PortStart: -1, - PortEnd: -1, - } - err = self.addSecurityGroupRules(secId, &inRule) - if err != nil { - return "", err - } - outRule := secrules.SecurityRule{ - Priority: 1, - Action: secrules.SecurityRuleAllow, - Protocol: "", - Direction: secrules.SecurityRuleEgress, - PortStart: -1, - PortEnd: -1, - } - err = self.addSecurityGroupRules(secId, &outRule) - if err != nil { - return "", err - } - return secId, nil -} - -func (self *SRegion) getSecurityGroupByTag(vpcId, secgroupId string) (*SSecurityGroup, error) { - params := make(map[string]string) - params["RegionId"] = self.RegionId - if len(vpcId) > 0 { - params["VpcId"] = vpcId - } - params["Tag.1.Key"] = "id" - params["Tag.1.Value"] = secgroupId - - secgrps := make([]SSecurityGroup, 0) - if body, err := self.ecsRequest("DescribeSecurityGroups", params); err != nil { - return nil, err - } else if err := body.Unmarshal(&secgrps, "SecurityGroups", "SecurityGroup"); err != nil { - return nil, err - } else if len(secgrps) != 1 { - return nil, httperrors.NewNotFoundError("failed to find SecurityGroup %s", secgroupId) - } - return &secgrps[0], nil -} - func (self *SPermission) String() string { action := secrules.SecurityRuleDeny if strings.ToLower(self.Policy) == "accept" { @@ -474,50 +434,6 @@ func (self *SPermission) String() string { return result } -func (self *SRegion) addTagToSecurityGroup(secgroupId, key, value string, index int) error { - if index > 5 || index < 1 { - index = 1 - } - params := map[string]string{"ResourceType": "securitygroup", "ResourceId": secgroupId} - params[fmt.Sprintf("Tag.%d.Key", index)] = key - params[fmt.Sprintf("Tag.%d.Value", index)] = value - _, err := self.ecsRequest("AddTags", params) - return err -} - -func (self *SRegion) revokeSecurityGroup(secgroupId, instanceId string, keep bool) error { - if !keep { - return self.leaveSecurityGroup(secgroupId, instanceId) - } - if secgroup, err := self.GetSecurityGroupDetails(secgroupId); err != nil { - return err - } else { - for _, permission := range secgroup.Permissions.Permission { - if rule, err := secrules.ParseSecurityRule(permission.String()); err != nil { - return err - } else { - rule.Priority = permission.Priority - if err := self.delSecurityGroupRule(secgroup.SecurityGroupId, rule); err != nil { - return err - } - } - } - if rule, err := secrules.ParseSecurityRule("in:allow any"); err != nil { - rule.Priority = 100 - if err := self.addSecurityGroupRules(secgroup.SecurityGroupId, rule); err != nil { - return err - } - } - if rule, err := secrules.ParseSecurityRule("out:allow any"); err != nil { - rule.Priority = 100 - if err := self.addSecurityGroupRules(secgroup.SecurityGroupId, rule); err != nil { - return err - } - } - } - return nil -} - func (self *SRegion) syncSecgroupRules(secgroupId string, rules []secrules.SecurityRule) error { if secgroup, err := self.GetSecurityGroupDetails(secgroupId); err != nil { return err @@ -584,10 +500,23 @@ func (self *SRegion) syncSecgroupRules(secgroupId string, rules []secrules.Secur return nil } -func (self *SRegion) assignSecurityGroup(secgroupId, instanceId string) error { +func (self *SRegion) AssignSecurityGroup(secgroupId, instanceId string) error { params := map[string]string{"InstanceId": instanceId, "SecurityGroupId": secgroupId} - _, err := self.ecsRequest("JoinSecurityGroup", params) - return err + if _, err := self.ecsRequest("JoinSecurityGroup", params); err != nil { + return err + } + instance, err := self.GetInstance(instanceId) + if err != nil { + return err + } + for _, _secgroupId := range instance.SecurityGroupIds.SecurityGroupId { + if _secgroupId != secgroupId { + if err := self.leaveSecurityGroup(_secgroupId, instanceId); err != nil { + return err + } + } + } + return nil } func (self *SRegion) leaveSecurityGroup(secgroupId, instanceId string) error { @@ -596,7 +525,7 @@ func (self *SRegion) leaveSecurityGroup(secgroupId, instanceId string) error { return err } -func (self *SRegion) deleteSecurityGroup(secGrpId string) error { +func (self *SRegion) DeleteSecurityGroup(vpcId, secGrpId string) error { params := make(map[string]string) params["SecurityGroupId"] = secGrpId diff --git a/pkg/util/aliyun/shell/secgroup.go b/pkg/util/aliyun/shell/secgroup.go index 5ccab97b8c..a215b87a54 100644 --- a/pkg/util/aliyun/shell/secgroup.go +++ b/pkg/util/aliyun/shell/secgroup.go @@ -9,12 +9,13 @@ import ( func init() { type SecurityGroupListOptions struct { - VpcId string `help:"VPC ID"` - Limit int `help:"page size"` - Offset int `help:"page offset"` + VpcId string `help:"VPC ID"` + SecurityGroupIds []string `help:"SecurityGroup ids"` + Limit int `help:"page size"` + Offset int `help:"page offset"` } shellutils.R(&SecurityGroupListOptions{}, "security-group-list", "List security group", func(cli *aliyun.SRegion, args *SecurityGroupListOptions) error { - secgrps, total, e := cli.GetSecurityGroups(args.VpcId, args.Offset, args.Limit) + secgrps, total, e := cli.GetSecurityGroups(args.VpcId, args.SecurityGroupIds, args.Offset, args.Limit) if e != nil { return e } diff --git a/pkg/util/aliyun/snapshot.go b/pkg/util/aliyun/snapshot.go index e0cada0372..87d9523970 100644 --- a/pkg/util/aliyun/snapshot.go +++ b/pkg/util/aliyun/snapshot.go @@ -51,14 +51,6 @@ func (self *SSnapshot) GetStatus() string { } } -func (self *SSnapshot) GetManagerId() string { - return self.region.client.providerId -} - -func (self *SSnapshot) GetRegionId() string { - return self.region.GetId() -} - func (self *SSnapshot) GetSize() int32 { return self.SourceDiskSize } diff --git a/pkg/util/aliyun/vpc.go b/pkg/util/aliyun/vpc.go index 9e36cc5d97..4ca890cbde 100644 --- a/pkg/util/aliyun/vpc.go +++ b/pkg/util/aliyun/vpc.go @@ -155,7 +155,7 @@ func (self *SVpc) GetIWireById(wireId string) (cloudprovider.ICloudWire, error) func (self *SVpc) fetchSecurityGroups() error { secgroups := make([]SSecurityGroup, 0) for { - parts, total, err := self.region.GetSecurityGroups(self.VpcId, len(secgroups), 50) + parts, total, err := self.region.GetSecurityGroups(self.VpcId, []string{}, len(secgroups), 50) if err != nil { return err } @@ -194,7 +194,7 @@ func (self *SVpc) Delete() error { } for i := 0; i < len(self.secgroups); i += 1 { secgroup := self.secgroups[i].(*SSecurityGroup) - err := self.region.deleteSecurityGroup(secgroup.SecurityGroupId) + err := self.region.DeleteSecurityGroup(self.VpcId, secgroup.SecurityGroupId) if err != nil { log.Errorf("deleteSecurityGroup for VPC delete fail %s", err) return err @@ -202,11 +202,3 @@ func (self *SVpc) Delete() error { } return self.region.DeleteVpc(self.VpcId) } - -func (self *SVpc) assignSecurityGroup(secgroupId string, instanceId string) error { - return self.region.assignSecurityGroup(secgroupId, instanceId) -} - -func (self *SVpc) revokeSecurityGroup(secgroupId string, instanceId string, keep bool) error { - return self.region.revokeSecurityGroup(secgroupId, instanceId, keep) -} diff --git a/pkg/util/azure/azure.go b/pkg/util/azure/azure.go index 651a848e3f..4fdd336c28 100644 --- a/pkg/util/azure/azure.go +++ b/pkg/util/azure/azure.go @@ -4,6 +4,7 @@ import ( "fmt" "io/ioutil" "net/http" + "strconv" "strings" "time" @@ -65,6 +66,7 @@ var DEFAULT_API_VERSION = map[string]string{ "Microsoft.Network": "2018-06-01", "Microsoft.ClassicNetwork/reservedIps": "2016-04-01", //2014-01-01,2014-06-01,2015-06-01,2015-12-01,2016-04-01,2016-11-01 "Microsoft.ClassicNetwork/networkSecurityGroups": "2016-11-01", //2015-06-01,2015-12-01,2016-04-01,2016-11-01 + "Microsoft.ClassicCompute/domainNames": "2015-12-01", //2014-01-01, 2014-06-01, 2015-06-01, 2015-10-01, 2015-12-01, 2016-04-01, 2016-11-01, 2017-11-01, 2017-11-15 } func NewAzureClient(providerId string, providerName string, accessKey string, secret string, envName string) (*SAzureClient, error) { @@ -100,8 +102,10 @@ func (self *SAzureClient) getDefaultClient() (*autorest.Client, error) { return nil, err } client.Authorizer = authorizer - // client.RequestInspector = LogRequest() - // client.ResponseInspector = LogResponse() + if DEBUG { + client.RequestInspector = LogRequest() + client.ResponseInspector = LogResponse() + } return &client, nil } @@ -110,7 +114,7 @@ func (self *SAzureClient) jsonRequest(method, url string, body string) (jsonutil if err != nil { return nil, err } - return jsonRequest(cli, method, self.domain, url, body) + return jsonRequest(cli, method, self.domain, url, self.subscriptionId, body) } func (self *SAzureClient) Get(resourceId string, params []string, retVal interface{}) error { @@ -125,7 +129,7 @@ func (self *SAzureClient) Get(resourceId string, params []string, retVal interfa if err != nil { return err } - body, err := jsonRequest(cli, "GET", self.domain, path, "") + body, err := jsonRequest(cli, "GET", self.domain, path, self.subscriptionId, "") if err != nil { return err } @@ -145,7 +149,7 @@ func (self *SAzureClient) ListVmSizes(location string) (jsonutils.JSONObject, er return nil, fmt.Errorf("need subscription id") } url := fmt.Sprintf("/subscriptions/%s/providers/Microsoft.Compute/locations/%s/vmSizes", self.subscriptionId, location) - return jsonRequest(cli, "GET", self.domain, url, "") + return jsonRequest(cli, "GET", self.domain, url, self.subscriptionId, "") } func (self *SAzureClient) ListClassicDisks() (jsonutils.JSONObject, error) { @@ -157,7 +161,7 @@ func (self *SAzureClient) ListClassicDisks() (jsonutils.JSONObject, error) { return nil, fmt.Errorf("need subscription id") } url := fmt.Sprintf("/subscriptions/%s/services/disks", self.subscriptionId) - return jsonRequest(cli, "GET", self.domain, url, "") + return jsonRequest(cli, "GET", self.domain, url, self.subscriptionId, "") } func (self *SAzureClient) ListAll(resourceType string, retVal interface{}) error { @@ -172,7 +176,7 @@ func (self *SAzureClient) ListAll(resourceType string, retVal interface{}) error if len(resourceType) > 0 { url += fmt.Sprintf("/providers/%s", resourceType) } - body, err := jsonRequest(cli, "GET", self.domain, url, "") + body, err := jsonRequest(cli, "GET", self.domain, url, self.subscriptionId, "") if err != nil { return err } @@ -187,7 +191,7 @@ func (self *SAzureClient) ListSubscriptions() (jsonutils.JSONObject, error) { if err != nil { return nil, err } - return jsonRequest(cli, "GET", self.domain, "/subscriptions", "") + return jsonRequest(cli, "GET", self.domain, "/subscriptions", self.subscriptionId, "") } func (self *SAzureClient) List(golbalResource string, retVal interface{}) error { @@ -202,7 +206,7 @@ func (self *SAzureClient) List(golbalResource string, retVal interface{}) error if len(self.subscriptionId) > 0 && len(golbalResource) > 0 { url += fmt.Sprintf("/%s", golbalResource) } - body, err := jsonRequest(cli, "GET", self.domain, url, "") + body, err := jsonRequest(cli, "GET", self.domain, url, self.subscriptionId, "") if err != nil { return err } @@ -218,7 +222,7 @@ func (self *SAzureClient) ListByTypeWithResourceGroup(resourceGroupName string, return fmt.Errorf("Missing subscription Info") } url := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/%s", self.subscriptionId, resourceGroupName, Type) - body, err := jsonRequest(cli, "GET", self.domain, url, "") + body, err := jsonRequest(cli, "GET", self.domain, url, self.subscriptionId, "") if err != nil { return err } @@ -230,7 +234,7 @@ func (self *SAzureClient) Delete(resourceId string) error { if err != nil { return err } - _, err = jsonRequest(cli, "DELETE", self.domain, resourceId, "") + _, err = jsonRequest(cli, "DELETE", self.domain, resourceId, self.subscriptionId, "") return err } @@ -240,7 +244,7 @@ func (self *SAzureClient) PerformAction(resourceId string, action string, body s return nil, err } url := fmt.Sprintf("%s/%s", resourceId, action) - return jsonRequest(cli, "POST", self.domain, url, body) + return jsonRequest(cli, "POST", self.domain, url, self.subscriptionId, body) } func (self *SAzureClient) fetchResourceGroup(cli *autorest.Client, location string) error { @@ -255,7 +259,7 @@ func (self *SAzureClient) fetchResourceGroup(cli *autorest.Client, location stri if len(self.ressourceGroups) == 0 { //Create Default resourceGroup _url := fmt.Sprintf("/subscriptions/%s/resourcegroups/Default", self.subscriptionId) - body, err := jsonRequest(cli, "PUT", self.domain, _url, fmt.Sprintf(`{"name": "Default", "location": "%s"}`, location)) + body, err := jsonRequest(cli, "PUT", self.domain, _url, self.subscriptionId, fmt.Sprintf(`{"name": "Default", "location": "%s"}`, location)) if err != nil { return err } @@ -274,13 +278,47 @@ func (self *SAzureClient) checkParams(body jsonutils.JSONObject, params []string for i := 0; i < len(params); i++ { data, err := body.GetString(params[i]) if err != nil { - return nil, fmt.Errorf("Missing %s params") + return nil, fmt.Errorf("Missing %s params", params[i]) } result[params[i]] = data } return result, nil } +type AzureErrorDetail struct { + Code string `json:"code,omitempty"` + Message string `json:"message,omitempty"` + Target string `json:"target,omitempty"` +} + +type AzureError struct { + Code string `json:"code,omitempty"` + Details []AzureErrorDetail `json:"details,omitempty"` + Message string `json:"message,omitempty"` +} + +func (self *SAzureClient) getUniqName(cli *autorest.Client, resourceType, name string, body jsonutils.JSONObject) (string, string, error) { + url := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/%s/%s", self.subscriptionId, self.ressourceGroups[0].Name, resourceType, name) + if _, err := jsonRequest(cli, "GET", self.domain, url, self.subscriptionId, ""); err != nil { + if err == cloudprovider.ErrNotFound { + return url, body.String(), nil + } + return "", "", err + } + for i := 0; i < 20; i++ { + url = fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/%s/%s-%d", self.subscriptionId, self.ressourceGroups[0].Name, resourceType, name, i) + if _, err := jsonRequest(cli, "GET", self.domain, url, self.subscriptionId, ""); err == cloudprovider.ErrNotFound { + if err == cloudprovider.ErrNotFound { + data := body.(*jsonutils.JSONDict) + data.Set("name", jsonutils.NewString(fmt.Sprintf("%s-%d", name, i))) + return url, body.String(), nil + } + return "", "", err + } + } + return "", "", fmt.Errorf("not find uniq name for %s[%s]", resourceType, name) +} + func (self *SAzureClient) Create(body jsonutils.JSONObject, retVal interface{}) error { cli, err := self.getDefaultClient() if err != nil { @@ -300,12 +338,19 @@ func (self *SAzureClient) Create(body jsonutils.JSONObject, retVal interface{}) if len(self.ressourceGroups) == 0 { return fmt.Errorf("Create Default resourceGroup error?") } - url := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/%s/%s", self.subscriptionId, self.ressourceGroups[0].Name, params["type"], params["name"]) - result, err := jsonRequest(cli, "PUT", self.domain, url, body.String()) + + url, reqString, err := self.getUniqName(cli, params["type"], params["name"], body) if err != nil { return err } - return result.Unmarshal(retVal) + result, err := jsonRequest(cli, "PUT", self.domain, url, self.subscriptionId, reqString) + if err != nil { + return err + } + if retVal != nil { + return result.Unmarshal(retVal) + } + return nil } func (self *SAzureClient) CheckNameAvailability(Type string, body string) (jsonutils.JSONObject, error) { @@ -317,7 +362,7 @@ func (self *SAzureClient) CheckNameAvailability(Type string, body string) (jsonu return nil, fmt.Errorf("Missing subscription ID") } url := fmt.Sprintf("/subscriptions/%s/providers/%s/checkNameAvailability", self.subscriptionId, Type) - return jsonRequest(cli, "POST", self.domain, url, body) + return jsonRequest(cli, "POST", self.domain, url, self.subscriptionId, body) } func (self *SAzureClient) Update(body jsonutils.JSONObject, retVal interface{}) error { @@ -326,7 +371,7 @@ func (self *SAzureClient) Update(body jsonutils.JSONObject, retVal interface{}) return err } url, err := body.GetString("id") - result, err := jsonRequest(cli, "PUT", self.domain, url, body.String()) + result, err := jsonRequest(cli, "PUT", self.domain, url, self.subscriptionId, body.String()) if err != nil { return err } @@ -336,8 +381,86 @@ func (self *SAzureClient) Update(body jsonutils.JSONObject, retVal interface{}) return nil } -func jsonRequest(client *autorest.Client, method, domain, baseUrl string, body string) (jsonutils.JSONObject, error) { - return _jsonRequest(client, method, domain, baseUrl, body) +func waitRegisterComplete(client *autorest.Client, domain, subscriptionId string, serviceType string) error { + for i := 1; i < 10; i++ { + result, err := _jsonRequest(client, "GET", domain, fmt.Sprintf("/subscriptions/%s/providers", subscriptionId), "") + if err != nil { + return err + } + value, err := result.GetArray("value") + if err != nil { + return err + } + for _, v := range value { + namespace, _ := v.GetString("namespace") + if namespace == serviceType { + state, _ := v.GetString("registrationState") + if state == "Registered" { + return nil + } + log.Debugf("service %s state %s waite %d second ...", serviceType, state, i*10) + } + } + time.Sleep(time.Second * time.Duration(i*10)) + } + return fmt.Errorf("wait service %s register timeout", serviceType) +} + +func registerService(client *autorest.Client, domain, subscriptionId string, serviceType string) error { + registryUrl := fmt.Sprintf("/subscriptions/%s/providers/%s/register", subscriptionId, serviceType) + result, err := _jsonRequest(client, "POST", domain, registryUrl, "") + if err != nil || result.Contains("error") { + return fmt.Errorf("failed to register %s service", serviceType) + } + if state, _ := result.GetString("registrationState"); state == "Registered" { + return nil + } + return waitRegisterComplete(client, domain, subscriptionId, serviceType) +} + +func recoverFromError(client *autorest.Client, domain, subscriptionId string, azureErr AzureError) bool { + switch azureErr.Code { + case "SubscriptionNotRegistered": + services := []string{"Microsoft.Network"} + for _, service := range services { + if err := registerService(client, domain, subscriptionId, service); err != nil { + log.Errorf("register %s error: %v", service, err) + return false + } + } + return true + case "MissingSubscriptionRegistration": + for _, detail := range azureErr.Details { + log.Errorf("The subscription is not registered to use namespace '%s', try register it", detail.Target) + if err := registerService(client, domain, subscriptionId, detail.Target); err != nil { + log.Errorf("register %s error: %v", detail.Target, err) + return false + } + } + return true + default: + return false + } + return false +} + +func jsonRequest(client *autorest.Client, method, domain, baseUrl string, subscriptionId string, body string) (jsonutils.JSONObject, error) { + result, err := _jsonRequest(client, method, domain, baseUrl, body) + if err != nil { + return nil, err + } + if result.Contains("error") { + azureError := AzureError{} + if err := result.Unmarshal(&azureError, "error"); err != nil { + return nil, fmt.Errorf(result.String()) + } + if recoverFromError(client, domain, subscriptionId, azureError) { + return _jsonRequest(client, method, domain, baseUrl, body) + } + log.Errorf("Azure %s request: %s \nbody: %s error: %v", method, baseUrl, body, result.String()) + return nil, fmt.Errorf(result.String()) + } + return result, nil } func waitForComplatetion(client *autorest.Client, req *http.Request, resp *http.Response, timeout time.Duration) (jsonutils.JSONObject, error) { @@ -358,10 +481,19 @@ func waitForComplatetion(client *autorest.Client, req *http.Request, resp *http. return nil, err } if asyncResp.StatusCode == 202 { + if _location := asyncResp.Header.Get("Location"); len(_location) > 0 { + location = _location + } if time.Now().Sub(startTime) > timeout { return nil, fmt.Errorf("Process request %s %s timeout", req.Method, req.URL.String()) } - time.Sleep(time.Second * 5) + timeSleep := time.Second * 5 + if _timeSleep := asyncResp.Header.Get("Retry-After"); len(_timeSleep) > 0 { + if _time, err := strconv.Atoi(_timeSleep); err != nil { + timeSleep = time.Second * time.Duration(_time) + } + } + time.Sleep(timeSleep) continue } if asyncResp.ContentLength == 0 { @@ -464,15 +596,7 @@ func _jsonRequest(client *autorest.Client, method, domain, baseURL, body string) return nil, err } _data := strings.Replace(string(data), "\r", "", -1) - result, err = jsonutils.Parse([]byte(_data)) - if err != nil { - return nil, err - } - if result.Contains("error") { - log.Errorf("Azure %s request: %s \nbody: %s error: %v", req.Method, req.URL.String(), body, result.String()) - return nil, fmt.Errorf(result.String()) - } - return result, nil + return jsonutils.Parse([]byte(_data)) } func (self *SAzureClient) UpdateAccount(tenantId, secret, envName string) error { diff --git a/pkg/util/azure/classic_disk.go b/pkg/util/azure/classic_disk.go index 8946493c8e..dd11cf71b7 100644 --- a/pkg/util/azure/classic_disk.go +++ b/pkg/util/azure/classic_disk.go @@ -101,7 +101,9 @@ func (self *SRegion) GetClassicDisks() ([]SClassicDisk, error) { } func (self *SClassicDisk) GetMetadata() *jsonutils.JSONDict { - return nil + data := jsonutils.NewDict() + data.Add(jsonutils.NewString(models.HYPERVISOR_AZURE), "hypervisor") + return data } func (self *SClassicDisk) CreateISnapshot(name, desc string) (cloudprovider.ICloudSnapshot, error) { diff --git a/pkg/util/azure/classic_host.go b/pkg/util/azure/classic_host.go index e756d8788d..d20ed16f4e 100644 --- a/pkg/util/azure/classic_host.go +++ b/pkg/util/azure/classic_host.go @@ -26,7 +26,7 @@ func (self *SClassicHost) GetId() string { } func (self *SClassicHost) GetName() string { - return fmt.Sprintf("%s-classic", self.zone.region.client.subscriptionId) + return fmt.Sprintf("%s/%s-classic", self.zone.region.GetGlobalId(), self.zone.region.client.subscriptionId) } func (self *SClassicHost) GetGlobalId() string { @@ -73,7 +73,7 @@ func (self *SClassicHost) GetMemSizeMB() int { return 0 } func (self *SClassicHost) GetEnabled() bool { - return false + return true } func (self *SClassicHost) GetHostStatus() string { diff --git a/pkg/util/azure/classic_instance.go b/pkg/util/azure/classic_instance.go index 47ff97c309..b0c9bc118f 100644 --- a/pkg/util/azure/classic_instance.go +++ b/pkg/util/azure/classic_instance.go @@ -12,15 +12,28 @@ import ( "yunion.io/x/pkg/util/osprofile" ) +type FormattedMessage struct { + Language string + Message string +} + +type GuestAgentStatus struct { + ProtocolVersion string `json:"protocolVersion,omitempty"` + Timestamp time.Time `json:"timestamp,omitempty"` + GuestAgentVersion string `json:"guestAgentVersion,omitempty"` + Status string `json:"status,omitempty"` + FormattedMessage FormattedMessage `json:"formattedMessage,omitempty"` +} + type ClassicVirtualMachineInstanceView struct { Status string `json:"status,omitempty"` PowerState string `json:"powerState,omitempty"` PublicIpAddresses []string `json:"publicIpAddresses,omitempty"` FullyQualifiedDomainName string `json:"fullyQualifiedDomainName,omitempty"` - UpdateDomain int - FaultDomain int - StatusMessage string + UpdateDomain int `json:"updateDomain,omitempty"` + FaultDomain int `json:"faultDomain,omitempty"` + StatusMessage string `json:"statusMessage,omitempty"` PrivateIpAddress string `json:"privateIpAddress,omitempty"` InstanceIpAddresses []string `json:"instanceIpAddresses,omitempty"` ComputerName string `json:"computerName,omitempty"` @@ -28,9 +41,9 @@ type ClassicVirtualMachineInstanceView struct { } type SubResource struct { - ID string - Name string - Type string + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` } type ClassicDisk struct { @@ -92,6 +105,7 @@ type ClassicNetworkProfile struct { } type ClassicVirtualMachineProperties struct { + DomainName *SubResource `json:"domainName,omitempty"` InstanceView *ClassicVirtualMachineInstanceView `json:"instanceView,omitempty"` NetworkProfile ClassicNetworkProfile `json:"networkProfile,omitempty"` HardwareProfile ClassicHardwareProfile `json:"hardwareProfile,omitempty"` @@ -304,6 +318,12 @@ func (self *SClassicInstance) DeleteVM() error { if err := self.host.zone.region.DeleteVM(self.ID); err != nil { return err } + if self.Properties.NetworkProfile.NetworkSecurityGroup != nil { + self.host.zone.region.client.Delete(self.Properties.NetworkProfile.NetworkSecurityGroup.ID) + } + if self.Properties.DomainName != nil { + self.host.zone.region.client.Delete(self.Properties.DomainName.ID) + } return nil } @@ -444,6 +464,42 @@ func (self *SClassicInstance) GetIEIP() (cloudprovider.ICloudEIP, error) { return nil, nil } +type assignSecurityGroup struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Properties assignProperties `json:"properties,omitempty"` + Type string `json:"type,omitempty"` +} + +type assignProperties struct { + NetworkSecurityGroup SubResource `json:"networkSecurityGroup,omitempty"` +} + +func (self *SClassicInstance) AssignSecurityGroup(secgroupId string) error { + if self.Properties.NetworkProfile.NetworkSecurityGroup != nil { + if self.Properties.NetworkProfile.NetworkSecurityGroup.ID == secgroupId { + return nil + } + self.host.zone.region.client.Delete(fmt.Sprintf("%s/associatedNetworkSecurityGroups/%s", self.ID, self.Properties.NetworkProfile.NetworkSecurityGroup.Name)) + } + + secgroup, err := self.host.zone.region.GetClassicSecurityGroupDetails(secgroupId) + if err != nil { + return err + } + data := assignSecurityGroup{ + ID: fmt.Sprintf("%s/associatedNetworkSecurityGroups/%s", self.ID, secgroup.Name), + Name: secgroup.Name, + Properties: assignProperties{ + NetworkSecurityGroup: SubResource{ + ID: secgroup.ID, + Name: secgroup.Name, + }, + }, + } + return self.host.zone.region.client.Update(jsonutils.Marshal(data), nil) +} + func (self *SClassicInstance) GetBillingType() string { return models.BILLING_TYPE_POSTPAID } @@ -451,11 +507,3 @@ func (self *SClassicInstance) GetBillingType() string { func (self *SClassicInstance) GetExpiredAt() time.Time { return time.Now() } - -func (self *SClassicInstance) AssignSecurityGroup(secgroupId string) error { - return cloudprovider.ErrNotImplemented -} - -func (self *SClassicInstance) RevokeSecurityGroup() error { - return cloudprovider.ErrNotImplemented -} diff --git a/pkg/util/azure/classic_secruitygroup.go b/pkg/util/azure/classic_secruitygroup.go index f52a6ea3c2..54baefd144 100644 --- a/pkg/util/azure/classic_secruitygroup.go +++ b/pkg/util/azure/classic_secruitygroup.go @@ -16,7 +16,7 @@ import ( type SClassicSecurityGroup struct { vpc *SClassicVpc - Properties *SecurityGroupPropertiesFormat `json:"properties,omitempty"` + Properties ClassicSecurityGroupProperties `json:"properties,omitempty"` ID string Name string Location string @@ -24,17 +24,22 @@ type SClassicSecurityGroup struct { Tags map[string]string } +type ClassicSecurityGroupProperties struct { + NetworkSecurityGroupId string `json:"networkSecurityGroupId,omitempty"` + State string `json:"state,omitempty"` +} + type ClassicSecurityGroupRuleProperties struct { - State string - Protocol string - SourcePortRange string - DestinationPortRange string - SourceAddressPrefix string - DestinationAddressPrefix string - Action string - Priority uint32 - Type string - IsDefault bool + State string `json:"state,omitempty"` + Protocol string `json:"protocol,omitempty"` + SourcePortRange string `json:"sourcePortRange,omitempty"` + DestinationPortRange string `json:"destinationPortRange,omitempty"` + SourceAddressPrefix string `json:"sourceAddressPrefix,omitempty"` + DestinationAddressPrefix string `json:"destinationAddressPrefix,omitempty"` + Action string `json:"action,omitempty"` + Priority int32 `json:"priority,omitempty"` + Type string `json:"type,omitempty"` + IsDefault bool `json:"isDefault,omitempty"` } type SClassicSecurityGroupRule struct { @@ -125,6 +130,10 @@ func (self *ClassicSecurityGroupRuleProperties) String() string { return strings.Join(result, ";") } +func (self *SClassicSecurityGroup) GetVpcId() string { + return "classic" +} + func (self *SClassicSecurityGroup) GetMetadata() *jsonutils.JSONDict { if len(self.Tags) == 0 { return nil @@ -151,12 +160,7 @@ func (self *SClassicSecurityGroup) GetName() string { func (self *SClassicSecurityGroup) GetRules() ([]secrules.SecurityRule, error) { rules := make([]secrules.SecurityRule, 0) - secgrouprules := []SClassicSecurityGroupRule{} - body, err := self.vpc.region.client.jsonRequest("GET", fmt.Sprintf("%s/securityRules", self.ID), "") - if err != nil { - return nil, err - } - err = body.Unmarshal(&secgrouprules, "value") + secgrouprules, err := self.vpc.region.getClassicSecurityGroupRules(self.ID) if err != nil { return nil, err } @@ -193,8 +197,16 @@ func (self *SClassicSecurityGroup) IsEmulated() bool { return false } -func (region *SRegion) CreateClassicSecurityGroup(secName, tagId string) (*SClassicSecurityGroup, error) { - return nil, cloudprovider.ErrNotImplemented +func (region *SRegion) CreateClassicSecurityGroup(name string) (*SClassicSecurityGroup, error) { + if name == "Default" { + name = "Default-copy" + } + secgroup := SClassicSecurityGroup{ + Name: name, + Type: "Microsoft.ClassicNetwork/networkSecurityGroups", + Location: region.Name, + } + return &secgroup, region.client.Create(jsonutils.Marshal(secgroup), &secgroup) } func (region *SRegion) GetClassicSecurityGroups() ([]SClassicSecurityGroup, error) { @@ -217,6 +229,10 @@ func (region *SRegion) GetClassicSecurityGroupDetails(secgroupId string) (*SClas return &secgroup, region.client.Get(secgroupId, []string{}, &secgroup) } +func (region *SRegion) deleteClassicSecurityGroup(secgroupId string) error { + return region.client.Delete(secgroupId) +} + func (self *SClassicSecurityGroup) Refresh() error { sec, err := self.vpc.region.GetClassicSecurityGroupDetails(self.ID) if err != nil { @@ -225,102 +241,120 @@ func (self *SClassicSecurityGroup) Refresh() error { return jsonutils.Update(self, sec) } -func (region *SRegion) checkClassicSecurityGroup(tagId, name string) (*SClassicSecurityGroup, error) { - secgroups, err := region.GetClassicSecurityGroups() +func convertClassicSecurityGroupRules(rule secrules.SecurityRule, priority int32) ([]SClassicSecurityGroupRule, error) { + name := strings.Replace(rule.String(), ":", "_", -1) + name = strings.Replace(name, " ", "_", -1) + name = strings.Replace(name, "-", "_", -1) + name = strings.Replace(name, "/", "_", -1) + name = fmt.Sprintf("%s_%d", name, rule.Priority) + rules := []SClassicSecurityGroupRule{} + secRule := SClassicSecurityGroupRule{ + Name: name, + Properties: ClassicSecurityGroupRuleProperties{ + Action: utils.Capitalize(string(rule.Action)), + Priority: priority, + Type: utils.Capitalize(string(rule.Direction)) + "bound", + Protocol: utils.Capitalize(rule.Protocol), + SourcePortRange: "*", + DestinationPortRange: "*", + SourceAddressPrefix: "*", + DestinationAddressPrefix: "*", + }, + } + if rule.Protocol == secrules.PROTO_ANY { + secRule.Properties.Protocol = "*" + } + if rule.Protocol == secrules.PROTO_ICMP { + return nil, nil + } + ipAddr := "*" + if rule.IPNet != nil { + ipAddr = rule.IPNet.String() + } + if rule.Direction == secrules.DIR_IN { + secRule.Properties.SourceAddressPrefix = ipAddr + } else { + secRule.Properties.DestinationAddressPrefix = ipAddr + } + if len(rule.Ports) > 0 { + for _, port := range rule.Ports { + secRule.Properties.DestinationPortRange = fmt.Sprintf("%d", port) + rules = append(rules, secRule) + } + return rules, nil + } else if rule.PortStart > 0 && rule.PortEnd > 0 { + secRule.Properties.DestinationPortRange = fmt.Sprintf("%d-%d", rule.PortStart, rule.PortEnd) + } + rules = append(rules, secRule) + return rules, nil +} + +func (self *SRegion) getClassicSecurityGroupRules(secgroupId string) ([]SClassicSecurityGroupRule, error) { + rules := []SClassicSecurityGroupRule{} + result, err := self.client.jsonRequest("GET", fmt.Sprintf("%s/securityRules?api-version=2015-06-01", secgroupId), "") if err != nil { return nil, err } - for i := 0; i < len(secgroups); i++ { - for k, v := range secgroups[i].Tags { - if k == "id" && v == tagId { - return &secgroups[i], nil - } - } - } - return region.CreateClassicSecurityGroup(name, tagId) -} - -func (region *SRegion) updateClassicSecurityGroupRules(secgroupId string, rules []secrules.SecurityRule) (string, error) { - secgroup, err := region.GetClassicSecurityGroupDetails(secgroupId) - if err != nil { - return "", err - } - securityRules := []SecurityRules{} - priority := int32(100) - for i := 0; i < len(rules); i++ { - if rule := convertSecurityGroupRule(rules[i], priority); rule != nil { - securityRules = append(securityRules, *rule) - priority++ - } - } - secgroup.Properties.SecurityRules = &securityRules - secgroup.Properties.ProvisioningState = "" - return secgroup.ID, region.client.Update(jsonutils.Marshal(secgroup), nil) -} - -func (region *SRegion) AssiginClassicSecurityGroup(instanceId, secgroupId string) error { - instance, err := region.GetClassicInstance(instanceId) - if err != nil { - return err - } - secgroup, err := region.GetClassicSecurityGroupDetails(secgroupId) - if err != nil { - return err - } - instance.Properties.NetworkProfile.NetworkSecurityGroup = &SubResource{ - ID: secgroupId, - Name: secgroup.Name, - Type: secgroup.Type, - } - return region.client.Update(jsonutils.Marshal(instance), nil) + return rules, result.Unmarshal(&rules, "value") } func (self *SRegion) syncClassicSecgroupRules(secgroupId string, rules []secrules.SecurityRule) (string, error) { - secgroup, err := self.GetClassicSecurityGroupDetails(secgroupId) + secgrouprules, err := self.getClassicSecurityGroupRules(secgroupId) if err != nil { return "", err } - sort.Sort(secrules.SecurityRuleSet(rules)) - sort.Sort(SecurityRulesSet(*secgroup.Properties.SecurityRules)) - - newRules := []secrules.SecurityRule{} - - i, j := 0, 0 - for i < len(rules) || j < len(*secgroup.Properties.SecurityRules) { - if i < len(rules) && j < len(*secgroup.Properties.SecurityRules) { - srcRule := (*secgroup.Properties.SecurityRules)[j].Properties.String() - destRule := rules[i].String() - cmp := strings.Compare(srcRule, destRule) - if cmp == 0 { - // keep secRule - newRules = append(newRules, rules[i]) - i++ - j++ - } else if cmp > 0 { - // remove srcRule - j++ - } else { - // add destRule - newRules = append(newRules, rules[i]) - i++ - } - } else if i >= len(rules) { - // del other rules - j++ - } else if j >= len(*secgroup.Properties.SecurityRules) { - // add rule - newRules = append(newRules, rules[i]) - i++ + for _, rule := range secgrouprules { + if rule.Properties.Priority >= 65000 { + continue + } + if err := self.client.Delete(rule.ID); err != nil { + return "", err } } - return self.updateClassicSecurityGroupRules(secgroup.ID, newRules) - -} - -func (self *SRegion) syncClassicSecurityGroup(tagId, name string, rules []secrules.SecurityRule) (string, error) { - secgroup, err := self.checkClassicSecurityGroup(tagId, name) - if err != nil { - return "", err + sort.Sort(secrules.SecurityRuleSet(rules)) + priority := int32(100) + ruleStrs := []string{} + for i, _rule := range rules { + ruleStr := rules[i].String() + if !utils.IsInStringArray(ruleStr, ruleStrs) { + _rules, err := convertClassicSecurityGroupRules(_rule, priority) + if err != nil { + return "", err + } + priority++ + ruleStrs = append(ruleStrs, ruleStr) + for _, rule := range _rules { + if err := self.addClassicSecgroupRule(secgroupId, rule); err != nil { + return "", err + } + } + } } - return self.syncClassicSecgroupRules(secgroup.ID, rules) + return secgroupId, nil +} + +func (self *SRegion) addClassicSecgroupRule(secgroupId string, rule SClassicSecurityGroupRule) error { + url := fmt.Sprintf("%s/securityRules/%s?api-version=2015-06-01", secgroupId, rule.Name) + _, err := self.client.jsonRequest("PUT", url, jsonutils.Marshal(rule).String()) + return err +} + +func (region *SRegion) syncClassicSecurityGroup(secgroupId, name, desc string, rules []secrules.SecurityRule) (string, error) { + if len(secgroupId) > 0 { + if _, err := region.GetClassicSecurityGroupDetails(secgroupId); err != nil { + if err != cloudprovider.ErrNotFound { + return "", err + } + secgroupId = "" + } + } + + if len(secgroupId) == 0 { + secgroup, err := region.CreateClassicSecurityGroup(name) + if err != nil { + return "", err + } + secgroupId = secgroup.ID + } + return region.syncClassicSecgroupRules(secgroupId, rules) } diff --git a/pkg/util/azure/debug.go b/pkg/util/azure/debug.go index 8da73e4983..57beb087d0 100644 --- a/pkg/util/azure/debug.go +++ b/pkg/util/azure/debug.go @@ -9,6 +9,10 @@ import ( "github.com/Azure/go-autorest/autorest" ) +const ( + DEBUG = false +) + func LogRequest() autorest.PrepareDecorator { return func(p autorest.Preparer) autorest.Preparer { return autorest.PreparerFunc(func(r *http.Request) (*http.Request, error) { diff --git a/pkg/util/azure/host.go b/pkg/util/azure/host.go index 2fe2f279cc..a32a5ca6df 100644 --- a/pkg/util/azure/host.go +++ b/pkg/util/azure/host.go @@ -23,7 +23,7 @@ func (self *SHost) GetId() string { } func (self *SHost) GetName() string { - return fmt.Sprintf("%s", self.zone.region.client.subscriptionId) + return fmt.Sprintf("%s/%s", self.zone.region.GetGlobalId(), self.zone.region.client.subscriptionId) } func (self *SHost) GetGlobalId() string { @@ -42,18 +42,48 @@ func (self *SHost) Refresh() error { return nil } -func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, networkId string, ipAddr string, desc string, passwd string, storageType string, diskSizes []int, publicKey string, secgroupId string) (cloudprovider.ICloudVM, error) { - nicId := "" - if net := self.zone.getNetworkById(networkId); net == nil { - return nil, fmt.Errorf("invalid network ID %s", networkId) - } else if nic, err := self.zone.region.CreateNetworkInterface(fmt.Sprintf("%s-ipconfig", name), ipAddr, net.GetId(), secgroupId); err != nil { - return nil, err - } else { - nicId = nic.ID - } - vmId, err := self._createVM(name, imgId, int32(sysDiskSize), cpu, memMB, nicId, ipAddr, desc, passwd, storageType, diskSizes, publicKey) +func (self *SHost) searchNetorkInterface(IPAddr string, networkId string, secgroupId string) (*SInstanceNic, error) { + interfaces, err := self.zone.region.GetNetworkInterfaces() if err != nil { - self.zone.region.DeleteNetworkInterface(nicId) + return nil, err + } + for i, nic := range interfaces { + for _, ipConf := range nic.Properties.IPConfigurations { + if ipConf.Properties.PrivateIPAddress == IPAddr && networkId == ipConf.Properties.Subnet.ID && ipConf.Properties.PrivateIPAllocationMethod == "Static" { + if nic.Properties.NetworkSecurityGroup == nil || nic.Properties.NetworkSecurityGroup.ID != secgroupId { + nic.Properties.NetworkSecurityGroup = &SSecurityGroup{ID: secgroupId} + if err := self.zone.region.client.Update(jsonutils.Marshal(nic), nil); err != nil { + log.Errorf("assign secgroup %s for nic %s failed %d") + return nil, err + } + } + return &interfaces[i], nil + } + } + } + return nil, cloudprovider.ErrNotFound +} + +func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, networkId string, ipAddr string, desc string, passwd string, storageType string, diskSizes []int, publicKey string, secgroupId string) (cloudprovider.ICloudVM, error) { + net := self.zone.getNetworkById(networkId) + if net == nil { + return nil, fmt.Errorf("invalid network ID %s", networkId) + } + nic, err := self.searchNetorkInterface(ipAddr, net.GetId(), secgroupId) + if err != nil { + if err == cloudprovider.ErrNotFound { + nic, err = self.zone.region.CreateNetworkInterface(fmt.Sprintf("%s-ipconfig", name), ipAddr, net.GetId(), secgroupId) + if err != nil { + return nil, err + } + } else { + return nil, err + } + } + + vmId, err := self._createVM(name, imgId, int32(sysDiskSize), cpu, memMB, nic.ID, ipAddr, desc, passwd, storageType, diskSizes, publicKey) + if err != nil { + self.zone.region.DeleteNetworkInterface(nic.ID) return nil, err } if vm, err := self.zone.region.GetInstance(vmId); err != nil { diff --git a/pkg/util/azure/instance.go b/pkg/util/azure/instance.go index bcc5b093fc..3e5cbf50d2 100644 --- a/pkg/util/azure/instance.go +++ b/pkg/util/azure/instance.go @@ -108,42 +108,21 @@ type NetworkProfile struct { NetworkInterfaces []NetworkInterfaceReference `json:"networkInterfaces,omitempty"` } -type InstanceViewStatus struct { +type Statuses struct { Code string Level string - DisplayStatus string + DisplayStatus string `json:"displayStatus,omitempty"` Message string //Time time.Time } -type FormattedMessage struct { - Language string - Message string -} - -type GuestAgentStatus struct { - ProtocolVersion string - Timestamp time.Time - GuestAgentVersion string - Status string - FormattedMessage FormattedMessage +type VMAgent struct { + VmAgentVersion string `json:"vmAgentVersion,omitempty"` + Statuses Statuses `json:"statuses,omitempty"` } type VirtualMachineInstanceView struct { - UpdateDomain int - FaultDomain int - Status string - StatusMessage string - PowerState string - PrivateIpAddress string - PublicIpAddresses []string - FullyQualifiedDomainName string - GuestAgentStatus GuestAgentStatus - - ComputerName string - OsName string - OsVersion string - Statuses []InstanceViewStatus + Statuses []Statuses `json:"statuses,omitempty"` } type DomainName struct { @@ -958,7 +937,7 @@ func (self *SInstance) StartVM() error { if err := self.host.zone.region.StartVM(self.ID); err != nil { return err } - self.host.zone.region.client.jsonRequest("PATCH", self.ID, "") + self.host.zone.region.client.jsonRequest("PATCH", self.ID, jsonutils.Marshal(self).String()) return cloudprovider.WaitStatus(self, models.VM_RUNNING, 10*time.Second, 300*time.Second) } @@ -967,7 +946,7 @@ func (self *SInstance) StopVM(isForce bool) error { if err != nil { return err } - self.host.zone.region.client.jsonRequest("PATCH", self.ID, "") + self.host.zone.region.client.jsonRequest("PATCH", self.ID, jsonutils.Marshal(self).String()) return cloudprovider.WaitStatus(self, models.VM_READY, 10*time.Second, 300*time.Second) } @@ -997,6 +976,10 @@ func (self *SInstance) GetIEIP() (cloudprovider.ICloudEIP, error) { return nil, nil } +func (self *SInstance) AssignSecurityGroup(secgroupId string) error { + return self.host.zone.region.AssiginSecurityGroup(self.ID, secgroupId) +} + func (self *SInstance) GetBillingType() string { return models.BILLING_TYPE_POSTPAID } @@ -1004,20 +987,3 @@ func (self *SInstance) GetBillingType() string { func (self *SInstance) GetExpiredAt() time.Time { return time.Now() } - -func (self *SInstance) AssignSecurityGroup(secgroupId string) error { - return self.host.zone.region.AssiginSecurityGroup(self.ID, secgroupId) -} - -func (self *SInstance) RevokeSecurityGroup() error { - nics, err := self.getNics() - if err != nil { - return err - } - for _, nic := range nics { - if err := nic.revokeSecurityGroup(); err != nil { - return err - } - } - return nil -} diff --git a/pkg/util/azure/instancenic.go b/pkg/util/azure/instancenic.go index 25b00430bd..808c9307a5 100644 --- a/pkg/util/azure/instancenic.go +++ b/pkg/util/azure/instancenic.go @@ -1,8 +1,6 @@ package azure import ( - "fmt" - "regexp" "strings" "yunion.io/x/jsonutils" @@ -133,49 +131,7 @@ func (self *SRegion) GetNetworkInterfaces() ([]SInstanceNic, error) { return result, nil } -func (self *SRegion) isNetworkInstanceNameAvaliable(resourceGroupName, nicName string) (bool, error) { - nics := []SInstanceNic{} - err := self.client.ListByTypeWithResourceGroup(resourceGroupName, "Microsoft.Network/networkInterfaces", &nics) - if err != nil { - return false, err - } - for i := 0; i < len(nics); i++ { - if nics[i].Name == nicName { - return false, nil - } - } - return true, nil -} - -func getResourceGroupNameByID(id string) string { - reg := regexp.MustCompile("/resourceGroups/(.+)/providers/") - _resourceGroup := reg.FindStringSubmatch(id) - if len(_resourceGroup) == 2 { - return _resourceGroup[1] - } - return "" -} - func (self *SRegion) CreateNetworkInterface(nicName string, ipAddr string, subnetId string, secgrpId string) (*SInstanceNic, error) { - secgroup, err := self.GetSecurityGroupDetails(secgrpId) - if err != nil { - return nil, err - } - secgroup.Properties.ProvisioningState = "" - - resourceGroupName := getResourceGroupNameByID(subnetId) - nicNameBase := nicName - for i := 0; i < 5; i++ { - ok, err := self.isNetworkInstanceNameAvaliable(resourceGroupName, nicName) - if err != nil { - return nil, err - } - if ok { - break - } - nicName = fmt.Sprintf("%s-%d", nicNameBase, i) - } - instancenic := SInstanceNic{ Name: nicName, Location: self.Name, @@ -193,7 +149,7 @@ func (self *SRegion) CreateNetworkInterface(nicName string, ipAddr string, subne }, }, }, - NetworkSecurityGroup: secgroup, + NetworkSecurityGroup: &SSecurityGroup{ID: secgrpId}, }, Type: "Microsoft.Network/networkInterfaces", } diff --git a/pkg/util/azure/region.go b/pkg/util/azure/region.go index 756a46938f..e5b715f5d7 100644 --- a/pkg/util/azure/region.go +++ b/pkg/util/azure/region.go @@ -418,13 +418,50 @@ func (region *SRegion) GetIEips() ([]cloudprovider.ICloudEIP, error) { return ieips, nil } -func (region *SRegion) SyncSecurityGroup(secgroupId string, vpcId string, name string, desc string, rules []secrules.SecurityRule) (string, error) { +func (region *SRegion) DeleteSecurityGroup(vpcId, secgroupId string) error { + if vpcId == "classic" { + return region.deleteClassicSecurityGroup(secgroupId) + } + secgroup, err := region.GetSecurityGroupDetails(secgroupId) + if err != nil { + if err == cloudprovider.ErrNotFound { + return nil + } + return err + } + if secgroup.Properties.NetworkInterfaces != nil { + for _, nic := range *secgroup.Properties.NetworkInterfaces { + nic, err := region.GetNetworkInterfaceDetail(nic.ID) + if err != nil { + return err + } + nic.Properties.NetworkSecurityGroup = nil + if err := region.client.Update(jsonutils.Marshal(nic), nil); err != nil { + return err + } + } + } + return region.client.Delete(secgroupId) +} + +func (region *SRegion) SyncSecurityGroup(secgroupId, vpcId, name, desc string, rules []secrules.SecurityRule) (string, error) { + if vpcId == "classic" { + return region.syncClassicSecurityGroup(secgroupId, name, desc, rules) + } + if len(secgroupId) > 0 { + if _, err := region.GetSecurityGroupDetails(secgroupId); err != nil { + if err != cloudprovider.ErrNotFound { + return "", err + } + secgroupId = "" + } + } if len(secgroupId) == 0 { - secgroup, err := region.CreateSecurityGroup(name, "") + secgroup, err := region.CreateSecurityGroup(name) if err != nil { return "", err } secgroupId = secgroup.ID } - return region.updateClassicSecurityGroupRules(secgroupId, rules) + return region.updateSecurityGroupRules(secgroupId, rules) } diff --git a/pkg/util/azure/securitygroup.go b/pkg/util/azure/securitygroup.go index a4a7a63284..5cc52f9e97 100644 --- a/pkg/util/azure/securitygroup.go +++ b/pkg/util/azure/securitygroup.go @@ -60,11 +60,11 @@ type Interface struct { } type SecurityGroupPropertiesFormat struct { - SecurityRules *[]SecurityRules `json:"securityRules,omitempty"` - DefaultSecurityRules *[]SecurityRules `json:"defaultSecurityRules,omitempty"` - NetworkInterfaces *[]Interface `json:"networkInterfaces,omitempty"` - Subnets *[]Subnet `json:"subnets,omitempty"` - ProvisioningState string //Possible values are: 'Updating', 'Deleting', and 'Failed' + SecurityRules []SecurityRules `json:"securityRules,omitempty"` + DefaultSecurityRules []SecurityRules `json:"defaultSecurityRules,omitempty"` + NetworkInterfaces *[]Interface `json:"networkInterfaces,omitempty"` + Subnets *[]Subnet `json:"subnets,omitempty"` + ProvisioningState string //Possible values are: 'Updating', 'Deleting', and 'Failed' } type SSecurityGroup struct { vpc *SVpc @@ -306,9 +306,9 @@ func (self *SSecurityGroup) GetRules() ([]secrules.SecurityRule, error) { if self.Properties.SecurityRules == nil { return rules, nil } - sort.Sort(SecurityRulesSet(*self.Properties.SecurityRules)) + sort.Sort(SecurityRulesSet(self.Properties.SecurityRules)) priority := 100 - for _, _rule := range *self.Properties.SecurityRules { + for _, _rule := range self.Properties.SecurityRules { _rule.Properties.Priority = int32(priority) secRules, err := _rule.Properties.toRules() if err != nil { @@ -331,13 +331,18 @@ func (self *SSecurityGroup) IsEmulated() bool { return false } -func (region *SRegion) CreateSecurityGroup(secName string, tagId string) (*SSecurityGroup, error) { - securityName := fmt.Sprintf("%s-%s", region.Name, secName) +func (self *SSecurityGroup) GetVpcId() string { + return "normal" +} + +func (region *SRegion) CreateSecurityGroup(secName string) (*SSecurityGroup, error) { + if secName == "Default" { + secName = "Default-copy" + } secgroup := SSecurityGroup{ - Name: securityName, + Name: secName, Type: "Microsoft.Network/networkSecurityGroups", Location: region.Name, - Tags: map[string]string{"id": tagId}, } return &secgroup, region.client.Create(jsonutils.Marshal(secgroup), &secgroup) } @@ -452,23 +457,23 @@ func (region *SRegion) updateSecurityGroupRules(secgroupId string, rules []secru ruleStrs = append(ruleStrs, ruleStr) } } - secgroup.Properties.SecurityRules = &securityRules + secgroup.Properties.SecurityRules = securityRules secgroup.Properties.ProvisioningState = "" return secgroup.ID, region.client.Update(jsonutils.Marshal(secgroup), nil) } func (region *SRegion) AttachSecurityToInterfaces(secgroupId string, nicIds []string) error { - secgroup, err := region.GetSecurityGroupDetails(secgroupId) - if err != nil { - return err + for _, nicId := range nicIds { + nic, err := region.GetNetworkInterfaceDetail(nicId) + if err != nil { + return err + } + nic.Properties.NetworkSecurityGroup = &SSecurityGroup{ID: secgroupId} + if err := region.client.Update(jsonutils.Marshal(nic), nil); err != nil { + return err + } } - interfaces := []Interface{} - for i := 0; i < len(nicIds); i++ { - interfaces = append(interfaces, Interface{ID: nicIds[i]}) - } - secgroup.Properties.NetworkInterfaces = &interfaces - secgroup.Properties.ProvisioningState = "" - return region.client.Update(jsonutils.Marshal(secgroup), nil) + return nil } func (region *SRegion) AssiginSecurityGroup(instanceId, secgroupId string) error { diff --git a/pkg/util/azure/service.go b/pkg/util/azure/service.go new file mode 100644 index 0000000000..c5fa17abe7 --- /dev/null +++ b/pkg/util/azure/service.go @@ -0,0 +1,55 @@ +package azure + +import ( + "fmt" +) + +type SServices struct { + Value []SService `json:"value,omitempty"` +} + +type SService struct { + ID string `json:"id,omitempty"` + Namespace string `json:"namespace,omitempty"` + RegistrationState string `json:"registrationState,omitempty"` + ResourceTypes []ResourceType `json:"resourceTypes,omitempty"` +} + +type ResourceType struct { + ApiVersions []string `json:"apiVersions,omitempty"` + Capabilities string `json:"capabilities,omitempty"` + Locations []string `json:"locations,omitempty"` + ResourceType string `json:"locations,omitempty"` +} + +func (self *SRegion) ListServices() ([]SService, error) { + services := []SService{} + return services, self.client.List("providers", &services) +} + +func (self *SRegion) SerciceShow(serviceType string) (*SService, error) { + service := SService{} + return &service, self.client.Get("providers/"+serviceType, []string{}, &service) +} + +func (self *SRegion) serviceOperation(resourceType, operation string) error { + services, err := self.ListServices() + if err != nil { + return err + } + for _, service := range services { + if service.Namespace == resourceType { + _, err := self.client.jsonRequest("POST", fmt.Sprintf("%s/%s", service.ID, operation), "") + return err + } + } + return fmt.Errorf("failed to find namespace: %s", resourceType) +} + +func (self *SRegion) ServiceRegister(resourceType string) error { + return self.serviceOperation(resourceType, "register") +} + +func (self *SRegion) ServiceUnRegister(resourceType string) error { + return self.serviceOperation(resourceType, "unregister") +} diff --git a/pkg/util/azure/shell/secgroup.go b/pkg/util/azure/shell/secgroup.go index 76deb7d46f..b2f993135a 100644 --- a/pkg/util/azure/shell/secgroup.go +++ b/pkg/util/azure/shell/secgroup.go @@ -52,16 +52,24 @@ func init() { }) type SecurityGroupCreateOptions struct { - NAME string `help:"Security Group name"` - TagId string `help:"Add a id tag to secgroup"` + NAME string `help:"Security Group name"` + Classic bool `help:"Create classic Security Group"` } shellutils.R(&SecurityGroupCreateOptions{}, "security-group-create", "Create security group", func(cli *azure.SRegion, args *SecurityGroupCreateOptions) error { - if secgrp, err := cli.CreateSecurityGroup(args.NAME, args.TagId); err != nil { - return err - } else { + if args.Classic { + secgrp, err := cli.CreateClassicSecurityGroup(args.NAME) + if err != nil { + return err + } printObject(secgrp) return nil } + secgrp, err := cli.CreateSecurityGroup(args.NAME) + if err != nil { + return err + } + printObject(secgrp) + return nil }) } diff --git a/pkg/util/azure/shell/service.go b/pkg/util/azure/shell/service.go new file mode 100644 index 0000000000..2f1c26bb5f --- /dev/null +++ b/pkg/util/azure/shell/service.go @@ -0,0 +1,41 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/util/azure" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type ServiceListOptions struct { + } + shellutils.R(&ServiceListOptions{}, "service-list", "List providers", func(cli *azure.SRegion, args *ServiceListOptions) error { + services, err := cli.ListServices() + if err != nil { + return err + } + printList(services, len(services), 0, 0, []string{}) + return nil + }) + + type ServiceOptions struct { + NAME string `help:"Name for service register"` + } + + shellutils.R(&ServiceOptions{}, "service-register", "Register service", func(cli *azure.SRegion, args *ServiceOptions) error { + return cli.ServiceRegister(args.NAME) + }) + + shellutils.R(&ServiceOptions{}, "service-unregister", "Unregister service", func(cli *azure.SRegion, args *ServiceOptions) error { + return cli.ServiceUnRegister(args.NAME) + }) + + shellutils.R(&ServiceOptions{}, "service-show", "Show service detail", func(cli *azure.SRegion, args *ServiceOptions) error { + service, err := cli.SerciceShow(args.NAME) + if err != nil { + return err + } + printObject(service) + return nil + }) + +} diff --git a/pkg/util/azure/shell/storage.go b/pkg/util/azure/shell/storage.go deleted file mode 100644 index 7728277f7d..0000000000 --- a/pkg/util/azure/shell/storage.go +++ /dev/null @@ -1,21 +0,0 @@ -package shell - -import ( - "yunion.io/x/onecloud/pkg/util/azure" - "yunion.io/x/onecloud/pkg/util/shellutils" -) - -func init() { - type StorageListOptions struct { - Limit int `help:"page size"` - Offset int `help:"page offset"` - } - shellutils.R(&StorageListOptions{}, "storage-list", "List storage types", func(cli *azure.SRegion, args *StorageListOptions) error { - storageType, err := cli.GetStorageTypes() - if err != nil { - return err - } - printList(storageType, len(storageType), args.Offset, args.Limit, []string{}) - return nil - }) -} diff --git a/pkg/util/azure/shell/vpc.go b/pkg/util/azure/shell/vpc.go index 6c068fdc0b..552a22c654 100644 --- a/pkg/util/azure/shell/vpc.go +++ b/pkg/util/azure/shell/vpc.go @@ -32,4 +32,23 @@ func init() { printObject(vpc) return nil }) + + shellutils.R(&VpcOptions{}, "vpc-delete", "Delete vpc", func(cli *azure.SRegion, args *VpcOptions) error { + return cli.DeleteVpc(args.ID) + }) + + type VpcCreateOptions struct { + NAME string `help:"vpc Name"` + CIDR string `help:"vpc cidr"` + Desc string `help:"vpc description"` + } + + shellutils.R(&VpcCreateOptions{}, "vpc-create", "Create vpc", func(cli *azure.SRegion, args *VpcCreateOptions) error { + vpc, err := cli.CreateIVpc(args.NAME, args.Desc, args.CIDR) + if err != nil { + return err + } + printObject(vpc) + return nil + }) } diff --git a/pkg/util/azure/snapshot.go b/pkg/util/azure/snapshot.go index 5db8d6da4c..ccc78658bd 100644 --- a/pkg/util/azure/snapshot.go +++ b/pkg/util/azure/snapshot.go @@ -22,8 +22,9 @@ type SSnapshot struct { Name string Location string ManagedBy string - Sku SnapshotSku + Sku *SnapshotSku Properties DiskProperties + Type string } func (self *SSnapshot) GetId() string { @@ -57,7 +58,23 @@ func (self *SSnapshot) IsEmulated() bool { } func (self *SRegion) CreateSnapshot(diskId, snapName, desc string) (*SSnapshot, error) { - snapshot := SSnapshot{} + disk, err := self.GetDisk(diskId) + if err != nil { + return nil, err + } + snapshot := SSnapshot{ + region: self, + Name: snapName, + Location: self.Name, + Properties: DiskProperties{ + CreationData: CreationData{ + CreateOption: "Copy", + SourceResourceID: diskId, + }, + DiskSizeGB: disk.Properties.DiskSizeGB, + }, + Type: "Microsoft.Compute/snapshots", + } return &snapshot, self.client.Create(jsonutils.Marshal(snapshot), &snapshot) } @@ -137,9 +154,11 @@ func (self *SRegion) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { classicSnapshots = append(classicSnapshots, _classicSnapshots...) isnapshots := make([]cloudprovider.ICloudSnapshot, len(snapshots)+len(classicSnapshots)) for i := 0; i < len(snapshots); i++ { + snapshots[i].region = self isnapshots[i] = &snapshots[i] } for i := 0; i < len(classicSnapshots); i++ { + classicSnapshots[i].region = self isnapshots[len(snapshots)+i] = &classicSnapshots[i] } return isnapshots, nil @@ -149,14 +168,6 @@ func (self *SSnapshot) GetDiskId() string { return self.Properties.CreationData.SourceResourceID } -func (self *SSnapshot) GetManagerId() string { - return self.region.client.providerId -} - -func (self *SSnapshot) GetRegionId() string { - return self.region.GetId() -} - func (self *SSnapshot) GetDiskType() string { return "" } diff --git a/pkg/util/azure/storage.go b/pkg/util/azure/storage.go index d2f5dcce26..c8599de7c4 100644 --- a/pkg/util/azure/storage.go +++ b/pkg/util/azure/storage.go @@ -15,6 +15,8 @@ type Capabilitie struct { Value string } +var STORAGETYPES = []string{"Standard_LRS", "Premium_LRS", "StandardSSD_LRS"} + type SStorage struct { zone *SZone diff --git a/pkg/util/azure/storagecache.go b/pkg/util/azure/storagecache.go index 0715ea4393..bcaf4036a5 100644 --- a/pkg/util/azure/storagecache.go +++ b/pkg/util/azure/storagecache.go @@ -134,65 +134,67 @@ func (self *SStoragecache) checkStorageAccount() (*SStorageAccount, error) { func (self *SStoragecache) uploadImage(userCred mcclient.TokenCredential, imageId string, osArch, osType, osDist string, isForce bool, tmpPath string) (string, error) { s := auth.GetAdminSession(options.Options.Region, "") - if meta, reader, err := modules.Images.Download(s, imageId); err != nil { + meta, reader, err := modules.Images.Download(s, imageId) + if err != nil { return "", err - } else { - // {"checksum":"d0ab0450979977c6ada8d85066a6e484","container_format":"bare","created_at":"2018-08-10T04:18:07","deleted":"False","disk_format":"vhd","id":"64189033-3ad4-413c-b074-6bf0b6be8508","is_public":"False","min_disk":"0","min_ram":"0","name":"centos-7.3.1611-20180104.vhd","owner":"5124d80475434da8b41fee48d5be94df","properties":{"os_arch":"x86_64","os_distribution":"CentOS","os_type":"Linux","os_version":"7.3.1611-VHD"},"protected":"False","size":"2028505088","status":"active","updated_at":"2018-08-10T04:20:59"} - log.Infof("meta data %s", meta) - - imageNameOnBlob, _ := meta.GetString("name") - if !strings.HasSuffix(imageNameOnBlob, ".vhd") { - imageNameOnBlob = fmt.Sprintf("%s.vhd", imageNameOnBlob) - } - tmpFile := fmt.Sprintf("%s/%s", tmpPath, imageNameOnBlob) - defer os.Remove(tmpFile) - f, err := os.Create(tmpFile) - if err != nil { - return "", err - } - defer f.Close() - if _, err := io.Copy(f, reader); err != nil { - return "", err - } - - storageaccount, err := self.checkStorageAccount() - if err != nil { - return "", err - } - - blobURI, err := storageaccount.UploadFile("image-cache", tmpFile) - if err != nil { - return "", err - } - - size, _ := meta.Int("size") - - imageBaseName := imageId - if imageBaseName[0] >= '0' && imageBaseName[0] <= '9' { - imageBaseName = fmt.Sprintf("img%s", imageId) - } - imageName := imageBaseName - nameIdx := 1 - - // check image name, avoid name conflict - for { - if _, err = self.region.GetImageByName(imageName); err != nil { - if err == cloudprovider.ErrNotFound { - break - } else { - return "", err - } - } - imageName = fmt.Sprintf("%s-%d", imageBaseName, nameIdx) - nameIdx += 1 - } - - if image, err := self.region.CreateImageByBlob(imageName, osType, blobURI, int32(size>>30)); err != nil { - return "", err - } else { - return image.GetGlobalId(), nil - } } + // { + // "checksum":"d0ab0450979977c6ada8d85066a6e484", + // "container_format":"bare", + // "created_at":"2018-08-10T04:18:07", + // "deleted":"False", + // "disk_format":"vhd", + // "id":"64189033-3ad4-413c-b074-6bf0b6be8508", + // "is_public":"False", + // "min_disk":"0", + // "min_ram":"0", + // "name":"centos-7.3.1611-20180104.vhd", + // "owner":"5124d80475434da8b41fee48d5be94df", + // "properties":{ + // "os_arch":"x86_64", + // "os_distribution":"CentOS", + // "os_type":"Linux", + // "os_version":"7.3.1611-VHD" + // }, + // "protected":"False", + // "size":"2028505088", + // "status":"active", + // "updated_at":"2018-08-10T04:20:59" + // } + log.Infof("meta data %s", meta) + + imageNameOnBlob, _ := meta.GetString("name") + if !strings.HasSuffix(imageNameOnBlob, ".vhd") { + imageNameOnBlob = fmt.Sprintf("%s.vhd", imageNameOnBlob) + } + tmpFile := fmt.Sprintf("%s/%s", tmpPath, imageNameOnBlob) + defer os.Remove(tmpFile) + f, err := os.Create(tmpFile) + if err != nil { + return "", err + } + defer f.Close() + if _, err := io.Copy(f, reader); err != nil { + return "", err + } + + storageaccount, err := self.checkStorageAccount() + if err != nil { + return "", err + } + + blobURI, err := storageaccount.UploadFile("image-cache", tmpFile) + if err != nil { + return "", err + } + + size, _ := meta.Int("size") + + image, err := self.region.CreateImageByBlob(imageId, osType, blobURI, int32(size>>30)) + if err != nil { + return "", err + } + return image.GetGlobalId(), nil } func (self *SStoragecache) CreateIImage(snapshotId, imageName, osType, imageDesc string) (cloudprovider.ICloudImage, error) { diff --git a/pkg/util/azure/vpc.go b/pkg/util/azure/vpc.go index ff506c61e0..4af4f48e75 100644 --- a/pkg/util/azure/vpc.go +++ b/pkg/util/azure/vpc.go @@ -76,7 +76,11 @@ func (self *SVpc) GetCidrBlock() string { } func (self *SVpc) Delete() error { - return self.region.client.Delete(self.ID) + return self.region.DeleteVpc(self.ID) +} + +func (self *SRegion) DeleteVpc(vpcId string) error { + return self.client.Delete(vpcId) } func (self *SVpc) getSecurityGroups() ([]SSecurityGroup, error) { diff --git a/pkg/util/azure/zone.go b/pkg/util/azure/zone.go index eba090104f..17851ed4a5 100644 --- a/pkg/util/azure/zone.go +++ b/pkg/util/azure/zone.go @@ -6,7 +6,6 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/pkg/utils" ) type SZone struct { @@ -67,38 +66,6 @@ func (self *SZone) getClassicHost() *SClassicHost { return self.classicHost } -func (self *SZone) getStorageTypes() (err error) { - if len(self.storageTypes) > 0 { - return nil - } - storages, err := self.region.GetStorageTypes() - if err != nil { - return err - } - self.storageTypes = []string{} - for i := 0; i < len(storages); i++ { - if !utils.IsInStringArray(storages[i].Name, self.storageTypes) { - self.storageTypes = append(self.storageTypes, storages[i].Name) - } - } - return nil -} - -func (self *SRegion) GetStorageTypes() ([]SStorage, error) { - storages := []SStorage{} - err := self.client.ListAll("Microsoft.Storage/skus", &storages) - if err != nil { - return nil, err - } - result := []SStorage{} - for i := 0; i < len(storages); i++ { - if utils.IsInStringArray(self.Name, storages[i].Locations) { - result = append(result, storages[i]) - } - } - return result, nil -} - func (self *SZone) GetIRegion() cloudprovider.ICloudRegion { return self.region } @@ -124,14 +91,8 @@ func (self *SZone) fetchClassicStorages() error { } func (self *SZone) fetchStorages() error { - if len(self.storageTypes) == 0 { - err := self.getStorageTypes() - if err != nil { - return err - } - } - self.istorages = make([]cloudprovider.ICloudStorage, len(self.storageTypes)) - for i, storageType := range self.storageTypes { + self.istorages = make([]cloudprovider.ICloudStorage, len(STORAGETYPES)) + for i, storageType := range STORAGETYPES { storage := SStorage{zone: self, storageType: storageType} self.istorages[i] = &storage } diff --git a/pkg/util/esxi/virtualmachine.go b/pkg/util/esxi/virtualmachine.go index f2858520af..10b07b5b07 100644 --- a/pkg/util/esxi/virtualmachine.go +++ b/pkg/util/esxi/virtualmachine.go @@ -233,6 +233,10 @@ func (dc *SVirtualMachine) ChangeConfig(instanceId string, ncpu int, vmem int) e return cloudprovider.ErrNotImplemented } +func (self *SVirtualMachine) AssignSecurityGroup(secgroupId string) error { + return cloudprovider.ErrNotImplemented +} + func (self *SVirtualMachine) GetBillingType() string { return models.BILLING_TYPE_POSTPAID } @@ -240,11 +244,3 @@ func (self *SVirtualMachine) GetBillingType() string { func (self *SVirtualMachine) GetExpiredAt() time.Time { return time.Time{} } - -func (self *SVirtualMachine) AssignSecurityGroup(secgroupId string) error { - return cloudprovider.ErrNotImplemented -} - -func (self *SVirtualMachine) RevokeSecurityGroup() error { - return cloudprovider.ErrNotImplemented -} diff --git a/pkg/webconsole/server/tty_server.go b/pkg/webconsole/server/tty_server.go index bde779afa8..09955dcc2e 100644 --- a/pkg/webconsole/server/tty_server.go +++ b/pkg/webconsole/server/tty_server.go @@ -99,7 +99,7 @@ func initSocketHandler(so socketio.Socket, p *session.Pty) { log.Infof("exec: %s", p.Command) args := strings.Split(p.Command, " ") cmd := exec.Command(args[0], args[1:]...) - cmd.Env = append(cmd.Env, "TERM=screen-256color") + cmd.Env = append(cmd.Env, "TERM=xterm-256color") if _pty, err := pty.Start(cmd); err != nil { so.Emit(OUTPUT_EVENT, err.Error()+"\r\n") log.Errorf("exec error: %v", err)