From 3dba5360d79f5e084cb00d947738c015c2fccab4 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Mon, 2 Dec 2019 22:58:06 +0800 Subject: [PATCH] quota recode --- cmd/climc/shell/quotas.go | 27 + go.mod | 4 +- go.sum | 12 +- pkg/apis/compute/api.go | 67 +- pkg/apis/compute/bucket.go | 8 +- pkg/apis/compute/cloudaccount.go | 3 +- pkg/apis/compute/cloudregions_const.go | 1 + pkg/apis/compute/disk.go | 31 +- pkg/apis/compute/guest_const.go | 8 + pkg/apis/compute/input.go | 15 + pkg/apis/compute/network.go | 4 +- pkg/apis/compute/secgroup.go | 9 +- pkg/apis/input.go | 97 +++ pkg/apis/scheduler/api.go | 3 + pkg/cloudcommon/db/caller.go | 1 + pkg/cloudcommon/db/db_dispatcher.go | 4 +- pkg/cloudcommon/db/db_joint_dispatcher.go | 2 +- pkg/cloudcommon/db/enabledstatusstandalone.go | 11 + pkg/cloudcommon/db/interface.go | 2 +- pkg/cloudcommon/db/jointbase.go | 10 + pkg/cloudcommon/db/modelbase.go | 6 +- pkg/cloudcommon/db/quotas/errors.go | 6 + pkg/cloudcommon/db/quotas/handler.go | 384 +++------- pkg/cloudcommon/db/quotas/interface.go | 48 ++ pkg/cloudcommon/db/quotas/lock.go | 27 + pkg/cloudcommon/db/quotas/models.go | 189 +++-- pkg/cloudcommon/db/quotas/quotakeys.go | 304 ++++++++ pkg/cloudcommon/db/quotas/quotakeys_test.go | 244 ++++++ pkg/cloudcommon/db/quotas/quotas.go | 241 +++--- pkg/cloudcommon/db/quotas/stores.go | 74 +- pkg/cloudcommon/db/quotas/usageworker.go | 143 ++-- pkg/cloudcommon/db/resourcebase.go | 12 + pkg/cloudcommon/db/sharablevirtual.go | 10 + pkg/cloudcommon/db/standalone.go | 9 + pkg/cloudcommon/db/statusstandalone.go | 11 + pkg/cloudcommon/db/taskman/interface.go | 4 +- pkg/cloudcommon/db/taskman/tasks.go | 85 ++- pkg/cloudcommon/db/virtualresource.go | 15 +- pkg/cloudcommon/options/options.go | 8 + pkg/cloudprovider/fakeregion.go | 4 + pkg/cloudprovider/resources.go | 1 + pkg/compute/guestdrivers/aliyun.go | 14 +- pkg/compute/guestdrivers/aws.go | 14 +- pkg/compute/guestdrivers/azure.go | 14 +- pkg/compute/guestdrivers/baremetals.go | 15 +- pkg/compute/guestdrivers/container.go | 16 +- pkg/compute/guestdrivers/ctyun.go | 15 +- pkg/compute/guestdrivers/esxi.go | 16 +- pkg/compute/guestdrivers/google.go | 15 +- pkg/compute/guestdrivers/huawei.go | 15 +- pkg/compute/guestdrivers/kvm.go | 16 +- pkg/compute/guestdrivers/openstack.go | 14 +- pkg/compute/guestdrivers/qcloud.go | 15 +- pkg/compute/guestdrivers/ucloud.go | 16 +- pkg/compute/guestdrivers/zstack.go | 15 +- pkg/compute/hostdrivers/base.go | 13 +- pkg/compute/models/baremetalagents.go | 13 +- pkg/compute/models/buckets.go | 150 ++-- pkg/compute/models/cloudaccounts.go | 42 +- pkg/compute/models/cloudregions.go | 7 +- pkg/compute/models/dbinstances.go | 14 + pkg/compute/models/disks.go | 184 ++++- pkg/compute/models/dynamicschedtags.go | 13 +- pkg/compute/models/elasticcache_accounts.go | 10 +- pkg/compute/models/elasticcache_acls.go | 10 +- pkg/compute/models/elasticcache_backups.go | 10 +- pkg/compute/models/elasticcache_instances.go | 23 +- pkg/compute/models/elasticips.go | 126 +-- pkg/compute/models/filters.go | 86 +++ pkg/compute/models/guest_actions.go | 160 ++-- pkg/compute/models/guestdrivers.go | 3 +- pkg/compute/models/guestnetworks.go | 32 +- pkg/compute/models/guests.go | 256 +++++-- pkg/compute/models/helper.go | 3 +- pkg/compute/models/hosts.go | 44 +- pkg/compute/models/hoststorages.go | 13 +- pkg/compute/models/instance_snapshots.go | 10 +- pkg/compute/models/isolated_devices.go | 60 +- pkg/compute/models/keypairs.go | 13 +- pkg/compute/models/loadbalanceracls.go | 11 +- pkg/compute/models/loadbalanceragents.go | 14 +- .../models/loadbalancerbackendgroups.go | 12 +- pkg/compute/models/loadbalancerbackends.go | 12 +- pkg/compute/models/loadbalancercachedacls.go | 11 +- .../models/loadbalancercachedcertificates.go | 11 +- .../models/loadbalancercertificates.go | 11 +- pkg/compute/models/loadbalancerclusters.go | 37 +- .../models/loadbalancerlistenerrules.go | 10 +- pkg/compute/models/loadbalancerlisteners.go | 10 +- pkg/compute/models/loadbalancernetworks.go | 14 +- pkg/compute/models/loadbalancers.go | 24 +- pkg/compute/models/networks.go | 86 ++- pkg/compute/models/projectquota.go | 157 ++++ pkg/compute/models/quotametrics.go | 85 --- pkg/compute/models/quotas.go | 468 +++++++----- pkg/compute/models/regionquota.go | 355 +++++++++ pkg/compute/models/routetables.go | 14 +- pkg/compute/models/schedpolicies.go | 13 +- pkg/compute/models/schedtagjoint.go | 14 +- pkg/compute/models/schedtags.go | 14 +- pkg/compute/models/secgrouprules.go | 23 +- pkg/compute/models/secgroups.go | 21 +- pkg/compute/models/skus.go | 16 +- pkg/compute/models/snapshots.go | 64 +- pkg/compute/models/storages.go | 38 +- pkg/compute/models/usage.go | 39 +- pkg/compute/models/vpcs.go | 11 +- pkg/compute/models/wires.go | 40 +- pkg/compute/models/zonequota.go | 197 +++++ pkg/compute/models/zones.go | 14 +- pkg/compute/options/options.go | 14 +- pkg/compute/service/handlers.go | 13 + .../baremetal_convert_hypervisor_task.go | 2 +- pkg/compute/tasks/disk_base_task.go | 7 +- pkg/compute/tasks/disk_batch_create_task.go | 31 +- pkg/compute/tasks/guest_base_task.go | 12 +- pkg/compute/tasks/guest_batch_create_task.go | 36 +- pkg/compute/tasks/guest_change_config_task.go | 16 +- pkg/compute/tasks/guest_resize_disk_task.go | 2 +- .../tasks/instance_snapshot_and_clone_task.go | 19 +- .../tasks/instance_snapshot_create_task.go | 13 +- pkg/compute/tasks/pending_usage.go | 44 +- pkg/compute/tasks/schedule.go | 30 +- pkg/compute/usages/handler.go | 247 +++--- pkg/image/models/image_guest.go | 10 +- pkg/image/models/images.go | 18 +- pkg/image/models/quotas.go | 77 +- pkg/mcclient/modules/mod_quotas.go | 28 +- pkg/mcclient/options/disks.go | 4 +- pkg/mcclient/options/schedulers.go | 2 + pkg/mcclient/options/servers.go | 14 +- pkg/mcclient/utils/fetchnames.go | 91 +++ pkg/multicloud/aliyun/region.go | 4 + pkg/multicloud/aws/region.go | 14 +- pkg/multicloud/azure/region.go | 4 + pkg/multicloud/huawei/huawei.go | 8 +- pkg/multicloud/huawei/region.go | 4 + pkg/multicloud/objectstore/objectstore.go | 4 + pkg/multicloud/openstack/region.go | 4 + pkg/multicloud/qcloud/region.go | 4 + .../ucloud/latitud_and_longitude.go | 1 + pkg/multicloud/ucloud/region.go | 4 + pkg/multicloud/zstack/region.go | 7 +- scripts/advchecks.py | 2 +- .../testify/assert/assertion_format.go | 82 ++ .../testify/assert/assertion_forward.go | 164 ++++ .../testify/assert/assertion_order.go | 309 ++++++++ .../stretchr/testify/assert/assertions.go | 96 ++- .../github.com/stretchr/testify/mock/mock.go | 20 +- .../stretchr/testify/require/require.go | 716 +++++++++++------- .../stretchr/testify/require/require.go.tmpl | 2 +- .../testify/require/require_forward.go | 164 ++++ .../stretchr/testify/suite/suite.go | 80 +- vendor/modules.txt | 4 +- vendor/yunion.io/x/jsonutils/access.go | 7 + vendor/yunion.io/x/sqlchemy/inc.go | 156 ++++ 156 files changed, 5904 insertions(+), 2181 deletions(-) create mode 100644 pkg/apis/compute/input.go create mode 100644 pkg/apis/input.go create mode 100644 pkg/cloudcommon/db/quotas/interface.go create mode 100644 pkg/cloudcommon/db/quotas/lock.go create mode 100644 pkg/cloudcommon/db/quotas/quotakeys.go create mode 100644 pkg/cloudcommon/db/quotas/quotakeys_test.go create mode 100644 pkg/compute/models/filters.go create mode 100644 pkg/compute/models/projectquota.go delete mode 100644 pkg/compute/models/quotametrics.go create mode 100644 pkg/compute/models/regionquota.go create mode 100644 pkg/compute/models/zonequota.go create mode 100644 pkg/mcclient/utils/fetchnames.go create mode 100644 vendor/github.com/stretchr/testify/assert/assertion_order.go create mode 100644 vendor/yunion.io/x/sqlchemy/inc.go diff --git a/cmd/climc/shell/quotas.go b/cmd/climc/shell/quotas.go index 9c8c1ed0b3..6d9b23d0c7 100644 --- a/cmd/climc/shell/quotas.go +++ b/cmd/climc/shell/quotas.go @@ -54,6 +54,33 @@ func init() { printObject(result) return nil }) + R(&QuotaOptions{}, "project-quota", "Show project-quota for current user or tenant", func(s *mcclient.ClientSession, args *QuotaOptions) error { + params := jsonutils.Marshal(args) + result, err := modules.ProjectQuotas.GetQuota(s, params) + if err != nil { + return err + } + printObject(result) + return nil + }) + R(&QuotaOptions{}, "region-quota", "Show region-quota for current user or tenant", func(s *mcclient.ClientSession, args *QuotaOptions) error { + params := jsonutils.Marshal(args) + result, err := modules.RegionQuotas.GetQuota(s, params) + if err != nil { + return err + } + printObject(result) + return nil + }) + R(&QuotaOptions{}, "zone-quota", "Show zone-quota for current user or tenant", func(s *mcclient.ClientSession, args *QuotaOptions) error { + params := jsonutils.Marshal(args) + result, err := modules.ZoneQuotas.GetQuota(s, params) + if err != nil { + return err + } + printObject(result) + return nil + }) R(&QuotaOptions{}, "image-quota", "Show image quota for current user or tenant", func(s *mcclient.ClientSession, args *QuotaOptions) error { params := jsonutils.Marshal(args) result, err := modules.ImageQuotas.GetQuota(s, params) diff --git a/go.mod b/go.mod index b993673579..19f7b0fff6 100644 --- a/go.mod +++ b/go.mod @@ -102,7 +102,7 @@ require ( github.com/skip2/go-qrcode v0.0.0-20190110000554-dc11ecdae0a9 github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.3 // indirect - github.com/stretchr/testify v1.3.0 + github.com/stretchr/testify v1.4.0 github.com/tencentcloud/tencentcloud-sdk-go v0.0.0-20181108132626-805d01dd0e2e github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20191108095731-8ca4b370cde4 github.com/tinylib/msgp v1.1.0 // indirect @@ -137,7 +137,7 @@ require ( yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d yunion.io/x/pkg v0.0.0-20191121110824-e03b47b93fe0 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e - yunion.io/x/sqlchemy v0.0.0-20191122085525-2d3bfdb3f51c + yunion.io/x/sqlchemy v0.0.0-20191206101534-32ae27ae30cb yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3 ) diff --git a/go.sum b/go.sum index a60f7e7aa8..e6d71f9e32 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,7 @@ github.com/Azure/azure-sdk-for-go v36.1.0+incompatible h1:smHlbChr/JDmsyUqELZXLs github.com/Azure/azure-sdk-for-go v36.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-autorest v10.15.5+incompatible h1:vdxx6wM1rVkKt/3niByPVjguoLWkWImOcJNvEykgBzY= github.com/Azure/go-autorest v10.15.5+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/dd-trace-go v0.6.1 h1:nsZ2lohbSw1CKtfNRu3wPh1jFirv6XSz8vqNpuIYWbM= github.com/DataDog/dd-trace-go v0.6.1/go.mod h1:SmQTTcC37XMyEm75HV0AWiZIYxDiaNhRi49zorIpW+o= @@ -460,6 +461,8 @@ github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syncthing/syncthing v0.14.48-rc.4/go.mod h1:nw3siZwHPA6M8iSfjDCWQ402eqvEIasMQOE8nFOxy7M= github.com/tencentcloud/tencentcloud-sdk-go v0.0.0-20181108132626-805d01dd0e2e h1:CtKVGXKh2bfmepZ/YogAjvL/CrSy9NGZET500K7arf4= github.com/tencentcloud/tencentcloud-sdk-go v0.0.0-20181108132626-805d01dd0e2e/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4= @@ -592,6 +595,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.zx2c4.com/wireguard v0.0.20190908 h1:SUoXDdwSMtomLdvke+zz83/u9tNvl4hHmcTIWp38tow= golang.zx2c4.com/wireguard v0.0.20190908/go.mod h1:LhfXh5z6bLC2lW2ve6BzYZFwnnsXK3OQjySR0Yh2dO8= @@ -658,8 +662,8 @@ yunion.io/x/executor v0.0.0-20191202093616-92e2e6119257 h1:LZ6eC1uoLDAp7NQlaCy+D yunion.io/x/executor v0.0.0-20191202093616-92e2e6119257/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051 h1:vtZw2iwGrsARNSwRTREGjmr2BWPdxbmXVkb3kI1qu28= yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= -yunion.io/x/jsonutils v0.0.0-20191005115334-bb1c187fc0e7 h1:9NcHs2OFMyN8O8SmJ3sFm5AJSVXP3u1N3avJsbyX3RI= -yunion.io/x/jsonutils v0.0.0-20191005115334-bb1c187fc0e7/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= +yunion.io/x/jsonutils v0.0.0-20191203054634-2178957d86cf h1:F7lWnVAPAi7qfpKaatKosNbnXyKkmgMK28tLncJe5Lg= +yunion.io/x/jsonutils v0.0.0-20191203054634-2178957d86cf/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b h1:Z9z+7iegu0HXuL+S8taVWRd1P4b9JJOgPXIeoqYrj7c= yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE= yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d h1:59zrDL7Ft+hDukguJRmLr/Gdu/9V75x+yX99ovZwfaA= @@ -671,7 +675,7 @@ yunion.io/x/pkg v0.0.0-20191121110824-e03b47b93fe0 h1:gMENCnVkKO5BlwtbKSKW5liEeH yunion.io/x/pkg v0.0.0-20191121110824-e03b47b93fe0/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo= yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo= -yunion.io/x/sqlchemy v0.0.0-20191122085525-2d3bfdb3f51c h1:4zMTt6JASU+hwm9AOb1sQkAgXlKbludSIQ6G7i7mPPA= -yunion.io/x/sqlchemy v0.0.0-20191122085525-2d3bfdb3f51c/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI= +yunion.io/x/sqlchemy v0.0.0-20191206101534-32ae27ae30cb h1:NYGUHfl8aelOGMp/LCCbnPZwLZl2tybN9ZM+XMnja8Y= +yunion.io/x/sqlchemy v0.0.0-20191206101534-32ae27ae30cb/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI= yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3 h1:bfC8EhXYvyGYldRWlzxiCM39Zfj3s3+zham9mW2h2LE= yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng= diff --git a/pkg/apis/compute/api.go b/pkg/apis/compute/api.go index 7ff0986e26..c06f4cf086 100644 --- a/pkg/apis/compute/api.go +++ b/pkg/apis/compute/api.go @@ -121,10 +121,10 @@ type ServerConfigs struct { // ResourceType "shared|prepaid|dedicated"` ResourceType string `json:"resource_type"` InstanceType string `json:"instance_type"` - Project string `json:"project_id"` - Domain string `json:"domain_id"` - Backup bool `json:"backup"` - Count int `json:"count"` + // Project string `json:"project_id"` + // Domain string `json:"domain_id"` + Backup bool `json:"backup"` + Count int `json:"count"` Disks []*DiskConfig `json:"disks"` Networks []*NetworkConfig `json:"nets"` @@ -156,39 +156,40 @@ type DeployConfig struct { } type ServerCreateInput struct { - apis.Meta + apis.VirtualResourceCreateInput + DeletePreventableCreateInput *ServerConfigs - Name string `json:"name"` - GenerateName string `json:"generate_name"` - VmemSize int `json:"vmem_size"` - VcpuCount int `json:"vcpu_count"` - UserData string `json:"user_data"` + // Name string `json:"name"` + // GenerateName string `json:"generate_name"` + VmemSize int `json:"vmem_size"` + VcpuCount int `json:"vcpu_count"` + UserData string `json:"user_data"` - KeypairId string `json:"keypair_id"` - Password string `json:"password"` - Cdrom string `json:"cdrom"` - Vga string `json:"vga"` - Vdi string `json:"vdi"` - Bios string `json:"bios"` - Description string `json:"description"` - BootOrder string `json:"boot_order"` - EnableCloudInit bool `json:"enable_cloud_init"` - ResetPassword *bool `json:"reset_password"` - DisableDelete *bool `json:"disable_delete"` - ShutdownBehavior string `json:"shutdown_behavior"` - AutoStart bool `json:"auto_start"` - DeployConfigs []*DeployConfig `json:"deploy_configs"` - IsSystem bool `json:"is_system"` - Duration string `json:"duration"` - AutoPrepaidRecycle bool `json:"auto_prepaid_recycle,omitfalse"` - SecgroupId string `json:"secgrp_id"` - EipBw int `json:"eip_bw,omitzero"` - EipChargeType string `json:"eip_charge_type,omitempty"` - Eip string `json:"eip,omitempty"` - InstanceSnapshotId string `json:"instance_snapshot_id,omitempty"` - Secgroups []string `json:"secgroups"` + KeypairId string `json:"keypair_id"` + Password string `json:"password"` + Cdrom string `json:"cdrom"` + Vga string `json:"vga"` + Vdi string `json:"vdi"` + Bios string `json:"bios"` + // Description string `json:"description"` + BootOrder string `json:"boot_order"` + EnableCloudInit bool `json:"enable_cloud_init"` + ResetPassword *bool `json:"reset_password"` + // DisableDelete *bool `json:"disable_delete"` + ShutdownBehavior string `json:"shutdown_behavior"` + AutoStart bool `json:"auto_start"` + DeployConfigs []*DeployConfig `json:"deploy_configs"` + // IsSystem bool `json:"is_system"` + Duration string `json:"duration"` + AutoPrepaidRecycle bool `json:"auto_prepaid_recycle,omitfalse"` + SecgroupId string `json:"secgrp_id"` + EipBw int `json:"eip_bw,omitzero"` + EipChargeType string `json:"eip_charge_type,omitempty"` + Eip string `json:"eip,omitempty"` + InstanceSnapshotId string `json:"instance_snapshot_id,omitempty"` + Secgroups []string `json:"secgroups"` OsType string `json:"os_type"` // Fill by server diff --git a/pkg/apis/compute/bucket.go b/pkg/apis/compute/bucket.go index b6c61a2dfb..4394edff2d 100644 --- a/pkg/apis/compute/bucket.go +++ b/pkg/apis/compute/bucket.go @@ -36,13 +36,11 @@ const ( ) type BucketCreateInput struct { - apis.Meta + apis.VirtualResourceCreateInput + RegionalResourceCreateInput + ManagedResourceCreateInput - Name string `json:"name"` - Cloudregion string `json:"cloudregion"` - Manager string `json:"manager"` StorageClass string `json:"storage_class"` - Description string `json:"description"` } type BucketDetail struct { diff --git a/pkg/apis/compute/cloudaccount.go b/pkg/apis/compute/cloudaccount.go index d59cce942e..b6701620ea 100644 --- a/pkg/apis/compute/cloudaccount.go +++ b/pkg/apis/compute/cloudaccount.go @@ -53,7 +53,7 @@ type CloudaccountCredentialInput struct { } type CloudaccountCreateInput struct { - apis.Meta + apis.EnabledStatusStandaloneResourceCreateInput Provider string Brand string @@ -63,7 +63,6 @@ type CloudaccountCreateInput struct { Secret string AccessUrl string TenantId string - Name string Description string Enabled bool EnableAutoSync bool diff --git a/pkg/apis/compute/cloudregions_const.go b/pkg/apis/compute/cloudregions_const.go index 7e57bf4519..b7af1ac4fe 100644 --- a/pkg/apis/compute/cloudregions_const.go +++ b/pkg/apis/compute/cloudregions_const.go @@ -54,6 +54,7 @@ const ( CITY_BAO_DING = "Baoding" //保定 CITY_NAN_JING = "Nanjing" //南京 CITY_FO_SHAN = "Foshan" //佛山 + CITY_QUAN_ZHOU = "Quanzhou" //泉州 // 日本 CITY_TOKYO = "Tokyo" //东京 diff --git a/pkg/apis/compute/disk.go b/pkg/apis/compute/disk.go index f4bbc85458..cd5df60d39 100644 --- a/pkg/apis/compute/disk.go +++ b/pkg/apis/compute/disk.go @@ -19,7 +19,7 @@ import ( ) type DiskCreateInput struct { - apis.Meta + apis.VirtualResourceCreateInput *DiskConfig @@ -29,16 +29,14 @@ type DiskCreateInput struct { PreferWire string `json:"prefer_wire_id"` PreferHost string `json:"prefer_host_id"` - Name string `json:"name"` - Description string `json:"description"` - Hypervisor string `json:"hypervisor"` - Project string `json:"project"` - Domain string `json:"domain_id"` + Hypervisor string `json:"hypervisor"` + // Project string `json:"project"` + // Domain string `json:"domain_id"` } // ToServerCreateInput used by disk schedule func (req *DiskCreateInput) ToServerCreateInput() *ServerCreateInput { - return &ServerCreateInput{ + input := ServerCreateInput{ ServerConfigs: &ServerConfigs{ PreferRegion: req.PreferRegion, PreferZone: req.PreferZone, @@ -46,22 +44,29 @@ func (req *DiskCreateInput) ToServerCreateInput() *ServerCreateInput { PreferHost: req.PreferHost, Hypervisor: req.Hypervisor, Disks: []*DiskConfig{req.DiskConfig}, - Project: req.Project, - Domain: req.Domain, + // Project: req.Project, + // Domain: req.Domain, }, - Name: req.Name, } + input.Name = req.Name + input.Project = req.Project + input.ProjectId = req.ProjectId + input.Domain = req.Domain + input.DomainId = req.DomainId + return &input } func (req *ServerCreateInput) ToDiskCreateInput() *DiskCreateInput { - return &DiskCreateInput{ + input := DiskCreateInput{ DiskConfig: req.Disks[0], PreferRegion: req.PreferRegion, PreferHost: req.PreferHost, PreferZone: req.PreferZone, PreferWire: req.PreferWire, - Name: req.Name, - Project: req.Project, Hypervisor: req.Hypervisor, } + input.Name = req.Name + input.Project = req.Project + input.Domain = req.Domain + return &input } diff --git a/pkg/apis/compute/guest_const.go b/pkg/apis/compute/guest_const.go index d38caf92d4..862e0a20c2 100644 --- a/pkg/apis/compute/guest_const.go +++ b/pkg/apis/compute/guest_const.go @@ -244,3 +244,11 @@ const ( VM_METADATA_OS_NAME = "os_name" VM_METADATA_OS_VERSION = "os_version" ) + +func Hypervisors2HostTypes(hypervisors []string) []string { + hostTypes := make([]string, len(hypervisors)) + for i := range hypervisors { + hostTypes[i] = HYPERVISOR_HOSTTYPE[hypervisors[i]] + } + return hostTypes +} diff --git a/pkg/apis/compute/input.go b/pkg/apis/compute/input.go new file mode 100644 index 0000000000..8b351abf21 --- /dev/null +++ b/pkg/apis/compute/input.go @@ -0,0 +1,15 @@ +package compute + +type RegionalResourceCreateInput struct { + Cloudregion string `json:"cloudregion"` + CloudregionId string `json:"cloudregion_id"` +} + +type ManagedResourceCreateInput struct { + Manager string `json:"manager"` + ManagerId string `json:"manager_id"` +} + +type DeletePreventableCreateInput struct { + DisableDelete *bool `json:"disable_delete"` +} diff --git a/pkg/apis/compute/network.go b/pkg/apis/compute/network.go index 51b4932a2d..fcd7ed41e5 100644 --- a/pkg/apis/compute/network.go +++ b/pkg/apis/compute/network.go @@ -48,9 +48,7 @@ type NetworkListInput struct { } type NetworkCreateInput struct { - apis.Meta - - IsSystem bool `json:"is_system"` + apis.SharableVirtualResourceCreateInput // description: network name // unique: true diff --git a/pkg/apis/compute/secgroup.go b/pkg/apis/compute/secgroup.go index 19032f4911..328c65de97 100644 --- a/pkg/apis/compute/secgroup.go +++ b/pkg/apis/compute/secgroup.go @@ -26,7 +26,7 @@ import ( ) type SSecgroupRuleCreateInput struct { - apis.Meta + apis.ResourceBaseCreateInput Priority int Protocol string @@ -70,12 +70,9 @@ func (input *SSecgroupRuleCreateInput) Check() error { } type SSecgroupCreateInput struct { - apis.Meta + apis.SharableVirtualResourceCreateInput - Name string - Status string - Description string - Rules []SSecgroupRuleCreateInput + Rules []SSecgroupRuleCreateInput `json:"rules"` } type SSecgroupListFilterInput struct { diff --git a/pkg/apis/input.go b/pkg/apis/input.go new file mode 100644 index 0000000000..0b6dd58d4a --- /dev/null +++ b/pkg/apis/input.go @@ -0,0 +1,97 @@ +package apis + +type DomainizedResourceCreateInput struct { + // description: the owner domain name or id + // required: false + Domain string `json:"project_domain"` + + // description: the owner domain name or id, alias field of domain + // required: false + DomainId string `json:"domain_id"` +} + +type ProjectizedResourceCreateInput struct { + DomainizedResourceCreateInput + + // description: the owner project name or id + // required: false + Project string `json:"project"` + + // description: the owner project name or id, alias field of project + // required: false + ProjectId string `json:"project_id"` +} + +type SharableVirtualResourceCreateInput struct { + VirtualResourceCreateInput + + // description: indicate the resource is a public resource + // required: false + IsPublic *bool `json:"is_public"` + + // description: indicate the shared scope for a public resource, which can be domain or system or none + // required: false + PublicScope string `json:"public_scope"` +} + +type VirtualResourceCreateInput struct { + StatusStandaloneResourceCreateInput + ProjectizedResourceCreateInput + + // description: indicate the resource is a system resource, which is not visible to user + // required: false + IsSystem *bool `json:"is_system"` +} + +type EnabledStatusStandaloneResourceCreateInput struct { + StatusStandaloneResourceCreateInput + + // description: indicate the resource is enabled/disabled by administrator + // required: false + Enabled *bool `json:"enabled"` +} + +type StatusStandaloneResourceCreateInput struct { + StandaloneResourceCreateInput + + // description: the status of the resource + // required: false + Status string `json:"status"` +} + +type StandaloneResourceCreateInput struct { + ResourceBaseCreateInput + + // description: resource name, required if generated_name is not given + // unique: true + // required: false + // example: test-network + Name string `json:"name"` + + // description: generated resource name, given a pattern to generate name, required if name is not given + // unique: false + // required: false + // example: test### + GenerateName string `json:"generate_name"` + + // description: resource description + // required: false + // example: test create network + Description string `json:"description"` + + // description: the resource is an emulated resource + // required: false + IsEmulated *bool `json:"is_emulated"` +} + +type JoinResourceBaseCreateInput struct { + ResourceBaseCreateInput +} + +type ResourceBaseCreateInput struct { + ModelBaseCreateInput +} + +type ModelBaseCreateInput struct { + Meta +} diff --git a/pkg/apis/scheduler/api.go b/pkg/apis/scheduler/api.go index cdd335e6e8..80343a62bb 100644 --- a/pkg/apis/scheduler/api.go +++ b/pkg/apis/scheduler/api.go @@ -53,6 +53,9 @@ type ServerConfig struct { GuestStatus string `json:"guest_status"` Cdrom string `json:"cdrom"` + Project string `json:"project_id"` + Domain string `json:"domain_id"` + // DEPRECATED Metadata map[string]string `json:"__meta__"` ForGuests []*ForGuest `json:"for_guests"` diff --git a/pkg/cloudcommon/db/caller.go b/pkg/cloudcommon/db/caller.go index 5cb80a071a..0b238cddc0 100644 --- a/pkg/cloudcommon/db/caller.go +++ b/pkg/cloudcommon/db/caller.go @@ -89,6 +89,7 @@ func convertParams(params []*param) []reflect.Value { type param struct { pType reflect.Type input interface{} + isPtr bool } func newParam(pType reflect.Type, input interface{}) *param { diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index 0784565ba7..b11e11968d 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -1108,7 +1108,7 @@ func (dispatcher *DBModelDispatcher) Create(ctx context.Context, query jsonutils }() OpsLog.LogEvent(model, ACT_CREATE, model.GetShortDesc(ctx), userCred) - dispatcher.modelManager.OnCreateComplete(ctx, []IModel{model}, userCred, query, data) + dispatcher.modelManager.OnCreateComplete(ctx, []IModel{model}, userCred, ownerId, query, data) return getItemDetails(dispatcher.modelManager, model, ctx, userCred, query) } @@ -1239,7 +1239,7 @@ func (dispatcher *DBModelDispatcher) BatchCreate(ctx context.Context, query json lockman.LockClass(ctx, manager, GetLockClassKey(manager, ownerId)) defer lockman.ReleaseClass(ctx, manager, GetLockClassKey(manager, ownerId)) - manager.OnCreateComplete(ctx, models, userCred, query, multiData[0]) + manager.OnCreateComplete(ctx, models, userCred, ownerId, query, multiData[0]) } return results, nil } diff --git a/pkg/cloudcommon/db/db_joint_dispatcher.go b/pkg/cloudcommon/db/db_joint_dispatcher.go index f939c7f20d..de89206b61 100644 --- a/pkg/cloudcommon/db/db_joint_dispatcher.go +++ b/pkg/cloudcommon/db/db_joint_dispatcher.go @@ -187,7 +187,7 @@ func attachItems(dispatcher *DBJointModelDispatcher, master IStandaloneModel, sl } item.PostCreate(ctx, userCred, nil, query, data) OpsLog.LogAttachEvent(ctx, master, slave, userCred, jsonutils.Marshal(item)) - dispatcher.modelManager.OnCreateComplete(ctx, []IModel{item}, userCred, query, data) + dispatcher.modelManager.OnCreateComplete(ctx, []IModel{item}, userCred, nil, query, data) return getItemDetails(dispatcher.JointModelManager(), item, ctx, userCred, query) } diff --git a/pkg/cloudcommon/db/enabledstatusstandalone.go b/pkg/cloudcommon/db/enabledstatusstandalone.go index 9eb7b484b7..6ab76b928d 100644 --- a/pkg/cloudcommon/db/enabledstatusstandalone.go +++ b/pkg/cloudcommon/db/enabledstatusstandalone.go @@ -19,7 +19,9 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/onecloud/pkg/apis" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/logclient" "yunion.io/x/onecloud/pkg/util/rbacutils" @@ -78,3 +80,12 @@ func (self *SEnabledStatusStandaloneResourceBase) PerformDisable(ctx context.Con } return nil, nil } + +func (manager *SEnabledStatusStandaloneResourceBaseManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input apis.EnabledStatusStandaloneResourceCreateInput) (apis.EnabledStatusStandaloneResourceCreateInput, error) { + var err error + input.StatusStandaloneResourceCreateInput, err = manager.SStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.StatusStandaloneResourceCreateInput) + if err != nil { + return input, errors.Wrap(err, "SStatusStandaloneResourceBaseManager.ValidateCreateData") + } + return input, nil +} diff --git a/pkg/cloudcommon/db/interface.go b/pkg/cloudcommon/db/interface.go index df7f3481ee..bcec74d2e4 100644 --- a/pkg/cloudcommon/db/interface.go +++ b/pkg/cloudcommon/db/interface.go @@ -83,7 +83,7 @@ type IModelManager interface { BatchCreateValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) // ValidateCreateData dynamic called by dispatcher // ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) - OnCreateComplete(ctx context.Context, items []IModel, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) + OnCreateComplete(ctx context.Context, items []IModel, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) BatchPreValidate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict, count int) (func(), error) diff --git a/pkg/cloudcommon/db/jointbase.go b/pkg/cloudcommon/db/jointbase.go index a486eedd59..73a5017eb6 100644 --- a/pkg/cloudcommon/db/jointbase.go +++ b/pkg/cloudcommon/db/jointbase.go @@ -24,6 +24,7 @@ import ( "yunion.io/x/pkg/util/reflectutils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/rbacutils" ) @@ -226,3 +227,12 @@ func (manager *SJointResourceBaseManager) ResourceScope() rbacutils.TRbacScope { func (manager *SJointResourceBaseManager) NamespaceScope() rbacutils.TRbacScope { return rbacutils.ScopeSystem } + +func (manager *SJointResourceBaseManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input apis.JoinResourceBaseCreateInput) (apis.JoinResourceBaseCreateInput, error) { + var err error + input.ResourceBaseCreateInput, err = manager.SResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.ResourceBaseCreateInput) + if err != nil { + return input, err + } + return input, nil +} diff --git a/pkg/cloudcommon/db/modelbase.go b/pkg/cloudcommon/db/modelbase.go index 2ddf1e3760..1996196bd5 100644 --- a/pkg/cloudcommon/db/modelbase.go +++ b/pkg/cloudcommon/db/modelbase.go @@ -193,11 +193,11 @@ func (manager *SModelBaseManager) AllowCreateItem(ctx context.Context, userCred return false } -func (manager *SModelBaseManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - return data, nil +func (manager *SModelBaseManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input apis.ModelBaseCreateInput) (apis.ModelBaseCreateInput, error) { + return input, nil } -func (manager *SModelBaseManager) OnCreateComplete(ctx context.Context, items []IModel, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) { +func (manager *SModelBaseManager) OnCreateComplete(ctx context.Context, items []IModel, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { // do nothing } diff --git a/pkg/cloudcommon/db/quotas/errors.go b/pkg/cloudcommon/db/quotas/errors.go index c229022b3e..f6effe47a9 100644 --- a/pkg/cloudcommon/db/quotas/errors.go +++ b/pkg/cloudcommon/db/quotas/errors.go @@ -17,6 +17,8 @@ package quotas import ( "fmt" "strings" + + "yunion.io/x/onecloud/pkg/httperrors" ) type SOutOfQuotaError struct { @@ -29,6 +31,10 @@ type SOutOfQuotaErrors struct { errors []SOutOfQuotaError } +func (e *SOutOfQuotaError) Cause() error { + return httperrors.ErrOutOfQuota +} + func (e *SOutOfQuotaError) Error() string { return fmt.Sprintf("%s limit %d used %d", e.name, e.limit, e.used) } diff --git a/pkg/cloudcommon/db/quotas/handler.go b/pkg/cloudcommon/db/quotas/handler.go index 513002c8ac..783c45a3cb 100644 --- a/pkg/cloudcommon/db/quotas/handler.go +++ b/pkg/cloudcommon/db/quotas/handler.go @@ -25,7 +25,6 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" - "yunion.io/x/onecloud/pkg/appctx" "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/cloudcommon/consts" "yunion.io/x/onecloud/pkg/cloudcommon/db" @@ -88,51 +87,45 @@ func AddQuotaHandler(manager *SQuotaBaseManager, prefix string, app *appsrv.Appl auth.Authenticate(checkQuotaHanlder), nil, "check_quota", nil)*/ } -func (manager *SQuotaBaseManager) queryQuota(ctx context.Context, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platforma []string, refresh bool) (*jsonutils.JSONDict, IQuota, error) { +func (manager *SQuotaBaseManager) queryQuota(ctx context.Context, quota IQuota, refresh bool) (*jsonutils.JSONDict, error) { ret := jsonutils.NewDict() - quota := manager.newQuota() - err := manager.GetQuota(ctx, scope, ownerId, platforma, quota) - if err != nil { - return nil, nil, err - } + keys := quota.GetKeys() + usage := manager.newQuota() - err = manager.usageStore.GetQuota(ctx, scope, ownerId, platforma, usage) + err := manager.usageStore.GetQuota(ctx, keys, usage) if err != nil { - return nil, nil, err + return nil, err } - if usage.IsEmpty() || refresh { + if refresh { usageChan := make(chan IQuota) - manager.PostUsageJob(scope, ownerId, nil, usageChan, false, true) + manager.PostUsageJob(keys, usageChan, true) usage = <-usageChan } - pending := manager.newQuota() - err = manager.GetPendingUsage(ctx, scope, ownerId, nil, pending) + pendings, err := manager.GetPendingUsages(ctx, keys) if err != nil { - return nil, nil, err + return nil, err } + ret.Update(jsonutils.Marshal(keys)) ret.Update(quota.ToJSON("")) ret.Update(usage.ToJSON("usage")) - if !pending.IsEmpty() { - ret.Update(pending.ToJSON("pending")) - } - - if scope == rbacutils.ScopeDomain { - total, err := manager.getDomainTotalQuota(ctx, ownerId.GetProjectDomainId(), nil) - if err == nil { - ret.Update(total.ToJSON("total")) + if len(pendings) > 0 { + pendingArray := jsonutils.NewArray() + for _, q := range pendings { + pendingArray.Add(q.ToJSON("")) } + ret.Add(pendingArray, "pending") } - return ret, usage, nil + return ret, nil } func (manager *SQuotaBaseManager) getQuotaHanlder(ctx context.Context, w http.ResponseWriter, r *http.Request) { + params, query, _ := appsrv.FetchEnv(ctx, w, r) userCred := auth.FetchUserCredential(ctx, policy.FilterPolicyCredential) - params := appctx.AppContextParams(ctx) var ownerId mcclient.IIdentityProvider var scope rbacutils.TRbacScope @@ -157,27 +150,24 @@ func (manager *SQuotaBaseManager) getQuotaHanlder(ctx context.Context, w http.Re scope = rbacutils.ScopeProject } - quota, _, err := manager.queryQuota(ctx, scope, ownerId, nil, true) - + keys := OwnerIdQuotaKeys(scope, ownerId) + refresh := jsonutils.QueryBoolean(query, "refresh", false) + quotaList, err := manager.listQuotas(ctx, userCred, keys.DomainId, keys.ProjectId, refresh) if err != nil { httperrors.GeneralServerError(w, err) return } - - body := jsonutils.NewDict() - body.Add(quota, manager.KeywordPlural()) - - appsrv.SendJSON(w, body) + manager.sendQuotaList(w, quotaList) } -func FetchSetQuotaScope(ctx context.Context, userCred mcclient.TokenCredential, data jsonutils.JSONObject) (mcclient.IIdentityProvider, rbacutils.TRbacScope, rbacutils.TRbacScope, error) { +func (manager *SQuotaBaseManager) fetchSetQuotaScope(ctx context.Context, userCred mcclient.TokenCredential, data jsonutils.JSONObject) (mcclient.IIdentityProvider, rbacutils.TRbacScope, rbacutils.TRbacScope, error) { var scope rbacutils.TRbacScope ownerId, err := db.FetchProjectInfo(ctx, data) if err != nil { return nil, scope, scope, err } var requestScope rbacutils.TRbacScope - ownerScope := policy.PolicyManager.AllowScope(userCred, consts.GetServiceType(), quotaKeywords, policy.PolicyActionUpdate) + ownerScope := policy.PolicyManager.AllowScope(userCred, consts.GetServiceType(), manager.KeywordPlural(), policy.PolicyActionUpdate) if ownerId != nil { if len(ownerId.GetProjectId()) > 0 { // project level @@ -204,8 +194,8 @@ func FetchSetQuotaScope(ctx context.Context, userCred mcclient.TokenCredential, } func (manager *SQuotaBaseManager) setQuotaHanlder(ctx context.Context, w http.ResponseWriter, r *http.Request) { + params, _, body := appsrv.FetchEnv(ctx, w, r) userCred := auth.FetchUserCredential(ctx, policy.FilterPolicyCredential) - params := appctx.AppContextParams(ctx) projectId := params[""] domainId := params[""] @@ -215,18 +205,23 @@ func (manager *SQuotaBaseManager) setQuotaHanlder(ctx context.Context, w http.Re } else if len(domainId) > 0 { data.Add(jsonutils.NewString(domainId), "project_domain") } - ownerId, scope, allowScope, err := FetchSetQuotaScope(ctx, userCred, data) + ownerId, scope, _, err := manager.fetchSetQuotaScope(ctx, userCred, data) if err != nil { httperrors.GeneralServerError(w, err) return } - body, err := appsrv.FetchJSON(r) - if err != nil { - log.Errorf("Fail to decode JSON request body: %s", err) + if body == nil { httperrors.InvalidInputError(w, "fail to decode body") return } + + keys := SBaseQuotaKeys{} + body.Unmarshal(&keys, manager.KeywordPlural()) + paramKeys := OwnerIdQuotaKeys(scope, ownerId) + keys.DomainId = paramKeys.DomainId + keys.ProjectId = paramKeys.ProjectId + quota := manager.newQuota() err = body.Unmarshal(quota, manager.KeywordPlural()) if err != nil { @@ -234,8 +229,10 @@ func (manager *SQuotaBaseManager) setQuotaHanlder(ctx context.Context, w http.Re httperrors.InvalidInputError(w, "fail to decode body") return } + quota.SetKeys(keys) + oquota := manager.newQuota() - err = manager.GetQuota(ctx, scope, ownerId, nil, oquota) + err = manager.GetQuota(ctx, keys, oquota) if err != nil { log.Errorf("get quota fail %s", err) httperrors.GeneralServerError(w, err) @@ -246,72 +243,14 @@ func (manager *SQuotaBaseManager) setQuotaHanlder(ctx context.Context, w http.Re case QUOTA_ACTION_ADD: oquota.Add(quota) case QUOTA_ACTION_RESET: - oquota.FetchSystemQuota(scope, ownerId) + oquota.FetchSystemQuota() case QUOTA_ACTION_REPLACE: oquota = quota default: oquota.Update(quota) } - if consts.GetNonDefaultDomainProjects() { - // only if non-default-domain-project is turned ON, check the conformance between project quotas and domain quotas - if scope == rbacutils.ScopeProject { - total, err := manager.getDomainTotalQuota(ctx, ownerId.GetProjectDomainId(), []string{ownerId.GetProjectId()}) - if err != nil { - log.Errorf("get total quota fail %s", err) - httperrors.GeneralServerError(w, err) - return - } - - domainQuota := manager.newQuota() - err = manager.GetQuota(ctx, rbacutils.ScopeDomain, ownerId, nil, domainQuota) - if err != nil { - log.Errorf("GetQuota for domain %s fail %s", ownerId.GetProjectDomainId(), err) - httperrors.GeneralServerError(w, err) - return - } - - total.Add(oquota) - err = total.Exceed(quota, domainQuota) - if err != nil { - // exeed domain quota - cascade, _ := body.Bool(manager.KeywordPlural(), "cascade") - if !cascade { - log.Errorf("project quota exeed domain quota: %s", err) - httperrors.OutOfQuotaError(w, "project quota exeed domain quota") - return - } else { - if allowScope != rbacutils.ScopeSystem { - httperrors.OutOfQuotaError(w, "project quota exeed domain quota, no previlige to cascade set") - return - } else { - // cascade set domain quota - err = manager.SetQuota(ctx, userCred, rbacutils.ScopeDomain, ownerId, nil, total) - if err != nil { - log.Errorf("cascade set quota fail %s", err) - httperrors.GeneralServerError(w, err) - return - } - } - } - } - } else { - total, err := manager.getDomainTotalQuota(ctx, ownerId.GetProjectDomainId(), nil) - if err != nil { - log.Errorf("get total quota fail %s", err) - httperrors.GeneralServerError(w, err) - return - } - err = total.Exceed(quota, oquota) - if err != nil { - log.Errorf("project quota exeed domain quota: %s", err) - httperrors.OutOfQuotaError(w, "project quota exeed domain quota") - return - } - } - } - - err = manager.SetQuota(ctx, userCred, scope, ownerId, nil, oquota) + err = manager.SetQuota(ctx, userCred, oquota) if err != nil { log.Errorf("set quota fail %s", err) httperrors.GeneralServerError(w, err) @@ -323,6 +262,7 @@ func (manager *SQuotaBaseManager) setQuotaHanlder(ctx context.Context, w http.Re } func (manager *SQuotaBaseManager) listDomainQuotaHanlder(ctx context.Context, w http.ResponseWriter, r *http.Request) { + _, query, _ := appsrv.FetchEnv(ctx, w, r) userCred := auth.FetchUserCredential(ctx, policy.FilterPolicyCredential) if consts.IsRbacEnabled() { @@ -338,11 +278,16 @@ func (manager *SQuotaBaseManager) listDomainQuotaHanlder(ctx context.Context, w } } - quotaList, err := manager.listQuotas(ctx, "") + refresh := jsonutils.QueryBoolean(query, "refresh", false) + quotaList, err := manager.listQuotas(ctx, userCred, "", "", refresh) if err != nil { httperrors.GeneralServerError(w, err) return } + manager.sendQuotaList(w, quotaList) +} + +func (manager *SQuotaBaseManager) sendQuotaList(w http.ResponseWriter, quotaList []jsonutils.JSONObject) { rbody := jsonutils.NewDict() rbody.Set(manager.KeywordPlural(), jsonutils.NewArray(quotaList...)) appsrv.SendJSON(w, rbody) @@ -375,218 +320,81 @@ func (manager *SQuotaBaseManager) listProjectQuotaHanlder(ctx context.Context, w } domainId := owner.GetProjectDomainId() - quotaList, err := manager.listQuotas(ctx, domainId) + refresh := jsonutils.QueryBoolean(query, "refresh", false) + quotaList, err := manager.listQuotas(ctx, userCred, domainId, "", refresh) if err != nil { httperrors.GeneralServerError(w, err) return } - rbody := jsonutils.NewDict() - rbody.Set(manager.KeywordPlural(), jsonutils.NewArray(quotaList...)) - appsrv.SendJSON(w, rbody) + manager.sendQuotaList(w, quotaList) } -func (manager *SQuotaBaseManager) getDomainTotalQuota(ctx context.Context, targetDomainId string, excludes []string) (IQuota, error) { - q := manager.Query("domain_id", "tenant_id", "platform") - q = q.Equals("domain_id", targetDomainId) - q = q.IsNotEmpty("tenant_id") - if len(excludes) > 0 { - q = q.NotIn("tenant_id", excludes) - } - // dsable platform - q = q.IsNullOrEmpty("platform") - rows, err := q.Rows() - if err != nil && err != sql.ErrNoRows { - return nil, err - } - defer rows.Close() - - ret := manager.newQuota() - for rows.Next() { - var domainId, projectId, platformStr string - err := rows.Scan(&domainId, &projectId, &platformStr) - if err != nil { - return nil, errors.Wrap(err, "scan") - } - scope := rbacutils.ScopeProject - owner := db.SOwnerId{ - DomainId: domainId, - ProjectId: projectId, - } - platform := strings.Split(platformStr, nameSeparator) - - quota := manager.newQuota() - err = manager.GetQuota(ctx, scope, &owner, platform, quota) - if err != nil { - return nil, errors.Wrap(err, "GetQuota") - } - ret.Add(quota) - } - return ret, nil -} - -func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, targetDomainId string) ([]jsonutils.JSONObject, error) { - q := manager.Query("domain_id", "tenant_id", "platform") +func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, userCred mcclient.TokenCredential, targetDomainId string, targetProjectId string, refresh bool) ([]jsonutils.JSONObject, error) { + q := manager.Query() if len(targetDomainId) > 0 { q = q.Equals("domain_id", targetDomainId) + if len(targetProjectId) > 0 { + q = q.Equals("tenant_id", targetProjectId) + }else { + q = q.IsNotEmpty("tenant_id") + } } else { // domain only + q = q.IsNotEmpty("domain_id") q = q.IsNullOrEmpty("tenant_id") } - // if tuen OFF NonDefaultDOmainProjects - // no domain quota should return - if !consts.GetNonDefaultDomainProjects() { - q = q.IsNotEmpty("tenant_id") - } - // dsable platform - q = q.IsNullOrEmpty("platform") rows, err := q.Rows() - if err != nil && err != sql.ErrNoRows { - log.Errorf("query quotas fail %s", err) - return nil, httperrors.NewInternalServerError("query quotas %s", err) + if err != nil { + if errors.Cause(err) != sql.ErrNoRows { + return nil, httperrors.NewInternalServerError("query quotas %s", err) + }else { + return nil, nil + } } defer rows.Close() + quotaList := make([]IQuota, 0) + keyList := make([]IQuotaKeys, 0) ret := make([]jsonutils.JSONObject, 0) for rows.Next() { - var domainId, projectId, platformStr string - err := rows.Scan(&domainId, &projectId, &platformStr) + quota := manager.newQuota() + err := q.Row2Struct(rows, quota) if err != nil { - log.Errorf("scan domain_id, project_id, platform error %s", err) - return nil, httperrors.NewInternalServerError("scan quotas %s", err) + return nil, errors.Wrap(err, "q.Row2Struct") } - scope := rbacutils.ScopeProject - owner := db.SOwnerId{ - DomainId: domainId, - ProjectId: projectId, - } - if len(projectId) == 0 { - scope = rbacutils.ScopeDomain - } - platform := strings.Split(platformStr, nameSeparator) - quota, _, err := manager.queryQuota(ctx, scope, &owner, platform, false) - if err != nil { - log.Errorf("query quota for %s fail %s", getMemoryStoreKey(scope, &owner, platform), err) - continue - } - if len(projectId) > 0 { - quota.Set("tenant_id", jsonutils.NewString(projectId)) - quota.Set("domain_id", jsonutils.NewString(domainId)) - // fetch without cache expiration check - project, err := db.TenantCacheManager.FetchTenantByIdWithoutExpireCheck(ctx, projectId) - if err != nil { - if errors.Cause(err) == sql.ErrNoRows { - continue - } - log.Errorf("fail to query project %s", projectId) - return nil, err - } - quota.Set("tenant", jsonutils.NewString(project.Name)) - quota.Set("project_domain", jsonutils.NewString(project.Domain)) - } else { - quota.Set("domain_id", jsonutils.NewString(domainId)) - // fetch without cache expiration check - domain, err := db.TenantCacheManager.FetchDomainByIdWithoutExpireCheck(ctx, domainId) - if err != nil { - if errors.Cause(err) == sql.ErrNoRows { - continue - } - log.Errorf("fail to query domain %s", domainId) - return nil, err - } - quota.Set("project_domain", jsonutils.NewString(domain.Name)) - } - if len(platformStr) > 0 { - quota.Set("platform", jsonutils.NewString(platformStr)) - } - ret = append(ret, quota) + quotaList = append(quotaList, quota) + keyList = append(keyList, quota.GetKeys()) } - // if no projects for a domain and NonDefaultDomainProjects is turned ON - if len(ret) == 0 && len(targetDomainId) > 0 && consts.GetNonDefaultDomainProjects() { - // return the initial quota of targetDomainId - scope := rbacutils.ScopeDomain - owner := db.SOwnerId{ - DomainId: targetDomainId, + idNameMap, err := manager.keyList2IdNameMap(ctx, keyList) + var fields []string + for i := range quotaList { + keys := quotaList[i].GetKeys() + keyJson := jsonutils.Marshal(keys).(*jsonutils.JSONDict) + if idNameMap != nil { + // no error, do check + if len(fields) == 0 { + fields = keys.Fields() + } + values := keys.Values() + for i := range fields { + if strings.HasSuffix(fields[i], "_id") && len(values[i]) > 0 { + if len(idNameMap[fields[i]][values[i]]) == 0 { + manager.DeleteAllQuotas(ctx, userCred, keys) + manager.pendingStore.DeleteAllQuotas(ctx, userCred, keys) + manager.usageStore.DeleteAllQuotas(ctx, userCred, keys) + continue + }else { + keyJson.Add(jsonutils.NewString(idNameMap[fields[i]][values[i]]), fields[i][:len(fields[i])-3]) + } + } + } } - platform := []string{} - quota, _, err := manager.queryQuota(ctx, scope, &owner, platform, false) + quotaJson, err := manager.queryQuota(ctx, quotaList[i], refresh) if err != nil { - return nil, httperrors.NewInternalServerError("query domain initial quotas %s", err) + return nil, errors.Wrap(err, "manager.queryQuota") } - quota.Set("domain_id", jsonutils.NewString(targetDomainId)) - // fetch without cache expiration check - domain, err := db.TenantCacheManager.FetchDomainByIdWithoutExpireCheck(ctx, targetDomainId) - if err != nil { - log.Errorf("fail to query target domain %s", targetDomainId) - return nil, err - } - quota.Set("project_domain", jsonutils.NewString(domain.Name)) - ret = append(ret, quota) + quotaJson.Update(keyJson) + ret = append(ret, quotaJson) } return ret, nil } - -/*func checkQuotaHanlder(ctx context.Context, w http.ResponseWriter, r *http.Request) { - userCred := auth.FetchUserCredential(ctx, policy.FilterPolicyCredential) - - isAllow := false - if consts.IsRbacEnabled() { - isAllow = policy.PolicyManager.Allow(rbacutils.ScopeSystem, userCred, consts.GetServiceType(), - policy.PolicyDelegation, policy.PolicyActionGet) == rbacutils.Allow - } else { - isAllow = userCred.IsAllow(rbacutils.ScopeSystem, consts.GetServiceType(), - policy.PolicyDelegation, policy.PolicyActionGet) - } - if !isAllow { - httperrors.ForbiddenError(w, "not allow to delegate check quota") - return - } - if consts.IsRbacEnabled() { - if policy.PolicyManager.Allow(rbacutils.ScopeSystem, userCred, consts.GetServiceType(), - _manager.Keyword(), policy.PolicyActionGet) != rbacutils.Allow { - httperrors.ForbiddenError(w, "not allow to query quota") - return - } - } - - params := appctx.AppContextParams(ctx) - - var ownerId mcclient.IIdentityProvider - - projectId := params[""] - if len(projectId) == 0 { - ownerId = userCred - } else { - tenant, err := db.TenantCacheManager.FetchTenantByIdOrName(ctx, projectId) - if err != nil { - if err == sql.ErrNoRows { - httperrors.TenantNotFoundError(w, "project %s not found", projectId) - return - } else { - httperrors.GeneralServerError(w, err) - return - } - } - ownerId = &db.SOwnerId{ - DomainId: tenant.DomainId, - Domain: tenant.Domain, - ProjectId: tenant.Id, - Project: tenant.Name, - } - } - body, err := appsrv.FetchJSON(r) - quota := _manager.newQuota() - err = body.Unmarshal(quota, _manager.Keyword()) - if err != nil { - log.Errorf("Fail to decode JSON request body: %s", err) - httperrors.InvalidInputError(w, "fail to decode body") - return - } - used, err := _manager.CheckQuota(ctx, userCred, rbacutils.ScopeProject, ownerId, quota) - if err != nil { - httperrors.OutOfQuotaError(w, "Out of quota: %s", err) - return - } - rbody := jsonutils.NewDict() - rbody.Add(used.ToJSON(""), _manager.Keyword()) - appsrv.SendJSON(w, rbody) -}*/ diff --git a/pkg/cloudcommon/db/quotas/interface.go b/pkg/cloudcommon/db/quotas/interface.go new file mode 100644 index 0000000000..ae888388dc --- /dev/null +++ b/pkg/cloudcommon/db/quotas/interface.go @@ -0,0 +1,48 @@ +package quotas + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/mcclient" +) + +type IQuotaKeys interface { + Fields() []string + Values() []string + Compare(IQuotaKeys) int +} + +type IQuota interface { + GetKeys() IQuotaKeys + SetKeys(IQuotaKeys) + + FetchSystemQuota() + FetchUsage(ctx context.Context) error + Update(quota IQuota) + Add(quota IQuota) + Sub(quota IQuota) + Exceed(request IQuota, quota IQuota) error + IsEmpty() bool + ToJSON(prefix string) jsonutils.JSONObject +} + +type IQuotaStore interface { + GetQuota(ctx context.Context, keys IQuotaKeys, quota IQuota) error + GetChildrenQuotas(ctx context.Context, keys IQuotaKeys) ([]IQuota, error) + GetParentQuotas(ctx context.Context, keys IQuotaKeys) ([]IQuota, error) + + SetQuota(ctx context.Context, userCred mcclient.TokenCredential, quota IQuota) error + AddQuota(ctx context.Context, userCred mcclient.TokenCredential, diff IQuota, target IQuota) error + SubQuota(ctx context.Context, userCred mcclient.TokenCredential, diff IQuota, target IQuota) error + + DeleteQuota(ctx context.Context, userCred mcclient.TokenCredential, keys IQuotaKeys) error + DeleteAllQuotas(ctx context.Context, userCred mcclient.TokenCredential, keys IQuotaKeys) error +} + +type IQuotaManager interface { + db.IResourceModelManager + + FetchIdNames(ctx context.Context, idMap map[string]map[string]string) (map[string]map[string]string, error) +} diff --git a/pkg/cloudcommon/db/quotas/lock.go b/pkg/cloudcommon/db/quotas/lock.go new file mode 100644 index 0000000000..7088a6440f --- /dev/null +++ b/pkg/cloudcommon/db/quotas/lock.go @@ -0,0 +1,27 @@ +package quotas + +import ( + "context" + + "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" +) + +const ( + QuotaLockName = "quotas" +) + +func LockQuota(ctx context.Context, quota IQuota) { + LockQuotaKeys(ctx, quota.GetKeys()) +} + +func ReleaseQuota(ctx context.Context, quota IQuota) { + ReleaseQuotaKeys(ctx, quota.GetKeys()) +} + +func LockQuotaKeys(ctx context.Context, keys IQuotaKeys) { + lockman.LockRawObject(ctx, QuotaLockName, QuotaKeyString(keys)) +} + +func ReleaseQuotaKeys(ctx context.Context, keys IQuotaKeys) { + lockman.ReleaseRawObject(ctx, QuotaLockName, QuotaKeyString(keys)) +} diff --git a/pkg/cloudcommon/db/quotas/models.go b/pkg/cloudcommon/db/quotas/models.go index 7533fb9f2c..50b9fee22e 100644 --- a/pkg/cloudcommon/db/quotas/models.go +++ b/pkg/cloudcommon/db/quotas/models.go @@ -18,11 +18,12 @@ import ( "context" "database/sql" "reflect" - "strings" + "sort" "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/reflectutils" + "yunion.io/x/sqlchemy" identityapi "yunion.io/x/onecloud/pkg/apis/identity" "yunion.io/x/onecloud/pkg/cloudcommon/db" @@ -40,18 +41,16 @@ type SQuotaBaseManager struct { } const ( - nameSeparator = "." - - quotaKeyword = "quota" - quotaKeywords = "quotas" + // quotaKeyword = "quota" + // quotaKeywords = "quotas" quotaUsageKeyword = "quota-usage" quotaUsageKeywords = "quota-usages" ) -func NewQuotaBaseManager(model interface{}, tableName string, pendingStore IQuotaStore, usageStore IQuotaStore) SQuotaBaseManager { +func NewQuotaBaseManager(model interface{}, tableName string, pendingStore IQuotaStore, usageStore IQuotaStore, keyword, keywordPlural string) SQuotaBaseManager { return SQuotaBaseManager{ - SResourceBaseManager: db.NewResourceBaseManager(model, tableName, quotaKeyword, quotaKeywords), + SResourceBaseManager: db.NewResourceBaseManager(model, tableName, keyword, keywordPlural), pendingStore: pendingStore, usageStore: usageStore, autoCreate: true, @@ -66,89 +65,134 @@ func NewQuotaUsageManager(model interface{}, tableName string) SQuotaBaseManager type SQuotaBase struct { db.SResourceBase - - DomainId string `width:"128" charset:"ascii" nullable:"false" primary:"true" list:"user"` - ProjectId string `name:"tenant_id" width:"128" charset:"ascii" nullable:"false" primary:"true" list:"user"` - Platform string `width:"128" charset:"utf8" nullable:"false" primary:"true" list:"user"` } -func (manager *SQuotaBaseManager) getQuotaInternal(ctx context.Context, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string, quota IQuota) error { +func (manager *SQuotaBaseManager) GetIQuotaManager() IQuotaManager { + return manager.GetIResourceModelManager().(IQuotaManager) +} + +func (manager *SQuotaBaseManager) FetchIdNames(ctx context.Context, idMap map[string]map[string]string) (map[string]map[string]string, error) { + return idMap, nil +} + +func (manager *SQuotaBaseManager) getQuotaByKeys(ctx context.Context, keys IQuotaKeys, quota IQuota) error { q := manager.Query() - q = q.Equals("domain_id", ownerId.GetProjectDomainId()) - if scope == rbacutils.ScopeProject { - q = q.Equals("tenant_id", ownerId.GetProjectId()) - } else { - q = q.IsNullOrEmpty("tenant_id") + + fields := keys.Fields() + values := keys.Values() + for i := range fields { + if len(values[i]) == 0 { + q = q.IsNullOrEmpty(fields[i]) + } else { + q = q.Equals(fields[i], values[i]) + } } - var key string - if len(platform) > 0 { - key = strings.Join(platform, nameSeparator) - } - q = q.Equals("platform", key) + err := q.First(quota) - if err != nil && err != sql.ErrNoRows { - return err - } else if err == sql.ErrNoRows && manager.autoCreate { - quota.FetchSystemQuota(scope, ownerId) - return manager.setQuotaInternal(ctx, nil, scope, ownerId, platform, quota) + if err != nil { + if errors.Cause(err) != sql.ErrNoRows { + return errors.Wrap(err, "q.First") + } } + quota.SetKeys(keys) return nil } -func (manager *SQuotaBaseManager) setQuotaInternal(ctx context.Context, userCred mcclient.TokenCredential, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string, quota IQuota) error { - base := SQuotaBase{} - base.DomainId = ownerId.GetProjectDomainId() - if scope == rbacutils.ScopeProject { - base.ProjectId = ownerId.GetProjectId() +func filterParentByKey(q *sqlchemy.SQuery, fieldName string, value string) *sqlchemy.SQuery { + if len(value) > 0 { + q = q.Filter(sqlchemy.OR( + sqlchemy.IsNullOrEmpty(q.Field(fieldName)), + sqlchemy.Equals(q.Field(fieldName), value), + )) + } else { + q = q.Filter(sqlchemy.IsNullOrEmpty(q.Field(fieldName))) } - if len(platform) > 0 { - base.Platform = strings.Join(platform, nameSeparator) - } - base.SetModelManager(manager, quota.(db.IModel)) - - if !reflectutils.FillEmbededStructValue(reflect.Indirect(reflect.ValueOf(quota)), reflect.ValueOf(base)) { - return errors.Error("no embed SBaseQuota") - } - - return manager.TableSpec().InsertOrUpdate(quota) + return q } -func (manager *SQuotaBaseManager) deleteQuotaInternal(ctx context.Context, userCred mcclient.TokenCredential, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string) error { - q := manager.Query("domain_id", "tenant_id", "platform") - q = q.Equals("domain_id", ownerId.GetProjectDomainId()) - if scope == rbacutils.ScopeProject { - q = q.Equals("tenant_id", ownerId.GetProjectId()) +func filterChildrenByKey(q *sqlchemy.SQuery, fieldName string, value string) *sqlchemy.SQuery { + if len(value) > 0 { + q = q.Equals(fieldName, value) } - if platform != nil { - q = q.Equals("platform", strings.Join(platform, nameSeparator)) + return q +} + +func (manager *SQuotaBaseManager) getQuotasInternal(ctx context.Context, keys IQuotaKeys, isParent bool) ([]IQuota, error) { + q := manager.Query() + + fields := keys.Fields() + values := keys.Values() + for i := range fields { + if isParent { + q = filterParentByKey(q, fields[i], values[i]) + } else { + q = filterChildrenByKey(q, fields[i], values[i]) + } } rows, err := q.Rows() if err != nil { - if err != sql.ErrNoRows { - return errors.Wrap(err, "sql.Rows") + if errors.Cause(err) == sql.ErrNoRows { + return nil, nil + } else { + return nil, errors.Wrap(err, "q.Rows") + } + } + defer rows.Close() + results := make([]IQuota, 0) + for rows.Next() { + r := manager.newQuota() + err := q.Row2Struct(rows, r) + if err != nil { + return nil, errors.Wrap(err, "q.Row2Struct") + } + results = append(results, r) + } + sort.Sort(TQuotaList(results)) + return results, nil +} + +func (manager *SQuotaBaseManager) setQuotaInternal(ctx context.Context, userCred mcclient.TokenCredential, quota IQuota) error { + return manager.TableSpec().InsertOrUpdate(quota) +} + +func (manager *SQuotaBaseManager) addQuotaInternal(ctx context.Context, userCred mcclient.TokenCredential, diff IQuota, quota IQuota) error { + return manager.TableSpec().Increment(diff, quota) +} + +func (manager *SQuotaBaseManager) subQuotaInternal(ctx context.Context, userCred mcclient.TokenCredential, cancel IQuota, quota IQuota) error { + return manager.TableSpec().Decrement(cancel, quota) +} + +func (manager *SQuotaBaseManager) deleteQuotaByKeys(ctx context.Context, userCred mcclient.TokenCredential, keys IQuotaKeys) error { + quota := manager.newQuota() + quota.SetKeys(keys) + _, err := db.Update(quota.(db.IModel), func() error { + return quota.(db.IModel).MarkDelete() + }) + if err != nil { + if errors.Cause(err) != sql.ErrNoRows { + return errors.Wrap(err, "Delete") + } + } + return nil +} + +func (manager *SQuotaBaseManager) deleteAllQuotas(ctx context.Context, userCred mcclient.TokenCredential, keys IQuotaKeys) error { + quotas, err := manager.getQuotasInternal(ctx, keys, false) + if err != nil { + if errors.Cause(err) != sql.ErrNoRows { + return errors.Wrap(err, "manager.getQuotasInternal") } else { return nil } } - - defer rows.Close() - - for rows.Next() { - var domainId, tenantId, platform string - err := rows.Scan(&domainId, &tenantId, &platform) + for i := range quotas { + err := manager.deleteQuotaByKeys(ctx, userCred, quotas[i].GetKeys()) if err != nil { - return errors.Wrap(err, "rows.Scan") - } - base := SQuotaBase{ - DomainId: domainId, - ProjectId: tenantId, - Platform: platform, - } - base.SetModelManager(manager, &base) - err = base.Delete(ctx, nil) - if err != nil { - return errors.Wrap(err, "Update") + if errors.Cause(err) != sql.ErrNoRows { + return errors.Wrapf(err, "manager.deleteQuotaByKeys %s", QuotaKeyString(quotas[i].GetKeys())) + } } } return nil @@ -195,18 +239,21 @@ func (manager *SQuotaBaseManager) InitializeData() error { } quota := manager.newQuota() + baseKeys := OwnerIdQuotaKeys(scope, ownerId) + if !reflectutils.FillEmbededStructValue(reflect.Indirect(reflect.ValueOf(quota)), reflect.ValueOf(baseKeys)) { + log.Fatalf("invalid quota??? fail to find SBaseQuotaKey") + } err := metaQuota.GetQuota(context.Background(), scope, ownerId, quota) if err != nil && err != sql.ErrNoRows { log.Errorf("metaQuota.GetQuota error %s for %s", err, ownerId) continue } if quota.IsEmpty() { - quota.FetchSystemQuota(scope, ownerId) + quota.FetchSystemQuota() } - baseQuota := SQuotaBase{} + baseQuota := SBaseQuotaKeys{} baseQuota.DomainId = ownerId.GetProjectDomainId() baseQuota.ProjectId = ownerId.GetProjectId() - baseQuota.SetModelManager(manager, quota.(db.IModel)) reflectutils.FillEmbededStructValue(reflect.Indirect(reflect.ValueOf(quota)), reflect.ValueOf(baseQuota)) err = manager.TableSpec().Insert(quota) diff --git a/pkg/cloudcommon/db/quotas/quotakeys.go b/pkg/cloudcommon/db/quotas/quotakeys.go new file mode 100644 index 0000000000..51a07fe4ff --- /dev/null +++ b/pkg/cloudcommon/db/quotas/quotakeys.go @@ -0,0 +1,304 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package quotas + +import ( + "strings" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/rbacutils" +) + +type SBaseQuotaKeys struct { + DomainId string `width:"64" charset:"ascii" nullable:"false" primary:"true" list:"user"` + ProjectId string `name:"tenant_id" width:"64" charset:"ascii" nullable:"false" primary:"true" list:"user"` +} + +type SCloudResourceKeys struct { + SBaseQuotaKeys + // provider + Provider string `width:"32" charset:"ascii" nullable:"false" primary:"true" list:"user"` + // brand + Brand string `width:"32" charset:"ascii" nullable:"false" primary:"true" list:"user"` + // Env + CloudEnv string `width:"32" charset:"ascii" nullable:"false" primary:"true" list:"user"` + // cloudaccount + AccountId string `width:"64" charset:"ascii" nullable:"false" primary:"true" list:"user"` + // cloudprovider + ManagerId string `width:"64" charset:"ascii" nullable:"false" primary:"true" list:"user"` +} + +type SRegionalCloudResourceKeys struct { + SCloudResourceKeys + // region + RegionId string `width:"64" charset:"ascii" nullable:"false" primary:"true" list:"user"` +} + +type SZonalCloudResourceKeys struct { + SRegionalCloudResourceKeys + // zone + ZoneId string `width:"64" charset:"ascii" nullable:"false" primary:"true" list:"user"` +} + +func (k SBaseQuotaKeys) Fields() []string { + return []string{ + "domain_id", + "tenant_id", + } +} + +func (k SCloudResourceKeys) Fields() []string { + return append(k.SBaseQuotaKeys.Fields(), + "provider", + "brand", + "cloud_env", + "account_id", + "manager_id", + ) +} + +func (k SRegionalCloudResourceKeys) Fields() []string { + return append(k.SCloudResourceKeys.Fields(), + "region_id", + ) +} + +func (k SZonalCloudResourceKeys) Fields() []string { + return append(k.SRegionalCloudResourceKeys.Fields(), + "zone_id", + ) +} + +func (k SBaseQuotaKeys) Values() []string { + return []string{ + k.DomainId, + k.ProjectId, + } +} + +func (k SCloudResourceKeys) Values() []string { + return append(k.SBaseQuotaKeys.Values(), + k.Provider, + k.Brand, + k.CloudEnv, + k.AccountId, + k.ManagerId, + ) +} + +func (k SRegionalCloudResourceKeys) Values() []string { + return append(k.SCloudResourceKeys.Values(), + k.RegionId, + ) +} + +func (k SZonalCloudResourceKeys) Values() []string { + return append(k.SRegionalCloudResourceKeys.Values(), + k.ZoneId, + ) +} + +func (k1 SBaseQuotaKeys) Compare(ik IQuotaKeys) int { + k2 := ik.(SBaseQuotaKeys) + if k1.DomainId < k2.DomainId { + return -1 + } else if k1.DomainId > k2.DomainId { + return 1 + } + if k1.ProjectId < k2.ProjectId { + return -1 + } else if k1.ProjectId > k2.ProjectId { + return 1 + } + return 0 +} + +func (k1 SCloudResourceKeys) Compare(ik IQuotaKeys) int { + k2 := ik.(SCloudResourceKeys) + r := k1.SBaseQuotaKeys.Compare(k2.SBaseQuotaKeys) + if r != 0 { + return r + } + if k1.CloudEnv < k2.CloudEnv { + return -1 + } else if k1.CloudEnv > k2.CloudEnv { + return 1 + } + if k1.Provider < k2.Provider { + return -1 + } else if k1.Provider > k2.Provider { + return 1 + } + if k1.Brand < k2.Brand { + return -1 + } else if k1.Brand > k2.Brand { + return 1 + } + return 0 +} + +func (k1 SRegionalCloudResourceKeys) Compare(ik IQuotaKeys) int { + k2 := ik.(SRegionalCloudResourceKeys) + r := k1.SCloudResourceKeys.Compare(k2.SCloudResourceKeys) + if r != 0 { + return r + } + if k1.RegionId < k2.RegionId { + return -1 + } else if k1.RegionId > k2.RegionId { + return 1 + } + return 0 +} + +func (k1 SZonalCloudResourceKeys) Compare(ik IQuotaKeys) int { + k2 := ik.(SZonalCloudResourceKeys) + r := k1.SRegionalCloudResourceKeys.Compare(k2.SRegionalCloudResourceKeys) + if r != 0 { + return r + } + if k1.ZoneId < k2.ZoneId { + return -1 + } else if k1.ZoneId > k2.ZoneId { + return 1 + } + return 0 +} + +func QuotaKeyWeight(k IQuotaKeys) uint64 { + w := uint64(0) + for i, v := range k.Values() { + if len(v) > 0 { + w += (uint64(1) << uint(i)) + } + } + return w +} + +func (k SBaseQuotaKeys) Scope() rbacutils.TRbacScope { + if len(k.DomainId) > 0 && len(k.ProjectId) > 0 { + return rbacutils.ScopeProject + } else if len(k.DomainId) > 0 && len(k.ProjectId) == 0 { + return rbacutils.ScopeDomain + } else if len(k.DomainId) == 0 && len(k.ProjectId) == 0 { + return rbacutils.ScopeSystem + } else { + return rbacutils.ScopeNone + } +} + +func (k SBaseQuotaKeys) OwnerId() mcclient.IIdentityProvider { + return &db.SOwnerId{ + DomainId: k.DomainId, + ProjectId: k.ProjectId, + } +} + +func QuotaKeyString(k IQuotaKeys) string { + return strings.Join(k.Values(), "-") +} + +func OwnerIdQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) SBaseQuotaKeys { + switch scope { + case rbacutils.ScopeDomain: + return SBaseQuotaKeys{ + DomainId: ownerId.GetProjectDomainId(), + } + case rbacutils.ScopeProject: + return SBaseQuotaKeys{ + DomainId: ownerId.GetProjectDomainId(), + ProjectId: ownerId.GetProjectId(), + } + } + return SBaseQuotaKeys{} +} + +type TQuotaKeysRelation string + +const ( + QuotaKeysContain = TQuotaKeysRelation("contain") + QuotaKeysBelong = TQuotaKeysRelation("belong") + QuotaKeysEqual = TQuotaKeysRelation("equal") + QuotaKeysExclude = TQuotaKeysRelation("exclude") +) + +func stringRelation(s1, s2 string) TQuotaKeysRelation { + if s1 == s2 { + return QuotaKeysEqual + } else if len(s1) == 0 { + return QuotaKeysContain + } else if len(s2) == 0 { + return QuotaKeysBelong + } else { + return QuotaKeysExclude + } +} + +func relation(k1, k2 IQuotaKeys) TQuotaKeysRelation { + a1 := k1.Values() + a2 := k2.Values() + relationMap := make(map[TQuotaKeysRelation]int, 0) + for i := 0; i < len(a1); i += 1 { + rel := stringRelation(a1[i], a2[i]) + if _, ok := relationMap[rel]; ok { + relationMap[rel] = relationMap[rel] + 1 + } else { + relationMap[rel] = 1 + } + } + switch len(relationMap) { + case 1: + for k := range relationMap { + return k + } + case 2: + _, equalExist := relationMap[QuotaKeysEqual] + if equalExist { + if _, ok := relationMap[QuotaKeysContain]; ok { + return QuotaKeysContain + } + if _, ok := relationMap[QuotaKeysBelong]; ok { + return QuotaKeysBelong + } + return QuotaKeysExclude + } + } + return QuotaKeysExclude +} + +type TQuotaList []IQuota + +func (a TQuotaList) Len() int { return len(a) } +func (a TQuotaList) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a TQuotaList) Less(i, j int) bool { + iKeys := a[i].GetKeys() + jKeys := a[j].GetKeys() + relation := relation(iKeys, jKeys) + switch relation { + case QuotaKeysContain: + return true + case QuotaKeysBelong: + return false + } + iw := QuotaKeyWeight(iKeys) + jw := QuotaKeyWeight(jKeys) + if iw < jw { + return true + } else if iw > jw { + return false + } + return iKeys.Compare(jKeys) < 0 +} diff --git a/pkg/cloudcommon/db/quotas/quotakeys_test.go b/pkg/cloudcommon/db/quotas/quotakeys_test.go new file mode 100644 index 0000000000..c5a18e109a --- /dev/null +++ b/pkg/cloudcommon/db/quotas/quotakeys_test.go @@ -0,0 +1,244 @@ +package quotas + +import ( + "testing" +) + +func TestRelation(t *testing.T) { + keys := []SZonalCloudResourceKeys{ + //Top := + {}, + // Domain1 := + { + SRegionalCloudRegionKeys: SRegionalCloudRegionKeys{ + SCloudResourceKeys: SCloudResourceKeys{ + SBaseQuotaKeys: SBaseQuotaKeys{ + DomainId: "domain1", + }, + }, + }, + }, + // Domain2 := + { + SRegionalCloudRegionKeys: SRegionalCloudRegionKeys{ + SCloudResourceKeys: SCloudResourceKeys{ + SBaseQuotaKeys: SBaseQuotaKeys{ + DomainId: "domain2", + }, + }, + }, + }, + // Project11 := + { + SRegionalCloudRegionKeys: SRegionalCloudRegionKeys{ + SCloudResourceKeys: SCloudResourceKeys{ + SBaseQuotaKeys: SBaseQuotaKeys{ + DomainId: "domain1", + ProjectId: "project1", + }, + }, + }, + }, + // Project12 := + { + SRegionalCloudRegionKeys: SRegionalCloudRegionKeys{ + SCloudResourceKeys: SCloudResourceKeys{ + SBaseQuotaKeys: SBaseQuotaKeys{ + DomainId: "domain1", + ProjectId: "project2", + }, + }, + }, + }, + // Project21 := + { + SRegionalCloudRegionKeys: SRegionalCloudRegionKeys{ + SCloudResourceKeys: SCloudResourceKeys{ + SBaseQuotaKeys: SBaseQuotaKeys{ + DomainId: "domain2", + ProjectId: "project1", + }, + }, + }, + }, + // Project11Region1 := + { + SRegionalCloudRegionKeys: SRegionalCloudRegionKeys{ + SCloudResourceKeys: SCloudResourceKeys{ + SBaseQuotaKeys: SBaseQuotaKeys{ + DomainId: "domain1", + ProjectId: "project1", + }, + }, + RegionId: "region1", + }, + }, + // Project11Region2 := + { + SRegionalCloudRegionKeys: SRegionalCloudRegionKeys{ + SCloudResourceKeys: SCloudResourceKeys{ + SBaseQuotaKeys: SBaseQuotaKeys{ + DomainId: "domain1", + ProjectId: "project1", + }, + }, + RegionId: "region2", + }, + }, + // Project11Aliyun := + { + SRegionalCloudRegionKeys: SRegionalCloudRegionKeys{ + SCloudResourceKeys: SCloudResourceKeys{ + SBaseQuotaKeys: SBaseQuotaKeys{ + DomainId: "domain1", + ProjectId: "project1", + }, + Provider: "Aliyun", + }, + }, + }, + // Project11AliyunRegion1 := + { + SRegionalCloudRegionKeys: SRegionalCloudRegionKeys{ + SCloudResourceKeys: SCloudResourceKeys{ + SBaseQuotaKeys: SBaseQuotaKeys{ + DomainId: "domain1", + ProjectId: "project1", + }, + Provider: "Aliyun", + }, + RegionId: "region1", + }, + }, + } + want := [][]TQuotaKeysRelation{ + { + QuotaKeysEqual, + QuotaKeysContain, + QuotaKeysContain, + QuotaKeysContain, + QuotaKeysContain, + QuotaKeysContain, + QuotaKeysContain, + QuotaKeysContain, + QuotaKeysContain, + QuotaKeysContain, + }, + { + QuotaKeysBelong, + QuotaKeysEqual, + QuotaKeysExclude, + QuotaKeysContain, + QuotaKeysContain, + QuotaKeysExclude, + QuotaKeysContain, + QuotaKeysContain, + QuotaKeysContain, + QuotaKeysContain, + }, + { + QuotaKeysBelong, + QuotaKeysExclude, + QuotaKeysEqual, + QuotaKeysExclude, + QuotaKeysExclude, + QuotaKeysContain, + QuotaKeysExclude, + QuotaKeysExclude, + QuotaKeysExclude, + QuotaKeysExclude, + }, + { + QuotaKeysBelong, + QuotaKeysBelong, + QuotaKeysExclude, + QuotaKeysEqual, + QuotaKeysExclude, + QuotaKeysExclude, + QuotaKeysContain, + QuotaKeysContain, + QuotaKeysContain, + QuotaKeysContain, + }, + { + QuotaKeysBelong, + QuotaKeysBelong, + QuotaKeysExclude, + QuotaKeysExclude, + QuotaKeysEqual, + QuotaKeysExclude, + QuotaKeysExclude, + QuotaKeysExclude, + QuotaKeysExclude, + QuotaKeysExclude, + }, + { + QuotaKeysBelong, + QuotaKeysExclude, + QuotaKeysBelong, + QuotaKeysExclude, + QuotaKeysExclude, + QuotaKeysEqual, + QuotaKeysExclude, + QuotaKeysExclude, + QuotaKeysExclude, + QuotaKeysExclude, + }, + { + QuotaKeysBelong, + QuotaKeysBelong, + QuotaKeysExclude, + QuotaKeysBelong, + QuotaKeysExclude, + QuotaKeysExclude, + QuotaKeysEqual, + QuotaKeysExclude, + QuotaKeysExclude, + QuotaKeysContain, + }, + { + QuotaKeysBelong, + QuotaKeysBelong, + QuotaKeysExclude, + QuotaKeysBelong, + QuotaKeysExclude, + QuotaKeysExclude, + QuotaKeysExclude, + QuotaKeysEqual, + QuotaKeysExclude, + QuotaKeysExclude, + }, + { + QuotaKeysBelong, + QuotaKeysBelong, + QuotaKeysExclude, + QuotaKeysBelong, + QuotaKeysExclude, + QuotaKeysExclude, + QuotaKeysExclude, + QuotaKeysExclude, + QuotaKeysEqual, + QuotaKeysContain, + }, + { + QuotaKeysBelong, + QuotaKeysBelong, + QuotaKeysExclude, + QuotaKeysBelong, + QuotaKeysExclude, + QuotaKeysExclude, + QuotaKeysExclude, + QuotaKeysExclude, + QuotaKeysExclude, + QuotaKeysEqual, + }, + } + for i := range keys { + for j := range keys { + rel := relation(keys[i], keys[j]) + if rel != want[i][j] { + t.Errorf("%#v %#v got %s want %s", keys[i], keys[j], rel, want[i][j]) + } + } + } +} diff --git a/pkg/cloudcommon/db/quotas/quotas.go b/pkg/cloudcommon/db/quotas/quotas.go index 84065c95a6..7f085201da 100644 --- a/pkg/cloudcommon/db/quotas/quotas.go +++ b/pkg/cloudcommon/db/quotas/quotas.go @@ -16,39 +16,15 @@ package quotas import ( "context" - "database/sql" "yunion.io/x/jsonutils" - "yunion.io/x/log" + "yunion.io/x/pkg/errors" "yunion.io/x/onecloud/pkg/cloudcommon/db" - "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/rbacutils" ) -const ( - METADATA_KEY = "quota" -) - -type IQuota interface { - FetchSystemQuota(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) - FetchUsage(ctx context.Context, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string) error - Update(quota IQuota) - Add(quota IQuota) - Sub(quota IQuota) - Exceed(request IQuota, quota IQuota) error - IsEmpty() bool - ToJSON(prefix string) jsonutils.JSONObject -} - -/*type SQuotaManager struct { - keyword string - quotaType reflect.Type - persistenStore IQuotaStore - pendingStore IQuotaStore -}*/ - func (manager *SQuotaBaseManager) ResourceScope() rbacutils.TRbacScope { return rbacutils.ScopeProject } @@ -62,121 +38,160 @@ func (manager *SQuotaBaseManager) newQuota() IQuota { return model.(IQuota) } -func (manager *SQuotaBaseManager) CancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string, localUsage IQuota, cancelUsage IQuota) error { - lockman.LockClass(ctx, manager, mcclient.OwnerIdString(ownerId, scope)) - defer lockman.ReleaseClass(ctx, manager, mcclient.OwnerIdString(ownerId, scope)) +func (manager *SQuotaBaseManager) CancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota) error { + LockQuota(ctx, localUsage) + defer ReleaseQuota(ctx, localUsage) - return manager._cancelPendingUsage(ctx, userCred, scope, ownerId, nil, localUsage, cancelUsage) + return manager._cancelPendingUsage(ctx, userCred, localUsage, cancelUsage) } -func (manager *SQuotaBaseManager) _cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string, localUsage IQuota, cancelUsage IQuota) error { - - quota := manager.newQuota() - err := manager.pendingStore.GetQuota(ctx, scope, ownerId, platform, quota) +func (manager *SQuotaBaseManager) _cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota) error { + pendingUsage := manager.newQuota() + err := manager.pendingStore.SubQuota(ctx, userCred, cancelUsage, pendingUsage) if err != nil { - log.Errorf("%s", err) - return err - } - quota.Sub(cancelUsage) - err = manager.pendingStore.SetQuota(ctx, userCred, scope, ownerId, platform, quota) - if err != nil { - log.Errorf("%s", err) + return errors.Wrap(err, "manager.pendingStore.SubQuota") } + // if localUsage != nil { localUsage.Sub(cancelUsage) } - - // update usage - manager.PostUsageJob(scope, ownerId, platform, nil, false, false) - return err -} - -func (manager *SQuotaBaseManager) GetPendingUsage(ctx context.Context, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string, quota IQuota) error { - return manager.pendingStore.GetQuota(ctx, scope, ownerId, nil, quota) -} - -func (manager *SQuotaBaseManager) GetQuota(ctx context.Context, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string, quota IQuota) error { - err := manager.getQuotaInternal(ctx, scope, ownerId, nil, quota) - if err != nil && err != sql.ErrNoRows { - return err + // update usages + quotas, err := manager.usageStore.GetParentQuotas(ctx, cancelUsage.GetKeys()) + if err != nil { + return errors.Wrap(err, "manager.usageStore.GetParentQuotas") + } + for i := range quotas { + cancelUsage.SetKeys(quotas[i].GetKeys()) + err := manager.usageStore.AddQuota(ctx, userCred, cancelUsage, quotas[i]) + if err != nil { + return errors.Wrap(err, "manager.usageStore.AddQuota") + } } - /*if quota.IsEmpty() && manager.autoCreate { - quota.FetchSystemQuota() - }*/ return nil } -func (manager *SQuotaBaseManager) SetQuota(ctx context.Context, userCred mcclient.TokenCredential, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string, quota IQuota) error { - lockman.LockClass(ctx, manager, mcclient.OwnerIdString(ownerId, scope)) - defer lockman.ReleaseClass(ctx, manager, mcclient.OwnerIdString(ownerId, scope)) +/*func (manager *SQuotaBaseManager) GetPendingUsage(ctx context.Context, keys SQuotaKeys, quota IQuota) error { + return manager.pendingStore.GetQuota(ctx, keys, quota) +}*/ - return manager.setQuotaInternal(ctx, userCred, scope, ownerId, nil, quota) -} - -func (manager *SQuotaBaseManager) DeleteQuota(ctx context.Context, userCred mcclient.TokenCredential, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string) error { - lockman.LockClass(ctx, manager, mcclient.OwnerIdString(ownerId, scope)) - defer lockman.ReleaseClass(ctx, manager, mcclient.OwnerIdString(ownerId, scope)) - - return manager.deleteQuotaInternal(ctx, userCred, scope, ownerId, nil) -} - -func (manager *SQuotaBaseManager) CheckQuota(ctx context.Context, userCred mcclient.TokenCredential, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string, request IQuota) (IQuota, error) { - lockman.LockClass(ctx, manager, mcclient.OwnerIdString(ownerId, scope)) - defer lockman.ReleaseClass(ctx, manager, mcclient.OwnerIdString(ownerId, scope)) - - return manager._checkQuota(ctx, userCred, scope, ownerId, nil, request) -} - -func (manager *SQuotaBaseManager) _checkQuota(ctx context.Context, userCred mcclient.TokenCredential, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string, request IQuota) (IQuota, error) { - stored := manager.newQuota() - err := manager.GetQuota(ctx, scope, ownerId, platform, stored) +func (manager *SQuotaBaseManager) GetPendingUsages(ctx context.Context, keys IQuotaKeys) ([]IQuota, error) { + quotas, err := manager.pendingStore.GetChildrenQuotas(ctx, keys) if err != nil { - log.Errorf("fail to get quota %s", err) - return nil, err + return nil, errors.Wrap(err, "manager.pendingStore.GetChildrenQuotas") } + ret := make([]IQuota, 0) + for _, q := range quotas { + if !q.IsEmpty() { + ret = append(ret, q) + } + } + if len(ret) == 0 { + return nil, nil + } + return ret, nil +} + +func (manager *SQuotaBaseManager) GetQuota(ctx context.Context, keys IQuotaKeys, quota IQuota) error { + return manager.getQuotaByKeys(ctx, keys, quota) +} + +func (manager *SQuotaBaseManager) GetChildrenQuotas(ctx context.Context, keys IQuotaKeys) ([]IQuota, error) { + return manager.getQuotasInternal(ctx, keys, false) +} + +func (manager *SQuotaBaseManager) GetParentQuotas(ctx context.Context, keys IQuotaKeys) ([]IQuota, error) { + return manager.getQuotasInternal(ctx, keys, true) +} + +func (manager *SQuotaBaseManager) SetQuota(ctx context.Context, userCred mcclient.TokenCredential, quota IQuota) error { + LockQuota(ctx, quota) + defer ReleaseQuota(ctx, quota) + + return manager.setQuotaInternal(ctx, userCred, quota) +} + +func (manager *SQuotaBaseManager) AddQuota(ctx context.Context, userCred mcclient.TokenCredential, diff IQuota, target IQuota) error { + LockQuota(ctx, diff) + defer ReleaseQuota(ctx, diff) + + return manager.addQuotaInternal(ctx, userCred, diff, target) +} +func (manager *SQuotaBaseManager) SubQuota(ctx context.Context, userCred mcclient.TokenCredential, diff IQuota, target IQuota) error { + LockQuota(ctx, diff) + defer ReleaseQuota(ctx, diff) + + return manager.subQuotaInternal(ctx, userCred, diff, target) +} + +func (manager *SQuotaBaseManager) DeleteQuota(ctx context.Context, userCred mcclient.TokenCredential, keys IQuotaKeys) error { + LockQuotaKeys(ctx, keys) + defer ReleaseQuotaKeys(ctx, keys) + + return manager.deleteQuotaByKeys(ctx, userCred, keys) +} + +func (manager *SQuotaBaseManager) DeleteAllQuotas(ctx context.Context, userCred mcclient.TokenCredential, keys IQuotaKeys) error { + LockQuotaKeys(ctx, keys) + defer ReleaseQuotaKeys(ctx, keys) + + return manager.deleteAllQuotas(ctx, userCred, keys) +} + +func (manager *SQuotaBaseManager) CheckQuota(ctx context.Context, request IQuota) error { + LockQuota(ctx, request) + defer ReleaseQuota(ctx, request) + + return manager._checkQuota(ctx, request) +} + +func (manager *SQuotaBaseManager) __checkQuota(ctx context.Context, quota IQuota, request IQuota) error { used := manager.newQuota() - err = used.FetchUsage(ctx, scope, ownerId, platform) + err := manager.usageStore.GetQuota(ctx, quota.GetKeys(), used) if err != nil { - log.Errorf("fail to get quota usage %s", err) - return nil, err + return errors.Wrap(err, "manager.usageStore.GetQuotaByKeys") } - - pending := manager.newQuota() - err = manager.GetPendingUsage(ctx, scope, ownerId, platform, pending) + pendings, err := manager.pendingStore.GetChildrenQuotas(ctx, quota.GetKeys()) if err != nil { - log.Errorf("fail to get pending usage %s", err) - return nil, err + return errors.Wrap(err, "manager.pendingStore.GetChildrenQuotas") } - - used.Add(pending) - used.Add(request) - - err = used.Exceed(request, stored) - if err != nil { - return nil, err + for i := range pendings { + used.Add(pendings[i]) } - - return used, nil + return used.Exceed(quota, request) } -func (manager *SQuotaBaseManager) CheckSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string, quota IQuota) error { - lockman.LockClass(ctx, manager, mcclient.OwnerIdString(ownerId, scope)) - defer lockman.ReleaseClass(ctx, manager, mcclient.OwnerIdString(ownerId, scope)) - - return manager._checkSetPendingQuota(ctx, userCred, scope, ownerId, nil, quota) +func (manager *SQuotaBaseManager) _checkQuota(ctx context.Context, request IQuota) error { + quotas, err := manager.GetParentQuotas(ctx, request.GetKeys()) + if err != nil { + return errors.Wrap(err, "manager.getMatchedQuotas") + } + for i := range quotas { + err := manager.__checkQuota(ctx, quotas[i], request) + if err != nil { + return errors.Wrap(err, "manager.__checkQuota") + } + } + return nil } -func (manager *SQuotaBaseManager) _checkSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string, quota IQuota) error { - _, err := manager._checkQuota(ctx, userCred, scope, ownerId, platform, quota) +func (manager *SQuotaBaseManager) CheckSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential, quota IQuota) error { + LockQuota(ctx, quota) + defer ReleaseQuota(ctx, quota) + + return manager._checkSetPendingQuota(ctx, userCred, quota, true) +} + +func (manager *SQuotaBaseManager) _checkSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential, quota IQuota, setPending bool) error { + err := manager._checkQuota(ctx, quota) if err != nil { return err } - pending := manager.newQuota() - err = manager.pendingStore.GetQuota(ctx, scope, ownerId, platform, pending) - if err != nil { - log.Errorf("GetQuota fail %s", err) - return err + if setPending { + target := manager.newQuota() + err = manager.pendingStore.AddQuota(ctx, userCred, quota, target) + if err != nil { + return errors.Wrap(err, "manager.pendingStore.AddQuota") + } } - pending.Add(quota) - return manager.pendingStore.SetQuota(ctx, userCred, scope, ownerId, platform, pending) + return nil } diff --git a/pkg/cloudcommon/db/quotas/stores.go b/pkg/cloudcommon/db/quotas/stores.go index 02d03e8130..11e551aded 100644 --- a/pkg/cloudcommon/db/quotas/stores.go +++ b/pkg/cloudcommon/db/quotas/stores.go @@ -16,7 +16,6 @@ package quotas import ( "context" - "strings" "yunion.io/x/jsonutils" @@ -25,58 +24,9 @@ import ( "yunion.io/x/onecloud/pkg/util/rbacutils" ) -type IQuotaStore interface { - GetQuota(ctx context.Context, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string, quota IQuota) error - SetQuota(ctx context.Context, userCred mcclient.TokenCredential, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string, quota IQuota) error - DeleteQuota(ctx context.Context, userCred mcclient.TokenCredential, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string) error -} - -type SMemoryQuotaStore struct { - store map[string]jsonutils.JSONObject -} - -func NewMemoryQuotaStore() *SMemoryQuotaStore { - return &SMemoryQuotaStore{ - store: make(map[string]jsonutils.JSONObject), - } -} - -func getMemoryStoreKey(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, name []string) string { - keys := make([]string, 0) - keys = append(keys, ownerId.GetProjectDomainId()) - if scope == rbacutils.ScopeProject { - keys = append(keys, ownerId.GetProjectId()) - } - if len(name) > 0 { - keys = append(keys, name...) - } - return strings.Join(keys, nameSeparator) -} - -func (self *SMemoryQuotaStore) GetQuota(ctx context.Context, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, name []string, quota IQuota) error { - key := getMemoryStoreKey(scope, ownerId, name) - json, ok := self.store[key] - if ok { - return json.Unmarshal(quota) - } - return nil -} - -func (self *SMemoryQuotaStore) SetQuota(ctx context.Context, userCred mcclient.TokenCredential, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, name []string, quota IQuota) error { - key := getMemoryStoreKey(scope, ownerId, name) - if quota.IsEmpty() { - delete(self.store, key) - } else { - self.store[key] = jsonutils.Marshal(quota) - } - return nil -} - -func (self *SMemoryQuotaStore) DeleteQuota(ctx context.Context, userCred mcclient.TokenCredential, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, name []string) error { - key := getMemoryStoreKey(scope, ownerId, name) - delete(self.store, key) - return nil -} +const ( + METADATA_KEY = "quota" +) type SDBQuotaStore struct { } @@ -106,21 +56,3 @@ func (store *SDBQuotaStore) GetQuota(ctx context.Context, scope rbacutils.TRbacS } return nil } - -/*func (store *SDBQuotaStore) SetQuota(ctx context.Context, userCred mcclient.TokenCredential, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, quota IQuota) error { - var tenant *db.STenant - var err error - - switch scope { - case rbacutils.ScopeDomain: - tenant, err = db.TenantCacheManager.FetchDomainById(ctx, ownerId.GetProjectDomainId()) - default: - tenant, err = db.TenantCacheManager.FetchTenantById(ctx, ownerId.GetProjectId()) - } - - if err != nil { - return err - } - quotaJson := jsonutils.Marshal(quota) - return tenant.SetMetadata(ctx, METADATA_KEY, quotaJson, userCred) -}*/ diff --git a/pkg/cloudcommon/db/quotas/usageworker.go b/pkg/cloudcommon/db/quotas/usageworker.go index 78ae712605..f15cf4e52b 100644 --- a/pkg/cloudcommon/db/quotas/usageworker.go +++ b/pkg/cloudcommon/db/quotas/usageworker.go @@ -17,20 +17,14 @@ package quotas import ( "context" "database/sql" - "strings" "sync" + "strings" "yunion.io/x/log" + "yunion.io/x/pkg/errors" - identityapi "yunion.io/x/onecloud/pkg/apis/identity" "yunion.io/x/onecloud/pkg/appsrv" - "yunion.io/x/onecloud/pkg/cloudcommon/consts" - "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/onecloud/pkg/mcclient/auth" - "yunion.io/x/onecloud/pkg/mcclient/modules" - "yunion.io/x/onecloud/pkg/util/httputils" - "yunion.io/x/onecloud/pkg/util/rbacutils" ) var ( @@ -41,13 +35,6 @@ var ( usageDirtyMapLock = &sync.Mutex{} ) -type sUsageCalculateJob struct { - manager *SQuotaBaseManager - scope rbacutils.TRbacScope - ownerId mcclient.IIdentityProvider - platform []string -} - func setDirty(key string) { usageDirtyMapLock.Lock() defer usageDirtyMapLock.Unlock() @@ -72,8 +59,8 @@ func isDirty(key string) bool { return false } -func (manager *SQuotaBaseManager) PostUsageJob(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string, usageChan chan IQuota, cleanEmpty bool, realTime bool) { - key := getMemoryStoreKey(scope, ownerId, platform) +func (manager *SQuotaBaseManager) PostUsageJob(keys IQuotaKeys, usageChan chan IQuota, realTime bool) { + key := QuotaKeyString(keys) setDirty(key) var worker *appsrv.SWorkerManager @@ -92,53 +79,13 @@ func (manager *SQuotaBaseManager) PostUsageJob(scope rbacutils.TRbacScope, owner clearDirty(key) usage := manager.newQuota() - err := usage.FetchUsage(ctx, scope, ownerId, platform) + usage.SetKeys(keys) + err := usage.FetchUsage(ctx) if err != nil { return } - var save bool - if usage.IsEmpty() && cleanEmpty { - // check existence of project - s := auth.GetAdminSession(ctx, consts.GetRegion(), "v1") - if scope == rbacutils.ScopeDomain { - domain, err := modules.Domains.GetById(s, ownerId.GetProjectDomainId(), nil) - if err == nil { - // update cache - domainId, _ := domain.GetString("id") - domainName, _ := domain.GetString("name") - db.TenantCacheManager.Save(ctx, domainId, domainName, identityapi.KeystoneDomainRoot, identityapi.KeystoneDomainRoot) - save = true - } else if httputils.ErrorCode(err) == 404 { - // remove cache and quota - db.TenantCacheManager.Delete(ctx, ownerId.GetProjectDomainId()) - save = false - } - } else { - proj, err := modules.Projects.GetById(s, ownerId.GetProjectId(), nil) - if err == nil { - // update cache - projId, _ := proj.GetString("id") - projName, _ := proj.GetString("name") - projDomainId, _ := proj.GetString("domain_id") - projDomain, _ := proj.GetString("project_domain") - db.TenantCacheManager.Save(ctx, projId, projName, projDomainId, projDomain) - save = true - } else if httputils.ErrorCode(err) == 404 { - // remove cache and quota - db.TenantCacheManager.Delete(ctx, ownerId.GetProjectId()) - save = false - } - } - } else { - save = true - } - if save { - manager.usageStore.SetQuota(ctx, nil, scope, ownerId, platform, usage) - } else { - manager.usageStore.DeleteQuota(ctx, nil, scope, ownerId, platform) - manager.DeleteQuota(ctx, nil, scope, ownerId, platform) - } + manager.usageStore.SetQuota(ctx, nil, usage) clearDirty(key) @@ -150,9 +97,15 @@ func (manager *SQuotaBaseManager) PostUsageJob(scope rbacutils.TRbacScope, owner func (manager *SQuotaBaseManager) CalculateQuotaUsages(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) { log.Infof("CalculateQuotaUsages") - rows, err := manager.Query("domain_id", "tenant_id", "platform").IsNullOrEmpty("platform").Rows() + quota := manager.newQuota() + keys := quota.GetKeys() + keyFields := keys.Fields() + q := manager.Query(keyFields...) + + keyList := make([]IQuotaKeys, 0) + rows, err := q.Rows() if err != nil { - if err != sql.ErrNoRows { + if errors.Cause(err) != sql.ErrNoRows { log.Errorf("query quotas fail %s", err) } return @@ -160,22 +113,62 @@ func (manager *SQuotaBaseManager) CalculateQuotaUsages(ctx context.Context, user defer rows.Close() for rows.Next() { - var domainId, projectId, platform string - err := rows.Scan(&domainId, &projectId, &platform) + quota := manager.newQuota() + err = q.Row2Struct(rows, quota) if err != nil { - log.Errorf("scan domain_id, project_id, platform error %s", err) + log.Errorf("Row2Struct fail %s", err) return } - scope := rbacutils.ScopeProject - owner := db.SOwnerId{ - DomainId: domainId, - ProjectId: projectId, + keyList = append(keyList, quota.GetKeys()) + } + + var fields []string + + idNameMap, _ := manager.keyList2IdNameMap(ctx, keyList) + for _, keys := range keyList { + if idNameMap != nil { + // no error, do check + if len(fields) == 0 { + fields = keys.Fields() + } + values := keys.Values() + for i := range fields { + if strings.HasSuffix(fields[i], "_id") && len(values[i]) > 0 && len(idNameMap[fields[i]][values[i]]) == 0 { + manager.DeleteAllQuotas(ctx, userCred, keys) + manager.pendingStore.DeleteAllQuotas(ctx, userCred, keys) + manager.usageStore.DeleteAllQuotas(ctx, userCred, keys) + continue + } + } } - if len(projectId) == 0 { - scope = rbacutils.ScopeDomain - } - platforms := strings.Split(platform, nameSeparator) - // log.Debugf("PostUsageJob %s %s %s", scope, owner, platforms) - manager.PostUsageJob(scope, &owner, platforms, nil, true, false) + manager.PostUsageJob(keys, nil, false) } } + + +func (manager *SQuotaBaseManager) keyList2IdNameMap(ctx context.Context, keyList []IQuotaKeys) (map[string]map[string]string, error) { + idMap := make(map[string]map[string]string) + var fields []string + for _, keys := range keyList { + if len(fields) == 0 { + fields = keys.Fields() + } + values := keys.Values() + for i := range fields { + if strings.HasSuffix(fields[i], "_id") && len(values[i]) > 0 { + if _, ok := idMap[fields[i]]; !ok { + idMap[fields[i]] = make(map[string]string) + } + if _, ok := idMap[fields[i]][values[i]]; !ok { + idMap[fields[i]][values[i]] = "" + } + } + } + } + ret, err := manager.GetIQuotaManager().FetchIdNames(ctx, idMap) + if err != nil { + return nil, err + }else { + return ret, nil + } +} \ No newline at end of file diff --git a/pkg/cloudcommon/db/resourcebase.go b/pkg/cloudcommon/db/resourcebase.go index 309e733c55..9af84936e7 100644 --- a/pkg/cloudcommon/db/resourcebase.go +++ b/pkg/cloudcommon/db/resourcebase.go @@ -18,9 +18,12 @@ import ( "context" "time" + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/timeutils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -108,3 +111,12 @@ func (model *SResourceBase) MarkUnDelete() error { func (model *SResourceBase) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { return DeleteModel(ctx, userCred, model.GetIResourceModel()) } + +func (manager *SResourceBaseManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input apis.ResourceBaseCreateInput) (apis.ResourceBaseCreateInput, error) { + var err error + input.ModelBaseCreateInput, err = manager.SModelBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.ModelBaseCreateInput) + if err != nil { + return input, errors.Wrap(err, "SModelBaseManager.ValidateCreateData") + } + return input, nil +} diff --git a/pkg/cloudcommon/db/sharablevirtual.go b/pkg/cloudcommon/db/sharablevirtual.go index a1f0447ecd..3e8a67fbf2 100644 --- a/pkg/cloudcommon/db/sharablevirtual.go +++ b/pkg/cloudcommon/db/sharablevirtual.go @@ -19,6 +19,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" @@ -312,3 +313,12 @@ func (model *SSharableVirtualResourceBase) GetExtraDetails(ctx context.Context, } return model.getMoreDetails(ctx, userCred, query, extra), nil } + +func (manager *SSharableVirtualResourceBaseManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input apis.SharableVirtualResourceCreateInput) (apis.SharableVirtualResourceCreateInput, error) { + var err error + input.VirtualResourceCreateInput, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.VirtualResourceCreateInput) + if err != nil { + return input, errors.Wrap(err, "manager.VirtualResourceBaseManager.ValidateCreateData") + } + return input, nil +} diff --git a/pkg/cloudcommon/db/standalone.go b/pkg/cloudcommon/db/standalone.go index e36e1e2833..abcb66556b 100644 --- a/pkg/cloudcommon/db/standalone.go +++ b/pkg/cloudcommon/db/standalone.go @@ -470,6 +470,15 @@ func (model *SStandaloneResourceBase) AppendDescription(userCred mcclient.TokenC return nil } +func (manager *SStandaloneResourceBaseManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input apis.StandaloneResourceCreateInput) (apis.StandaloneResourceCreateInput, error) { + var err error + input.ResourceBaseCreateInput, err = manager.SResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.ResourceBaseCreateInput) + if err != nil { + return input, errors.Wrap(err, "SResourceBaseManager.ValidateCreateData") + } + return input, nil +} + /* func (model SStandaloneResourceBase) GetExternalId() string { return model.ExternalId diff --git a/pkg/cloudcommon/db/statusstandalone.go b/pkg/cloudcommon/db/statusstandalone.go index e696171629..4af977565b 100644 --- a/pkg/cloudcommon/db/statusstandalone.go +++ b/pkg/cloudcommon/db/statusstandalone.go @@ -19,8 +19,10 @@ import ( "fmt" "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/apis" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/rbacutils" @@ -89,3 +91,12 @@ func (model *SStatusStandaloneResourceBase) GetDetailsStatus(ctx context.Context ret.Add(jsonutils.NewString(model.Status), "status") return ret, nil } + +func (manager *SStatusStandaloneResourceBaseManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input apis.StatusStandaloneResourceCreateInput) (apis.StatusStandaloneResourceCreateInput, error) { + var err error + input.StandaloneResourceCreateInput, err = manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.StandaloneResourceCreateInput) + if err != nil { + return input, errors.Wrap(err, "SStandaloneResourceBaseManager.ValidateCreateData") + } + return input, nil +} diff --git a/pkg/cloudcommon/db/taskman/interface.go b/pkg/cloudcommon/db/taskman/interface.go index 998e812749..e849c49032 100644 --- a/pkg/cloudcommon/db/taskman/interface.go +++ b/pkg/cloudcommon/db/taskman/interface.go @@ -39,6 +39,6 @@ type ITask interface { SetStageComplete(ctx context.Context, data *jsonutils.JSONDict) SetStageFailed(ctx context.Context, reason string) - GetPendingUsage(quota quotas.IQuota) error - ClearPendingUsage() error + GetPendingUsage(quota quotas.IQuota, index int) error + ClearPendingUsage(index int) error } diff --git a/pkg/cloudcommon/db/taskman/tasks.go b/pkg/cloudcommon/db/taskman/tasks.go index e7347350e4..239935234d 100644 --- a/pkg/cloudcommon/db/taskman/tasks.go +++ b/pkg/cloudcommon/db/taskman/tasks.go @@ -21,11 +21,13 @@ import ( "net/http" "reflect" "runtime/debug" + "strconv" "strings" "time" "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/util/reflectutils" "yunion.io/x/pkg/util/stringutils" "yunion.io/x/pkg/util/timeutils" @@ -178,12 +180,23 @@ func (self *STask) GetName() string { return self.TaskName } -func fetchTaskParams(ctx context.Context, taskName string, taskData *jsonutils.JSONDict, - parentTaskId string, parentTaskNotifyUrl string, - pendingUsage quotas.IQuota) *jsonutils.JSONDict { +func fetchTaskParams( + ctx context.Context, + taskName string, + taskData *jsonutils.JSONDict, + parentTaskId string, + parentTaskNotifyUrl string, + pendingUsages []quotas.IQuota, +) *jsonutils.JSONDict { var data *jsonutils.JSONDict if taskData != nil { - data = taskData.CopyExcludes(PARENT_TASK_ID_KEY, PARENT_TASK_NOTIFY_KEY, PENDING_USAGE_KEY) + excludeKeys := []string{ + PARENT_TASK_ID_KEY, PARENT_TASK_NOTIFY_KEY, PENDING_USAGE_KEY, + } + for i := 1; taskData.Contains(pendingUsageKey(i)); i += 1 { + excludeKeys = append(excludeKeys, pendingUsageKey(i)) + } + data = taskData.CopyExcludes(excludeKeys...) } else { data = jsonutils.NewDict() } @@ -210,16 +223,29 @@ func fetchTaskParams(ctx context.Context, taskName string, taskData *jsonutils.J } } } - if pendingUsage != nil { - data.Add(jsonutils.Marshal(pendingUsage), PENDING_USAGE_KEY) + if len(pendingUsages) > 0 { + for i := range pendingUsages { + pendingUsage := pendingUsages[i] + if gotypes.IsNil(pendingUsage) || pendingUsage.IsEmpty() { + continue + } + key := pendingUsageKey(i) + data.Add(jsonutils.Marshal(pendingUsage), key) + } } return data } -func (manager *STaskManager) NewTask(ctx context.Context, taskName string, obj db.IStandaloneModel, - userCred mcclient.TokenCredential, taskData *jsonutils.JSONDict, - parentTaskId string, parentTaskNotifyUrl string, - pendingUsage quotas.IQuota) (*STask, error) { +func (manager *STaskManager) NewTask( + ctx context.Context, + taskName string, + obj db.IStandaloneModel, + userCred mcclient.TokenCredential, + taskData *jsonutils.JSONDict, + parentTaskId string, + parentTaskNotifyUrl string, + pendingUsage ...quotas.IQuota, +) (*STask, error) { if !isTaskExist(taskName) { return nil, fmt.Errorf("task %s not found", taskName) } @@ -253,10 +279,16 @@ func (manager *STaskManager) NewTask(ctx context.Context, taskName string, obj d return &task, nil } -func (manager *STaskManager) NewParallelTask(ctx context.Context, taskName string, objs []db.IStandaloneModel, - userCred mcclient.TokenCredential, taskData *jsonutils.JSONDict, - parentTaskId string, parentTaskNotifyUrl string, - pendingUsage quotas.IQuota) (*STask, error) { +func (manager *STaskManager) NewParallelTask( + ctx context.Context, + taskName string, + objs []db.IStandaloneModel, + userCred mcclient.TokenCredential, + taskData *jsonutils.JSONDict, + parentTaskId string, + parentTaskNotifyUrl string, + pendingUsage ...quotas.IQuota, +) (*STask, error) { if !isTaskExist(taskName) { return nil, fmt.Errorf("task %s not found", taskName) } @@ -675,18 +707,28 @@ func (self *STask) IsCurrentStageComplete() bool { } } -func (self *STask) GetPendingUsage(quota quotas.IQuota) error { - quotaJson, err := self.Params.Get(PENDING_USAGE_KEY) +func (self *STask) GetPendingUsage(quota quotas.IQuota, index int) error { + key := pendingUsageKey(index) + quotaJson, err := self.Params.Get(key) if err != nil { return err } return quotaJson.Unmarshal(quota) } -func (self *STask) SetPendingUsage(quota quotas.IQuota) error { +func pendingUsageKey(index int) string { + key := PENDING_USAGE_KEY + if index > 0 { + key += "." + strconv.FormatInt(int64(index), 10) + } + return key +} + +func (self *STask) SetPendingUsage(quota quotas.IQuota, index int) error { _, err := db.Update(self, func() error { - params := self.Params.CopyExcludes(PENDING_USAGE_KEY) - params.Add(jsonutils.Marshal(quota), PENDING_USAGE_KEY) + key := pendingUsageKey(index) + params := self.Params.CopyExcludes(key) + params.Add(jsonutils.Marshal(quota), key) self.Params = params return nil }) @@ -696,9 +738,10 @@ func (self *STask) SetPendingUsage(quota quotas.IQuota) error { return err } -func (self *STask) ClearPendingUsage() error { +func (self *STask) ClearPendingUsage(index int) error { _, err := db.Update(self, func() error { - params := self.Params.CopyExcludes(PENDING_USAGE_KEY) + key := pendingUsageKey(index) + params := self.Params.CopyExcludes(key) self.Params = params return nil }) diff --git a/pkg/cloudcommon/db/virtualresource.go b/pkg/cloudcommon/db/virtualresource.go index 31702fdd58..9b64373ac8 100644 --- a/pkg/cloudcommon/db/virtualresource.go +++ b/pkg/cloudcommon/db/virtualresource.go @@ -21,6 +21,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/timeutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" @@ -163,12 +164,16 @@ func (manager *SVirtualResourceBaseManager) FetchByIdOrName(userCred mcclient.II return FetchByIdOrName(manager, userCred, idStr) } -func (manager *SVirtualResourceBaseManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - isSystem, err := data.Bool("is_system") - if err == nil && isSystem && !IsAdminAllowCreate(userCred, manager) { - return nil, httperrors.NewNotSufficientPrivilegeError("non-admin user not allowed to create system object") +func (manager *SVirtualResourceBaseManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input apis.VirtualResourceCreateInput) (apis.VirtualResourceCreateInput, error) { + var err error + if input.IsSystem != nil && *input.IsSystem && !IsAdminAllowCreate(userCred, manager) { + return input, httperrors.NewNotSufficientPrivilegeError("non-admin user not allowed to create system object") } - return manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + input.StatusStandaloneResourceCreateInput, err = manager.SStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.StatusStandaloneResourceCreateInput) + if err != nil { + return input, errors.Wrap(err, "SStatusStandaloneResourceBaseManager.ValidateCreateData") + } + return input, nil } func (model *SVirtualResourceBase) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error { diff --git a/pkg/cloudcommon/options/options.go b/pkg/cloudcommon/options/options.go index 8165394c01..f7e632321c 100644 --- a/pkg/cloudcommon/options/options.go +++ b/pkg/cloudcommon/options/options.go @@ -36,6 +36,12 @@ import ( "yunion.io/x/onecloud/pkg/util/atexit" ) +const ( + DefaultQuotaUnlimit = "unlimit" + DefaultQuotaZero = "zero" + DefaultQuotaDefault = "default" +) + type BaseOptions struct { Region string `help:"Region name or ID" alias:"auth-region"` @@ -70,6 +76,8 @@ type BaseOptions struct { IsSlaveNode bool `help:"Region service slave node"` CronJobWorkerCount int `help:"Cron job worker count" default:"4"` + DefaultQuotaValue string `help:"default quota value" choices:"unlimit|zero|default"` + CalculateQuotaUsageIntervalSeconds int `help:"interval to calculate quota usages, default 30 minutes" default:"900"` NonDefaultDomainProjects bool `help:"allow projects in non-default domains" default:"false"` diff --git a/pkg/cloudprovider/fakeregion.go b/pkg/cloudprovider/fakeregion.go index 01757fe190..4504f6b78e 100644 --- a/pkg/cloudprovider/fakeregion.go +++ b/pkg/cloudprovider/fakeregion.go @@ -41,6 +41,10 @@ func (region *SFakeOnPremiseRegion) GetStatus() string { return "available" } +func (region *SFakeOnPremiseRegion) GetCloudEnv() string { + return "" +} + func (region *SFakeOnPremiseRegion) Refresh() error { return nil } diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index 94df375d25..999c2d7bea 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -132,6 +132,7 @@ type ICloudRegion interface { GetIElasticcacheById(id string) (ICloudElasticcache, error) CreateIElasticcaches(ec *SCloudElasticCacheInput) (ICloudElasticcache, error) + GetCloudEnv() string GetProvider() string GetICloudEvents(start time.Time, end time.Time, withReadEvent bool) ([]ICloudEvent, error) //获取公有云操作日志接口 diff --git a/pkg/compute/guestdrivers/aliyun.go b/pkg/compute/guestdrivers/aliyun.go index c57da04266..6d98706a27 100644 --- a/pkg/compute/guestdrivers/aliyun.go +++ b/pkg/compute/guestdrivers/aliyun.go @@ -21,11 +21,13 @@ import ( "yunion.io/x/pkg/utils" api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/billing" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) type SAliyunGuestDriver struct { @@ -45,11 +47,13 @@ func (self *SAliyunGuestDriver) GetProvider() string { return api.CLOUD_PROVIDER_ALIYUN } -func (self *SAliyunGuestDriver) GetQuotaPlatformID() []string { - return []string{ - api.CLOUD_ENV_PUBLIC_CLOUD, - api.CLOUD_PROVIDER_ALIYUN, - } +func (self *SAliyunGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys { + keys := models.SComputeResourceKeys{} + keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId) + keys.CloudEnv = api.CLOUD_ENV_PUBLIC_CLOUD + keys.Provider = api.CLOUD_PROVIDER_ALIYUN + // ignore brand + return keys } func (self *SAliyunGuestDriver) GetDefaultSysDiskBackend() string { diff --git a/pkg/compute/guestdrivers/aws.go b/pkg/compute/guestdrivers/aws.go index d04ab110a7..eb2cebe8ec 100644 --- a/pkg/compute/guestdrivers/aws.go +++ b/pkg/compute/guestdrivers/aws.go @@ -26,11 +26,13 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/billing" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) type SAwsGuestDriver struct { @@ -92,11 +94,13 @@ func (self *SAwsGuestDriver) GetProvider() string { return api.CLOUD_PROVIDER_AWS } -func (self *SAwsGuestDriver) GetQuotaPlatformID() []string { - return []string{ - api.CLOUD_ENV_PUBLIC_CLOUD, - api.CLOUD_PROVIDER_AWS, - } +func (self *SAwsGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys { + keys := models.SComputeResourceKeys{} + keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId) + keys.CloudEnv = api.CLOUD_ENV_PUBLIC_CLOUD + keys.Provider = api.CLOUD_PROVIDER_AWS + // ignore brand + return keys } func (self *SAwsGuestDriver) GetDefaultSysDiskBackend() string { diff --git a/pkg/compute/guestdrivers/azure.go b/pkg/compute/guestdrivers/azure.go index ad6e776841..77286f6592 100644 --- a/pkg/compute/guestdrivers/azure.go +++ b/pkg/compute/guestdrivers/azure.go @@ -22,11 +22,13 @@ import ( "yunion.io/x/pkg/utils" api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/billing" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) type SAzureGuestDriver struct { @@ -46,11 +48,13 @@ func (self *SAzureGuestDriver) GetProvider() string { return api.CLOUD_PROVIDER_AZURE } -func (self *SAzureGuestDriver) GetQuotaPlatformID() []string { - return []string{ - api.CLOUD_ENV_PUBLIC_CLOUD, - api.CLOUD_PROVIDER_AZURE, - } +func (self *SAzureGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys { + keys := models.SComputeResourceKeys{} + keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId) + keys.CloudEnv = api.CLOUD_ENV_PUBLIC_CLOUD + keys.Provider = api.CLOUD_PROVIDER_AZURE + // ignore brand + return keys } func (self *SAzureGuestDriver) GetDefaultSysDiskBackend() string { diff --git a/pkg/compute/guestdrivers/baremetals.go b/pkg/compute/guestdrivers/baremetals.go index 705ca3e5b7..66457f1c0c 100644 --- a/pkg/compute/guestdrivers/baremetals.go +++ b/pkg/compute/guestdrivers/baremetals.go @@ -36,6 +36,7 @@ import ( "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/logclient" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) type SBaremetalGuestDriver struct { @@ -55,12 +56,14 @@ func (self *SBaremetalGuestDriver) GetProvider() string { return api.CLOUD_PROVIDER_ONECLOUD } -func (self *SBaremetalGuestDriver) GetQuotaPlatformID() []string { - return []string{ - api.CLOUD_ENV_ON_PREMISE, - api.CLOUD_PROVIDER_ONECLOUD, - api.HYPERVISOR_BAREMETAL, - } +func (self *SBaremetalGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys { + keys := models.SComputeResourceKeys{} + keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId) + keys.CloudEnv = api.CLOUD_ENV_ON_PREMISE + keys.Provider = api.CLOUD_PROVIDER_ONECLOUD + // ignore brand + keys.Hypervisor = api.HYPERVISOR_BAREMETAL + return keys } func (self *SBaremetalGuestDriver) GetDefaultSysDiskBackend() string { diff --git a/pkg/compute/guestdrivers/container.go b/pkg/compute/guestdrivers/container.go index bb5abb50cb..f66161329d 100644 --- a/pkg/compute/guestdrivers/container.go +++ b/pkg/compute/guestdrivers/container.go @@ -22,12 +22,14 @@ import ( "yunion.io/x/log" api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/httputils" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) var ( @@ -55,12 +57,14 @@ func (self *SContainerDriver) GetProvider() string { return api.CLOUD_PROVIDER_ONECLOUD } -func (self *SContainerDriver) GetQuotaPlatformID() []string { - return []string{ - api.CLOUD_ENV_ON_PREMISE, - api.CLOUD_PROVIDER_ONECLOUD, - api.HYPERVISOR_CONTAINER, - } +// for backward compatibility, deprecated driver +func (self *SContainerDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys { + keys := models.SComputeResourceKeys{} + keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId) + keys.CloudEnv = api.CLOUD_ENV_ON_PREMISE + keys.Provider = api.CLOUD_PROVIDER_ONECLOUD + keys.Hypervisor = api.HYPERVISOR_CONTAINER + return keys } func (self *SContainerDriver) GetDefaultSysDiskBackend() string { diff --git a/pkg/compute/guestdrivers/ctyun.go b/pkg/compute/guestdrivers/ctyun.go index efc03fe38c..48b4943e04 100644 --- a/pkg/compute/guestdrivers/ctyun.go +++ b/pkg/compute/guestdrivers/ctyun.go @@ -22,6 +22,9 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/util/rbacutils" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" ) type SCtyunGuestDriver struct { @@ -36,11 +39,13 @@ func (self *SCtyunGuestDriver) GetProvider() string { return api.CLOUD_PROVIDER_CTYUN } -func (self *SCtyunGuestDriver) GetQuotaPlatformID() []string { - return []string{ - api.CLOUD_ENV_PUBLIC_CLOUD, - api.CLOUD_PROVIDER_CTYUN, - } +func (self *SCtyunGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys { + keys := models.SComputeResourceKeys{} + keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId) + keys.CloudEnv = api.CLOUD_ENV_PUBLIC_CLOUD + keys.Provider = api.CLOUD_PROVIDER_CTYUN + // ignore brand + return keys } func (self *SCtyunGuestDriver) GetDefaultSysDiskBackend() string { diff --git a/pkg/compute/guestdrivers/esxi.go b/pkg/compute/guestdrivers/esxi.go index 74969573a0..fbabdbd083 100644 --- a/pkg/compute/guestdrivers/esxi.go +++ b/pkg/compute/guestdrivers/esxi.go @@ -25,6 +25,7 @@ import ( "yunion.io/x/pkg/utils" api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/compute/options" @@ -33,6 +34,7 @@ import ( "yunion.io/x/onecloud/pkg/multicloud/esxi" "yunion.io/x/onecloud/pkg/util/billing" "yunion.io/x/onecloud/pkg/util/httputils" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) type SESXiGuestDriver struct { @@ -58,12 +60,14 @@ func (self *SESXiGuestDriver) GetProvider() string { return api.CLOUD_PROVIDER_VMWARE } -func (self *SESXiGuestDriver) GetQuotaPlatformID() []string { - return []string{ - api.CLOUD_ENV_ON_PREMISE, - api.CLOUD_PROVIDER_VMWARE, - api.HYPERVISOR_ESXI, - } +func (self *SESXiGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys { + keys := models.SComputeResourceKeys{} + keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId) + keys.CloudEnv = api.CLOUD_ENV_ON_PREMISE + keys.Provider = api.CLOUD_PROVIDER_VMWARE + // ignore brand + // ignore hypervisor keys.Hypervisor = api.HYPERVISOR_ESXI + return keys } func (self *SESXiGuestDriver) GetDefaultSysDiskBackend() string { diff --git a/pkg/compute/guestdrivers/google.go b/pkg/compute/guestdrivers/google.go index a1d0aba747..841db708b9 100644 --- a/pkg/compute/guestdrivers/google.go +++ b/pkg/compute/guestdrivers/google.go @@ -16,7 +16,10 @@ package guestdrivers import ( api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) type SGoogleGuestDriver struct { @@ -28,11 +31,13 @@ func init() { models.RegisterGuestDriver(&driver) } -func (self *SGoogleGuestDriver) GetQuotaPlatformID() []string { - return []string{ - api.CLOUD_ENV_PUBLIC_CLOUD, - api.CLOUD_PROVIDER_GOOGLE, - } +func (self *SGoogleGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys { + keys := models.SComputeResourceKeys{} + keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId) + keys.CloudEnv = api.CLOUD_ENV_PUBLIC_CLOUD + keys.Provider = api.CLOUD_PROVIDER_GOOGLE + // ignore brand + return keys } func (self *SGoogleGuestDriver) GetHypervisor() string { diff --git a/pkg/compute/guestdrivers/huawei.go b/pkg/compute/guestdrivers/huawei.go index 2d902b63cf..3f96f196a2 100644 --- a/pkg/compute/guestdrivers/huawei.go +++ b/pkg/compute/guestdrivers/huawei.go @@ -20,9 +20,12 @@ import ( "yunion.io/x/pkg/utils" api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/billing" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) type SHuaweiGuestDriver struct { @@ -42,11 +45,13 @@ func (self *SHuaweiGuestDriver) GetProvider() string { return api.CLOUD_PROVIDER_HUAWEI } -func (self *SHuaweiGuestDriver) GetQuotaPlatformID() []string { - return []string{ - api.CLOUD_ENV_PUBLIC_CLOUD, - api.CLOUD_PROVIDER_HUAWEI, - } +func (self *SHuaweiGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys { + keys := models.SComputeResourceKeys{} + keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId) + keys.CloudEnv = api.CLOUD_ENV_PUBLIC_CLOUD + keys.Provider = api.CLOUD_PROVIDER_HUAWEI + // ignore brand + return keys } func (self *SHuaweiGuestDriver) GetDefaultSysDiskBackend() string { diff --git a/pkg/compute/guestdrivers/kvm.go b/pkg/compute/guestdrivers/kvm.go index 69a371775e..ca78b06e29 100644 --- a/pkg/compute/guestdrivers/kvm.go +++ b/pkg/compute/guestdrivers/kvm.go @@ -27,12 +27,14 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/httputils" "yunion.io/x/onecloud/pkg/util/logclient" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) type SKVMGuestDriver struct { @@ -52,12 +54,14 @@ func (self *SKVMGuestDriver) GetProvider() string { return api.CLOUD_PROVIDER_ONECLOUD } -func (self *SKVMGuestDriver) GetQuotaPlatformID() []string { - return []string{ - api.CLOUD_ENV_ON_PREMISE, - api.CLOUD_PROVIDER_ONECLOUD, - api.HYPERVISOR_KVM, - } +func (self *SKVMGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys { + keys := models.SComputeResourceKeys{} + keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId) + keys.CloudEnv = api.CLOUD_ENV_ON_PREMISE + keys.Provider = api.CLOUD_PROVIDER_ONECLOUD + // ignore brand keys.Brand = brand + keys.Hypervisor = api.HYPERVISOR_KVM + return keys } func (self *SKVMGuestDriver) GetDefaultSysDiskBackend() string { diff --git a/pkg/compute/guestdrivers/openstack.go b/pkg/compute/guestdrivers/openstack.go index 5c713b32cf..6ae4c893e7 100644 --- a/pkg/compute/guestdrivers/openstack.go +++ b/pkg/compute/guestdrivers/openstack.go @@ -22,12 +22,14 @@ import ( "yunion.io/x/pkg/utils" api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/billing" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) type SOpenStackGuestDriver struct { @@ -55,11 +57,13 @@ func (self *SOpenStackGuestDriver) GetProvider() string { return api.CLOUD_PROVIDER_OPENSTACK } -func (self *SOpenStackGuestDriver) GetQuotaPlatformID() []string { - return []string{ - api.CLOUD_ENV_PRIVATE_CLOUD, - api.CLOUD_PROVIDER_OPENSTACK, - } +func (self *SOpenStackGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys { + keys := models.SComputeResourceKeys{} + keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId) + keys.CloudEnv = api.CLOUD_ENV_PRIVATE_CLOUD + keys.Provider = api.CLOUD_PROVIDER_OPENSTACK + keys.Brand = brand + return keys } func (self *SOpenStackGuestDriver) IsSupportEip() bool { diff --git a/pkg/compute/guestdrivers/qcloud.go b/pkg/compute/guestdrivers/qcloud.go index d8b721d1f7..7abc0e5eb2 100644 --- a/pkg/compute/guestdrivers/qcloud.go +++ b/pkg/compute/guestdrivers/qcloud.go @@ -21,11 +21,13 @@ import ( "yunion.io/x/pkg/utils" api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/billing" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) type SQcloudGuestDriver struct { @@ -45,11 +47,14 @@ func (self *SQcloudGuestDriver) GetProvider() string { return api.CLOUD_PROVIDER_QCLOUD } -func (self *SQcloudGuestDriver) GetQuotaPlatformID() []string { - return []string{ - api.CLOUD_ENV_PUBLIC_CLOUD, - api.CLOUD_PROVIDER_QCLOUD, - } +func (self *SQcloudGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys { + keys := models.SComputeResourceKeys{} + keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId) + keys.CloudEnv = api.CLOUD_ENV_PUBLIC_CLOUD + keys.Provider = api.CLOUD_PROVIDER_QCLOUD + // ignore brand keys.Brand = brand + // ignore hypervisor + return keys } func (self *SQcloudGuestDriver) GetDefaultSysDiskBackend() string { diff --git a/pkg/compute/guestdrivers/ucloud.go b/pkg/compute/guestdrivers/ucloud.go index 1a5c562948..b8b89c7920 100644 --- a/pkg/compute/guestdrivers/ucloud.go +++ b/pkg/compute/guestdrivers/ucloud.go @@ -20,8 +20,11 @@ import ( "yunion.io/x/pkg/utils" api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) type SUCloudGuestDriver struct { @@ -36,11 +39,14 @@ func (self *SUCloudGuestDriver) GetProvider() string { return api.CLOUD_PROVIDER_UCLOUD } -func (self *SUCloudGuestDriver) GetQuotaPlatformID() []string { - return []string{ - api.CLOUD_ENV_PUBLIC_CLOUD, - api.CLOUD_PROVIDER_UCLOUD, - } +func (self *SUCloudGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys { + keys := models.SComputeResourceKeys{} + keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId) + keys.CloudEnv = api.CLOUD_ENV_PUBLIC_CLOUD + keys.Provider = api.CLOUD_PROVIDER_UCLOUD + // ignore brand + // ignore hypervisor + return keys } func (self *SUCloudGuestDriver) GetDefaultSysDiskBackend() string { diff --git a/pkg/compute/guestdrivers/zstack.go b/pkg/compute/guestdrivers/zstack.go index 372291cb0b..73be144540 100644 --- a/pkg/compute/guestdrivers/zstack.go +++ b/pkg/compute/guestdrivers/zstack.go @@ -22,11 +22,13 @@ import ( "yunion.io/x/pkg/utils" api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/billing" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) type SZStackGuestDriver struct { @@ -54,11 +56,14 @@ func (self *SZStackGuestDriver) GetProvider() string { return api.CLOUD_PROVIDER_ZSTACK } -func (self *SZStackGuestDriver) GetQuotaPlatformID() []string { - return []string{ - api.CLOUD_ENV_PRIVATE_CLOUD, - api.CLOUD_PROVIDER_ZSTACK, - } +func (self *SZStackGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys { + keys := models.SComputeResourceKeys{} + keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId) + keys.CloudEnv = api.CLOUD_ENV_PRIVATE_CLOUD + keys.Provider = api.CLOUD_PROVIDER_ZSTACK + keys.Brand = brand + // ignore hypervisor + return keys } func (self *SZStackGuestDriver) GetDefaultSysDiskBackend() string { diff --git a/pkg/compute/hostdrivers/base.go b/pkg/compute/hostdrivers/base.go index 02d9224288..a571bf0a8d 100644 --- a/pkg/compute/hostdrivers/base.go +++ b/pkg/compute/hostdrivers/base.go @@ -75,13 +75,14 @@ func (self *SBaseHostDriver) PrepareConvert(host *models.SHost, image, raid stri ServerConfigs: &api.ServerConfigs{ PreferHost: host.Id, }, - Description: "Baremetal convered Hypervisor", - VcpuCount: int(host.CpuCount), - VmemSize: host.MemSize, - AutoStart: true, - IsSystem: true, - Baremetal: true, + VcpuCount: int(host.CpuCount), + VmemSize: host.MemSize, + AutoStart: true, + Baremetal: true, } + params.Description = "Baremetal convered Hypervisor" + isSystem := true + params.IsSystem = &isSystem name, err := data.GetString("name") if err == nil { params.Name = name diff --git a/pkg/compute/models/baremetalagents.go b/pkg/compute/models/baremetalagents.go index 69fecfef85..ca0f2c70ee 100644 --- a/pkg/compute/models/baremetalagents.go +++ b/pkg/compute/models/baremetalagents.go @@ -23,6 +23,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/util/regutils" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" @@ -118,7 +119,17 @@ func (manager *SBaremetalagentManager) ValidateCreateData(ctx context.Context, u if count > 0 { return nil, httperrors.NewDuplicateResourceError("Duplicate manager_uri %s", mangerUri) } - return manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + input := apis.StandaloneResourceCreateInput{} + err = data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal StandaloneResourceCreateInput fail %s", err) + } + input, err = manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { + return nil, err + } + data.Update(jsonutils.Marshal(input)) + return data, nil } func (self *SBaremetalagent) AllowPerformEnable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { diff --git a/pkg/compute/models/buckets.go b/pkg/compute/models/buckets.go index a9aa624aec..d18ce8d64d 100644 --- a/pkg/compute/models/buckets.go +++ b/pkg/compute/models/buckets.go @@ -34,6 +34,7 @@ import ( "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/validators" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -391,9 +392,10 @@ func (manager *SBucketManager) ValidateCreateData( userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, - input *api.BucketCreateInput, -) (*jsonutils.JSONDict, error) { + input api.BucketCreateInput, +) (api.BucketCreateInput, error) { data := input.JSON(input) + cloudRegionV := validators.NewModelIdOrNameValidator("cloudregion", CloudregionManager.Keyword(), ownerId) managerV := validators.NewModelIdOrNameValidator("manager", CloudproviderManager.Keyword(), ownerId) for _, v := range []validators.IValidator{ @@ -402,26 +404,50 @@ func (manager *SBucketManager) ValidateCreateData( } { err := v.Validate(data) if err != nil { - return nil, err + return input, err } } - nameStr, _ := data.GetString("name") - if len(nameStr) == 0 { - return nil, httperrors.NewInputParameterError("missing name") - } - err := isValidBucketName(nameStr) + + err := data.Unmarshal(&input) if err != nil { - return nil, httperrors.NewInputParameterError("invalid bucket name: %s", err) + return input, httperrors.NewInternalServerError("unmarshal input fail %s", err) } - cloudprovider := managerV.Model.(*SCloudprovider) - quotaPlatformId := cloudprovider.GetQuotaPlatformID() - pendingUsage := SQuota{Bucket: 1} - if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, rbacutils.ScopeProject, ownerId, quotaPlatformId, &pendingUsage); err != nil { - return nil, httperrors.NewOutOfQuotaError("%s", err) + if len(input.Name) == 0 { + return input, httperrors.NewInputParameterError("missing name") + } + err = isValidBucketName(input.Name) + if err != nil { + return input, httperrors.NewInputParameterError("invalid bucket name %s: %s", input.Name, err) } - return manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + quotaKeys := fetchRegionalQuotaKeys(rbacutils.ScopeProject, ownerId, + cloudRegionV.Model.(*SCloudregion), + managerV.Model.(*SCloudprovider)) + pendingUsage := SRegionQuota{Bucket: 1} + pendingUsage.SetKeys(quotaKeys) + if err := RegionQuotaManager.CheckSetPendingQuota(ctx, userCred, &pendingUsage); err != nil { + return input, httperrors.NewOutOfQuotaError("%s", err) + } + + input.VirtualResourceCreateInput, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.VirtualResourceCreateInput) + if err != nil { + return input, err + } + return input, nil +} + +func (bucket *SBucket) GetQuotaKeys() (quotas.IQuotaKeys, error) { + region, _ := bucket.GetRegion() + if region == nil { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "no valid region") + } + return fetchRegionalQuotaKeys( + rbacutils.ScopeProject, + bucket.GetOwnerId(), + region, + bucket.GetCloudprovider(), + ), nil } func (bucket *SBucket) PostCreate( @@ -431,12 +457,16 @@ func (bucket *SBucket) PostCreate( query jsonutils.JSONObject, data jsonutils.JSONObject, ) { - cloudprovider := bucket.GetCloudprovider() - quotaPlatformId := cloudprovider.GetQuotaPlatformID() - pendingUsage := SQuota{Bucket: 1} - err := QuotaManager.CancelPendingUsage(ctx, userCred, rbacutils.ScopeProject, ownerId, quotaPlatformId, &pendingUsage, &pendingUsage) + pendingUsage := SRegionQuota{Bucket: 1} + keys, err := bucket.GetQuotaKeys() if err != nil { - log.Errorf("CancelPendingUsage error %s", err) + log.Errorf("bucket.GetQuotaKeys fail %s", err) + } else { + pendingUsage.SetKeys(keys) + err = RegionQuotaManager.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage) + if err != nil { + log.Errorf("CancelPendingUsage error %s", err) + } } bucket.SetStatus(userCred, api.BUCKET_STATUS_START_CREATE, "PostCreate") @@ -725,13 +755,14 @@ func (bucket *SBucket) PerformMakedir( if bucket.ObjectCntLimit > 0 && bucket.ObjectCntLimit < bucket.ObjectCnt+1 { return nil, httperrors.NewOutOfQuotaError("object count limit exceeds") } - manager := bucket.GetCloudprovider() - quotaPlatformId := manager.GetQuotaPlatformID() - pendingUsage := SQuota{ObjectGB: 0, ObjectCnt: 1} - if !pendingUsage.IsEmpty() { - if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, rbacutils.ScopeProject, bucket.GetOwnerId(), quotaPlatformId, &pendingUsage); err != nil { - return nil, httperrors.NewOutOfQuotaError("%s", err) - } + pendingUsage := SRegionQuota{ObjectGB: 0, ObjectCnt: 1} + keys, err := bucket.GetQuotaKeys() + if err != nil { + return nil, httperrors.NewInternalServerError("bucket.GetQuotaKeys %s", err) + } + pendingUsage.SetKeys(keys) + if err := RegionQuotaManager.CheckSetPendingQuota(ctx, userCred, &pendingUsage); err != nil { + return nil, httperrors.NewOutOfQuotaError("%s", err) } err = cloudprovider.Makedir(ctx, iBucket, key+"/") @@ -744,9 +775,7 @@ func (bucket *SBucket) PerformMakedir( bucket.syncWithCloudBucket(ctx, userCred, iBucket, nil, true) - if !pendingUsage.IsEmpty() { - QuotaManager.CancelPendingUsage(ctx, userCred, rbacutils.ScopeProject, bucket.GetOwnerId(), quotaPlatformId, &pendingUsage, &pendingUsage) - } + RegionQuotaManager.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage) return nil, nil } @@ -894,11 +923,14 @@ func (bucket *SBucket) PerformUpload( return nil, httperrors.NewOutOfQuotaError("object count limit exceeds") } - manager := bucket.GetCloudprovider() - quotaPlatformId := manager.GetQuotaPlatformID() - pendingUsage := SQuota{ObjectGB: int(inc.SizeBytes / 1000 / 1000 / 1000), ObjectCnt: inc.ObjectCount} + pendingUsage := SRegionQuota{ObjectGB: int(inc.SizeBytes / 1000 / 1000 / 1000), ObjectCnt: inc.ObjectCount} + keys, err := bucket.GetQuotaKeys() + if err != nil { + return nil, httperrors.NewInternalServerError("bucket.GetQuotaKeys fail %s", err) + } + pendingUsage.SetKeys(keys) if !pendingUsage.IsEmpty() { - if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, rbacutils.ScopeProject, bucket.GetOwnerId(), quotaPlatformId, &pendingUsage); err != nil { + if err := RegionQuotaManager.CheckSetPendingQuota(ctx, userCred, &pendingUsage); err != nil { return nil, httperrors.NewOutOfQuotaError("%s", err) } } @@ -914,7 +946,7 @@ func (bucket *SBucket) PerformUpload( bucket.syncWithCloudBucket(ctx, userCred, iBucket, nil, true) if !pendingUsage.IsEmpty() { - QuotaManager.CancelPendingUsage(ctx, userCred, rbacutils.ScopeProject, bucket.GetOwnerId(), quotaPlatformId, &pendingUsage, &pendingUsage) + RegionQuotaManager.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage) } return nil, nil @@ -1102,39 +1134,12 @@ func (manager *SBucketManager) usageQByCloudEnv(q *sqlchemy.SQuery, providers [] return CloudProviderFilter(q, q.Field("manager_id"), providers, brands, cloudEnv) } -func (manager *SBucketManager) usageQByRange(q *sqlchemy.SQuery, rangeObj db.IStandaloneModel) *sqlchemy.SQuery { - if rangeObj == nil { - return q - } - - kw := rangeObj.Keyword() - switch kw { - case "zone": - zone := rangeObj.(*SZone) - q = q.Filter(sqlchemy.Equals(q.Field("cloudregion_id"), zone.CloudregionId)) - case "wire": - wire := rangeObj.(*SWire) - zone := wire.GetZone() - q = q.Filter(sqlchemy.Equals(q.Field("cloudregion_id"), zone.CloudregionId)) - case "host": - host := rangeObj.(*SHost) - zone := host.GetZone() - q = q.Filter(sqlchemy.Equals(q.Field("cloudregion_id"), zone.CloudregionId)) - case "cloudprovider": - q = q.Filter(sqlchemy.Equals(q.Field("manager_id"), rangeObj.GetId())) - case "cloudaccount": - cloudproviders := CloudproviderManager.Query().SubQuery() - subq := cloudproviders.Query(cloudproviders.Field("id")).Equals("cloudaccount_id", rangeObj.GetId()).SubQuery() - q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq)) - case "cloudregion": - q = q.Filter(sqlchemy.Equals(q.Field("cloudregion_id"), rangeObj.GetId())) - } - - return q +func (manager *SBucketManager) usageQByRanges(q *sqlchemy.SQuery, rangeObjs []db.IStandaloneModel) *sqlchemy.SQuery { + return rangeObjectsFilter(q, rangeObjs, q.Field("cloudregion_id"), nil, q.Field("manager_id")) } -func (manager *SBucketManager) usageQ(q *sqlchemy.SQuery, rangeObj db.IStandaloneModel, providers []string, brands []string, cloudEnv string) *sqlchemy.SQuery { - q = manager.usageQByRange(q, rangeObj) +func (manager *SBucketManager) usageQ(q *sqlchemy.SQuery, rangeObjs []db.IStandaloneModel, providers []string, brands []string, cloudEnv string) *sqlchemy.SQuery { + q = manager.usageQByRanges(q, rangeObjs) q = manager.usageQByCloudEnv(q, providers, brands, cloudEnv) return q } @@ -1145,7 +1150,7 @@ type SBucketUsages struct { Bytes int64 } -func (manager *SBucketManager) TotalCount(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, rangeObj db.IStandaloneModel, providers []string, brands []string, cloudEnv string) SBucketUsages { +func (manager *SBucketManager) TotalCount(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, rangeObjs []db.IStandaloneModel, providers []string, brands []string, cloudEnv string) SBucketUsages { usage := SBucketUsages{} buckets := manager.Query().SubQuery() q := buckets.Query( @@ -1153,15 +1158,8 @@ func (manager *SBucketManager) TotalCount(scope rbacutils.TRbacScope, ownerId mc sqlchemy.SUM("objects", buckets.Field("object_cnt")), sqlchemy.SUM("bytes", buckets.Field("size_bytes")), ) - q = manager.usageQ(q, rangeObj, providers, brands, cloudEnv) - switch scope { - case rbacutils.ScopeSystem: - // do nothing - case rbacutils.ScopeDomain: - q = q.Equals("domain_id", ownerId.GetProjectDomainId()) - case rbacutils.ScopeProject: - q = q.Equals("tenant_id", ownerId.GetProjectId()) - } + q = manager.usageQ(q, rangeObjs, providers, brands, cloudEnv) + q = scopeOwnerIdFilter(q, scope, ownerId) err := q.First(&usage) if err != nil { log.Errorf("Query bucket usage error %s", err) diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go index 3bf5177293..b2bd08a17d 100644 --- a/pkg/compute/models/cloudaccounts.go +++ b/pkg/compute/models/cloudaccounts.go @@ -259,33 +259,33 @@ func (self *SCloudaccount) ValidateUpdateData(ctx context.Context, userCred mccl return self.SEnabledStatusStandaloneResourceBase.ValidateUpdateData(ctx, userCred, query, data) } -func (manager *SCloudaccountManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { +func (manager *SCloudaccountManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.CloudaccountCreateInput) (api.CloudaccountCreateInput, error) { // check domainId err := db.ValidateCreateDomainId(ownerId.GetProjectDomainId()) if err != nil { - return nil, err + return input, err } + data := jsonutils.Marshal(input).(*jsonutils.JSONDict) tenantV := validators.NewModelIdOrNameValidator("tenant", "tenant", ownerId) tenantV.Optional(true) err = tenantV.Validate(data) if err != nil { - return nil, err + return input, err } - input := &api.CloudaccountCreateInput{} - err = data.Unmarshal(input) + err = data.Unmarshal(&input) if err != nil { - return nil, httperrors.NewInputParameterError("failed to unmarshal input params error: %v", err) + return input, httperrors.NewInputParameterError("failed to unmarshal input params error: %v", err) } if !cloudprovider.IsSupported(input.Provider) { - return nil, httperrors.NewInputParameterError("Unsupported provider %s", input.Provider) + return input, httperrors.NewInputParameterError("Unsupported provider %s", input.Provider) } providerDriver, _ := cloudprovider.GetProviderFactory(input.Provider) - err = providerDriver.ValidateCreateCloudaccountData(ctx, userCred, input) + err = providerDriver.ValidateCreateCloudaccountData(ctx, userCred, &input) if err != nil { - return nil, err + return input, err } if len(input.Brand) > 0 && input.Brand != providerDriver.GetName() { brands := providerDriver.GetSupportedBrands() @@ -293,7 +293,7 @@ func (manager *SCloudaccountManager) ValidateCreateData(ctx context.Context, use brands = append(brands, providerDriver.GetName()) } if !utils.IsInStringArray(input.Brand, brands) { - return nil, httperrors.NewUnsupportOperationError("Not support brand %s, only support %s", input.Brand, brands) + return input, httperrors.NewUnsupportOperationError("Not support brand %s, only support %s", input.Brand, brands) } } input.IsPublicCloud = providerDriver.IsPublicCloud() @@ -309,29 +309,29 @@ func (manager *SCloudaccountManager) ValidateCreateData(ctx context.Context, use cnt, err := q.CountWithError() if err != nil { - return nil, httperrors.NewInternalServerError("check uniqness fail %s", err) + return input, httperrors.NewInternalServerError("check uniqness fail %s", err) } if cnt > 0 { - return nil, httperrors.NewConflictError("The account has been registered") + return input, httperrors.NewConflictError("The account has been registered") } accountId, err := cloudprovider.IsValidCloudAccount(input.AccessUrl, input.Account, input.Secret, input.Provider) if err != nil { if err == cloudprovider.ErrNoSuchProvder { - return nil, httperrors.NewResourceNotFoundError("no such provider %s", input.Provider) + return input, httperrors.NewResourceNotFoundError("no such provider %s", input.Provider) } //log.Debugf("ValidateCreateData %s", err.Error()) - return nil, httperrors.NewInputParameterError("invalid cloud account info error: %s", err.Error()) + return input, httperrors.NewInputParameterError("invalid cloud account info error: %s", err.Error()) } // check accountId uniqueness if len(accountId) > 0 { cnt, err := manager.Query().Equals("account_id", accountId).CountWithError() if err != nil { - return nil, httperrors.NewInternalServerError("check account_id duplication error %s", err) + return input, httperrors.NewInternalServerError("check account_id duplication error %s", err) } if cnt > 0 { - return nil, httperrors.NewDuplicateResourceError("the account has been registerd %s", accountId) + return input, httperrors.NewDuplicateResourceError("the account has been registerd %s", accountId) } data.Set("account_id", jsonutils.NewString(accountId)) } @@ -348,20 +348,20 @@ func (manager *SCloudaccountManager) ValidateCreateData(ctx context.Context, use params := jsonutils.Marshal(map[string]string{"domain_id": ownerId.GetProjectDomainId()}) tenants, err := modules.Projects.List(s, params) if err != nil { - return nil, err + return input, err } if tenants.Total == 0 { - return nil, httperrors.NewInputParameterError("There are no projects under the domain %s", ownerId.GetProjectDomainId()) + return input, httperrors.NewInputParameterError("There is no projects under the domain %s", ownerId.GetProjectDomainId()) } } } - data, err = manager.SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + input.EnabledStatusStandaloneResourceCreateInput, err = manager.SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.EnabledStatusStandaloneResourceCreateInput) if err != nil { - return nil, err + return input, err } - return input.JSON(input), nil + return input, nil } func (self *SCloudaccount) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error { diff --git a/pkg/compute/models/cloudregions.go b/pkg/compute/models/cloudregions.go index 48147cc8bb..430ca3a492 100644 --- a/pkg/compute/models/cloudregions.go +++ b/pkg/compute/models/cloudregions.go @@ -60,6 +60,7 @@ type SCloudregion struct { cloudprovider.SGeographicInfo + CloudEnv string `width:"32" charset:"ascii" list:"user"` Provider string `width:"64" charset:"ascii" list:"user" nullable:"false" default:"OneCloud"` } @@ -402,6 +403,7 @@ func (self *SCloudregion) syncWithCloudRegion(ctx context.Context, userCred mccl self.Status = cloudRegion.GetStatus() self.SGeographicInfo = cloudRegion.GetGeographicInfo() self.Provider = cloudRegion.GetProvider() + self.CloudEnv = cloudRegion.GetCloudEnv() self.IsEmulated = cloudRegion.IsEmulated() @@ -433,6 +435,7 @@ func (manager *SCloudregionManager) newFromCloudRegion(ctx context.Context, user region.Status = cloudRegion.GetStatus() region.Enabled = true region.Provider = cloudRegion.GetProvider() + region.CloudEnv = cloudRegion.GetCloudEnv() region.IsEmulated = cloudRegion.IsEmulated() @@ -704,9 +707,9 @@ func (manager *SCloudregionManager) ListItemFilter(ctx context.Context, q *sqlch return q, nil } -func (manager *SCloudregionManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { +/*func (manager *SCloudregionManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { return manager.SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) -} +}*/ func (self *SCloudregion) isManaged() bool { if len(self.ExternalId) > 0 { diff --git a/pkg/compute/models/dbinstances.go b/pkg/compute/models/dbinstances.go index 9430e5b9d7..9a49a3bc6b 100644 --- a/pkg/compute/models/dbinstances.go +++ b/pkg/compute/models/dbinstances.go @@ -40,6 +40,7 @@ import ( "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/billing" "yunion.io/x/onecloud/pkg/util/stringutils2" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) type SDBInstanceManager struct { @@ -1374,3 +1375,16 @@ func (manager *SDBInstanceManager) newFromCloudDBInstance(ctx context.Context, u return &instance, nil } + +func (man *SDBInstanceManager) TotalCount( + scope rbacutils.TRbacScope, + ownerId mcclient.IIdentityProvider, + rangeObjs []db.IStandaloneModel, + providers []string, brands []string, cloudEnv string, +) (int, error) { + q := man.Query() + q = scopeOwnerIdFilter(q, scope, ownerId) + q = CloudProviderFilter(q, q.Field("manager_id"), providers, brands, cloudEnv) + q = rangeObjectsFilter(q, rangeObjs, q.Field("cloudregion_id"), nil, q.Field("manager_id")) + return q.CountWithError() +} \ No newline at end of file diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 46cfea853f..2c951983ab 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -54,14 +54,14 @@ import ( ) type SDiskManager struct { - db.SSharableVirtualResourceBaseManager + db.SVirtualResourceBaseManager } var DiskManager *SDiskManager func init() { DiskManager = &SDiskManager{ - SSharableVirtualResourceBaseManager: db.NewSharableVirtualResourceBaseManager( + SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager( SDisk{}, "disks_tbl", "disk", @@ -72,7 +72,7 @@ func init() { } type SDisk struct { - db.SSharableVirtualResourceBase + db.SVirtualResourceBase db.SExternalizedResourceBase SBillingResourceBase @@ -147,7 +147,7 @@ func (manager *SDiskManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu queryDict.Remove("billing_type") } - q, err = manager.SSharableVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query) + q, err = manager.SVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query) if err != nil { return nil, err } @@ -369,7 +369,7 @@ func (self *SDisk) CustomizeCreate(ctx context.Context, userCred mcclient.TokenC return err } self.fetchDiskInfo(input.DiskConfig) - return self.SSharableVirtualResourceBase.CustomizeCreate(ctx, userCred, ownerId, query, data) + return self.SVirtualResourceBase.CustomizeCreate(ctx, userCred, ownerId, query, data) } func (self *SDisk) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { @@ -397,6 +397,37 @@ func (self *SDisk) ValidateUpdateData(ctx context.Context, userCred mcclient.Tok return self.SVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data) } +func diskCreateInput2ComputeQuotaKeys(input api.DiskCreateInput, ownerId mcclient.IIdentityProvider) SComputeResourceKeys { + // input.Hypervisor must be set + keys := GetDriver(input.Hypervisor).GetComputeQuotaKeys( + rbacutils.ScopeProject, + ownerId, + "", + ) + if len(input.PreferHost) > 0 { + hostObj, _ := HostManager.FetchById(input.PreferHost) + host := hostObj.(*SHost) + zone := host.GetZone() + keys.ZoneId = zone.Id + keys.RegionId = zone.CloudregionId + } else if len(input.PreferZone) > 0 { + zoneObj, _ := ZoneManager.FetchById(input.PreferZone) + zone := zoneObj.(*SZone) + keys.ZoneId = zone.Id + keys.RegionId = zone.CloudregionId + } else if len(input.PreferWire) > 0 { + wireObj, _ := WireManager.FetchById(input.PreferWire) + wire := wireObj.(*SWire) + zone := wire.GetZone() + keys.ZoneId = zone.Id + keys.RegionId = zone.CloudregionId + } else if len(input.PreferRegion) > 0 { + regionObj, _ := CloudregionManager.FetchById(input.PreferRegion) + keys.RegionId = regionObj.GetId() + } + return keys +} + func (manager *SDiskManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { input, err := cmdline.FetchDiskCreateInputByJSON(data) if err != nil { @@ -410,6 +441,8 @@ func (manager *SDiskManager) ValidateCreateData(ctx context.Context, userCred mc input.Project = ownerId.GetProjectId() input.Domain = ownerId.GetProjectDomainId() + var quotaKey quotas.IQuotaKeys + storageID := input.Storage if storageID != "" { storageObj, err := StorageManager.FetchByIdOrName(nil, storageID) @@ -418,7 +451,8 @@ func (manager *SDiskManager) ValidateCreateData(ctx context.Context, userCred mc } storage := storageObj.(*SStorage) - if provider := storage.GetCloudprovider(); provider != nil { + provider := storage.GetCloudprovider() + if provider != nil { if !provider.Enabled { return nil, httperrors.NewInputParameterError("provider %s(%s) is disabled, you need enable provider first", provider.Name, provider.Id) } @@ -432,7 +466,7 @@ func (manager *SDiskManager) ValidateCreateData(ctx context.Context, userCred mc host := storage.GetMasterHost() if host == nil { - return nil, httperrors.NewResourceNotFoundError("storage %s(%s) need onlne and attach host for create disk", storage.Name, storage.Id) + return nil, httperrors.NewResourceNotFoundError("storage %s(%s) need online and attach host for create disk", storage.Name, storage.Id) } input.Hypervisor = host.GetHostDriver().GetHypervisor() if len(diskConfig.Backend) == 0 { @@ -443,6 +477,14 @@ func (manager *SDiskManager) ValidateCreateData(ctx context.Context, userCred mc return nil, err } input.Storage = storage.Id + + quotaKey = fetchComputeQuotaKeys( + rbacutils.ScopeProject, + ownerId, + storage.getZone(), + provider, + input.Hypervisor, + ) } else { diskConfig.Backend = api.STORAGE_LOCAL serverInput, err := ValidateScheduleCreateData(ctx, userCred, input.ToServerCreateInput(), input.Hypervisor) @@ -450,15 +492,17 @@ func (manager *SDiskManager) ValidateCreateData(ctx context.Context, userCred mc return nil, err } input = serverInput.ToDiskCreateInput() + quotaKey = diskCreateInput2ComputeQuotaKeys(*input, ownerId) } - if _, err := manager.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data); err != nil { + input.VirtualResourceCreateInput, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.VirtualResourceCreateInput) + if err != nil { return nil, err } - pendingUsage := SQuota{Storage: diskConfig.SizeMb} - quotaName := GetDriver(input.Hypervisor).GetQuotaPlatformID() - if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, rbacutils.ScopeProject, userCred, quotaName, &pendingUsage); err != nil { + pendingUsage := SQuota{Storage: diskConfig.SizeMb} + pendingUsage.SetKeys(quotaKey) + if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, &pendingUsage); err != nil { return nil, httperrors.NewOutOfQuotaError("%s", err) } return input.JSON(input), nil @@ -549,11 +593,26 @@ func (disk *SDisk) SetStorageByHost(hostId string, diskConfig *api.DiskConfig, s return err } -func getDiskResourceRequirements(ctx context.Context, userCred mcclient.TokenCredential, data jsonutils.JSONObject, count int) SQuota { - diskSize, _ := data.Int("disk", "size") - return SQuota{ - Storage: int(diskSize) * count, +func getDiskResourceRequirements(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input api.DiskCreateInput, count int) SQuota { + req := SQuota{ + Storage: input.SizeMb * count, } + var quotaKey SComputeResourceKeys + if len(input.Storage) > 0 { + storageObj, _ := StorageManager.FetchById(input.Storage) + storage := storageObj.(*SStorage) + quotaKey = fetchComputeQuotaKeys( + rbacutils.ScopeProject, + ownerId, + storage.getZone(), + storage.GetCloudprovider(), + input.Hypervisor, + ) + } else { + quotaKey = diskCreateInput2ComputeQuotaKeys(input, ownerId) + } + req.SetKeys(quotaKey) + return req } /*func (manager *SDiskManager) convertToBatchCreateData(data jsonutils.JSONObject) *jsonutils.JSONDict { @@ -563,9 +622,14 @@ func getDiskResourceRequirements(ctx context.Context, userCred mcclient.TokenCre return newData }*/ -func (manager *SDiskManager) OnCreateComplete(ctx context.Context, items []db.IModel, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) { - pendingUsage := getDiskResourceRequirements(ctx, userCred, data, len(items)) - RunBatchCreateTask(ctx, items, userCred, data, pendingUsage, "DiskBatchCreateTask", "") +func (manager *SDiskManager) OnCreateComplete(ctx context.Context, items []db.IModel, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { + input := api.DiskCreateInput{} + err := data.Unmarshal(&input) + if err != nil { + log.Errorf("!!!data.Unmarshal api.DiskCreateInput fail %s", err) + } + pendingUsage := getDiskResourceRequirements(ctx, userCred, ownerId, input, len(items)) + RunBatchCreateTask(ctx, items, userCred, data, pendingUsage, SRegionQuota{}, "DiskBatchCreateTask", "") } func (self *SDisk) StartDiskCreateTask(ctx context.Context, userCred mcclient.TokenCredential, rebuild bool, snapshot string, parentTaskId string) error { @@ -777,6 +841,40 @@ func (disk *SDisk) PerformResize(ctx context.Context, userCred mcclient.TokenCre return nil, nil } +func (disk *SDisk) getHypervisor() string { + storage := disk.GetStorage() + if storage != nil { + host := storage.GetMasterHost() + if host != nil { + return host.GetHostDriver().GetHypervisor() + } + } + hypervisor := disk.GetMetadata("hypervisor", nil) + return hypervisor +} + +func (disk *SDisk) GetQuotaKeys() (quotas.IQuotaKeys, error) { + storage := disk.GetStorage() + if storage == nil { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "no valid storage") + } + provider := storage.GetCloudprovider() + if provider == nil { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "no valid manager") + } + zone := storage.getZone() + if zone == nil { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "no valid zone") + } + return fetchComputeQuotaKeys( + rbacutils.ScopeProject, + disk.GetOwnerId(), + zone, + provider, + disk.getHypervisor(), + ), nil +} + func (disk *SDisk) doResize(ctx context.Context, userCred mcclient.TokenCredential, data jsonutils.JSONObject, guest *SGuest) error { sizeStr, err := data.GetString("size") if err != nil { @@ -811,10 +909,14 @@ func (disk *SDisk) doResize(ctx context.Context, userCred mcclient.TokenCredenti } } pendingUsage := SQuota{Storage: int(addDisk)} - - quotaName := storage.getQuotaPlatformID() - if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, rbacutils.ScopeProject, userCred, quotaName, &pendingUsage); err != nil { - return httperrors.NewOutOfQuotaError(err.Error()) + keys, err := disk.GetQuotaKeys() + if err != nil { + return httperrors.NewInternalServerError("disk.GetQuotaKeys fail %s", err) + } + pendingUsage.SetKeys(keys) + err = QuotaManager.CheckSetPendingQuota(ctx, userCred, &pendingUsage) + if err != nil { + return httperrors.NewGeneralError(err) } if guest != nil { @@ -955,7 +1057,7 @@ func (self *SDisk) validateDeleteCondition(ctx context.Context, isPurge bool) er return httperrors.NewBadRequestError("not allow to delete %s disk with snapshots", storage.StorageType) } } - return self.SSharableVirtualResourceBase.ValidateDeleteCondition(ctx) + return self.SVirtualResourceBase.ValidateDeleteCondition(ctx) } func (self *SDisk) AllowDeleteItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { @@ -1379,13 +1481,33 @@ func (manager *SDiskManager) newFromCloudDisk(ctx context.Context, userCred mccl return &disk, nil } -func totalDiskSize(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, active tristate.TriState, ready tristate.TriState, includeSystem bool, pendingDelete bool) int { +func totalDiskSize( + scope rbacutils.TRbacScope, + ownerId mcclient.IIdentityProvider, + active tristate.TriState, + ready tristate.TriState, + includeSystem bool, + pendingDelete bool, + rangeObjs []db.IStandaloneModel, + providers []string, + brands []string, + cloudEnv string, + hypervisors []string, +) int { disks := DiskManager.Query().SubQuery() q := disks.Query(sqlchemy.SUM("total", disks.Field("disk_size"))) + storages := StorageManager.Query().SubQuery() + q = q.Join(storages, sqlchemy.Equals(storages.Field("id"), disks.Field("storage_id"))) + q = CloudProviderFilter(q, storages.Field("manager_id"), providers, brands, cloudEnv) + q = rangeObjectsFilter(q, rangeObjs, nil, storages.Field("zone_id"), storages.Field("manager_id")) + if len(hypervisors) > 0 { + hoststorages := HoststorageManager.Query().SubQuery() + hosts := HostManager.Query().SubQuery() + q = q.Join(hoststorages, sqlchemy.Equals(storages.Field("id"), hoststorages.Field("storage_id"))) + q = q.Join(hosts, sqlchemy.Equals(hoststorages.Field("host_id"), hosts.Field("id"))) + q = q.Filter(sqlchemy.In(hosts.Field("host_type"), api.Hypervisors2HostTypes(hypervisors))) + } if !active.IsNone() { - storages := StorageManager.Query().SubQuery() - q = q.Join(storages, sqlchemy.AND(sqlchemy.IsFalse(storages.Field("deleted")), - sqlchemy.Equals(storages.Field("id"), disks.Field("storage_id")))) if active.IsTrue() { q = q.Filter(sqlchemy.In(storages.Field("status"), []string{api.STORAGE_ENABLED, api.STORAGE_ONLINE})) } else { @@ -1626,7 +1748,7 @@ func (self *SDisk) RealDelete(ctx context.Context, userCred mcclient.TokenCreden guestdisk.Detach(ctx, userCred) } } - err := self.SSharableVirtualResourceBase.Delete(ctx, userCred) + err := self.SVirtualResourceBase.Delete(ctx, userCred) if err != nil { return err } @@ -1756,7 +1878,7 @@ func (self *SDisk) getMoreDetails(ctx context.Context, userCred mcclient.TokenCr } func (self *SDisk) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*jsonutils.JSONDict, error) { - extra, err := self.SSharableVirtualResourceBase.GetExtraDetails(ctx, userCred, query) + extra, err := self.SVirtualResourceBase.GetExtraDetails(ctx, userCred, query) if err != nil { return nil, err } @@ -1764,7 +1886,7 @@ func (self *SDisk) GetExtraDetails(ctx context.Context, userCred mcclient.TokenC } func (self *SDisk) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { - extra := self.SSharableVirtualResourceBase.GetCustomizeColumns(ctx, userCred, query) + extra := self.SVirtualResourceBase.GetCustomizeColumns(ctx, userCred, query) return self.getMoreDetails(ctx, userCred, extra) } @@ -1859,7 +1981,7 @@ func (self *SDisk) ClearHostSchedCache() error { } func (self *SDisk) GetShortDesc(ctx context.Context) *jsonutils.JSONDict { - desc := self.SSharableVirtualResourceBase.GetShortDesc(ctx) + desc := self.SVirtualResourceBase.GetShortDesc(ctx) desc.Add(jsonutils.NewInt(int64(self.DiskSize)), "size") storage := self.GetStorage() if storage != nil { diff --git a/pkg/compute/models/dynamicschedtags.go b/pkg/compute/models/dynamicschedtags.go index a0a1fb19bc..fbeab3d351 100644 --- a/pkg/compute/models/dynamicschedtags.go +++ b/pkg/compute/models/dynamicschedtags.go @@ -24,6 +24,7 @@ import ( "yunion.io/x/pkg/tristate" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" @@ -158,7 +159,17 @@ func (manager *SDynamicschedtagManager) ValidateCreateData(ctx context.Context, if err != nil { return nil, err } - return manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + input := apis.StandaloneResourceCreateInput{} + err = data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal StandaloneResourceCreateInput fail %s", err) + } + input, err = manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { + return nil, err + } + data.Update(jsonutils.Marshal(input)) + return data, nil } func (self *SDynamicschedtag) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { diff --git a/pkg/compute/models/elasticcache_accounts.go b/pkg/compute/models/elasticcache_accounts.go index 5577393f63..a1180a4d5b 100644 --- a/pkg/compute/models/elasticcache_accounts.go +++ b/pkg/compute/models/elasticcache_accounts.go @@ -25,6 +25,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" @@ -220,10 +221,17 @@ func (manager *SElasticcacheAccountManager) ValidateCreateData(ctx context.Conte return nil, httperrors.NewMissingParameterError("elasticcache_id") } - data, err := manager.SStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + input := apis.StatusStandaloneResourceCreateInput{} + var err error + err = data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal StandaloneResourceCreateInput fail %s", err) + } + input, err = manager.SStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) if err != nil { return nil, err } + data.Update(jsonutils.Marshal(input)) return region.GetDriver().ValidateCreateElasticcacheAccountData(ctx, userCred, ownerId, data) } diff --git a/pkg/compute/models/elasticcache_acls.go b/pkg/compute/models/elasticcache_acls.go index c4403e4a21..6c029112fd 100644 --- a/pkg/compute/models/elasticcache_acls.go +++ b/pkg/compute/models/elasticcache_acls.go @@ -25,6 +25,7 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" @@ -204,10 +205,17 @@ func (manager *SElasticcacheAclManager) ValidateCreateData(ctx context.Context, return nil, httperrors.NewMissingParameterError("elasticcache") } - data, err := manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + input := apis.StandaloneResourceCreateInput{} + var err error + err = data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal StandaloneResourceCreateInput fail %s", err) + } + input, err = manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) if err != nil { return nil, err } + data.Update(jsonutils.Marshal(input)) return region.GetDriver().ValidateCreateElasticcacheAclData(ctx, userCred, ownerId, data) } diff --git a/pkg/compute/models/elasticcache_backups.go b/pkg/compute/models/elasticcache_backups.go index 08a5582afd..1cd7ca18f2 100644 --- a/pkg/compute/models/elasticcache_backups.go +++ b/pkg/compute/models/elasticcache_backups.go @@ -25,6 +25,7 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" @@ -248,10 +249,17 @@ func (manager *SElasticcacheBackupManager) ValidateCreateData(ctx context.Contex return nil, err } - data, err := manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + input := apis.StandaloneResourceCreateInput{} + var err error + err = data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal StandaloneResourceCreateInput fail %s", err) + } + input, err = manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) if err != nil { return nil, err } + data.Update(jsonutils.Marshal(input)) return driver.ValidateCreateElasticcacheBackupData(ctx, userCred, ownerId, data) } diff --git a/pkg/compute/models/elasticcache_instances.go b/pkg/compute/models/elasticcache_instances.go index 8f9938886f..fd7979fb39 100644 --- a/pkg/compute/models/elasticcache_instances.go +++ b/pkg/compute/models/elasticcache_instances.go @@ -28,6 +28,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" "yunion.io/x/onecloud/pkg/apis/billing" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" @@ -490,10 +491,17 @@ func (manager *SElasticcacheManager) ValidateCreateData(ctx context.Context, use return nil, fmt.Errorf("getting region failed") } - data, err := manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + input := apis.VirtualResourceCreateInput{} + var err error + err = data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal VirtualResourceCreateInput fail %s", err) + } + input, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) if err != nil { return nil, err } + data.Update(jsonutils.Marshal(input)) return region.GetDriver().ValidateCreateElasticcacheData(ctx, userCred, nil, data) } @@ -1245,3 +1253,16 @@ func (self *SElasticcache) DeleteSubResources(ctx context.Context, userCred mccl }(m) } } + +func (man *SElasticcacheManager) TotalCount( + scope rbacutils.TRbacScope, + ownerId mcclient.IIdentityProvider, + rangeObjs []db.IStandaloneModel, + providers []string, brands []string, cloudEnv string, +) (int, error) { + q := man.Query() + q = scopeOwnerIdFilter(q, scope, ownerId) + q = CloudProviderFilter(q, q.Field("manager_id"), providers, brands, cloudEnv) + q = rangeObjectsFilter(q, rangeObjs, q.Field("cloudregion_id"), nil, q.Field("manager_id")) + return q.CountWithError() +} \ No newline at end of file diff --git a/pkg/compute/models/elasticips.go b/pkg/compute/models/elasticips.go index 3e6a1a9488..b239065137 100644 --- a/pkg/compute/models/elasticips.go +++ b/pkg/compute/models/elasticips.go @@ -27,6 +27,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" @@ -685,7 +686,7 @@ func (manager *SElasticipManager) ValidateCreateData(ctx context.Context, userCr return nil, httperrors.NewMissingParameterError("manager_id") } - provider, err := CloudproviderManager.FetchByIdOrName(nil, managerStr) + providerObj, err := CloudproviderManager.FetchByIdOrName(nil, managerStr) if err != nil { if err != sql.ErrNoRows { return nil, httperrors.NewGeneralError(err) @@ -693,7 +694,8 @@ func (manager *SElasticipManager) ValidateCreateData(ctx context.Context, userCr return nil, httperrors.NewResourceNotFoundError("Cloud provider %s not found", managerStr) } } - data.Add(jsonutils.NewString(provider.GetId()), "manager_id") + provider := providerObj.(*SCloudprovider) + data.Add(jsonutils.NewString(provider.Id), "manager_id") chargeType := jsonutils.GetAnyString(data, []string{"charge_type"}) if len(chargeType) == 0 { @@ -706,31 +708,55 @@ func (manager *SElasticipManager) ValidateCreateData(ctx context.Context, userCr data.Add(jsonutils.NewString(chargeType), "charge_type") - data, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + input := apis.VirtualResourceCreateInput{} + err = data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal VirtualResourceCreateInput fail %s", err) + } + input, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) if err != nil { return nil, err } + data.Update(jsonutils.Marshal(input)) //避免参数重名后还有pending.eip残留 - eipPendingUsage := &SQuota{Eip: 1} - - quotaPlatform := provider.(*SCloudprovider).GetQuotaPlatformID() - err = QuotaManager.CheckSetPendingQuota(ctx, userCred, rbacutils.ScopeProject, userCred, quotaPlatform, eipPendingUsage) + eipPendingUsage := &SRegionQuota{Eip: 1} + quotaKeys := fetchRegionalQuotaKeys(rbacutils.ScopeProject, ownerId, region, provider) + eipPendingUsage.SetKeys(quotaKeys) + err = RegionQuotaManager.CheckSetPendingQuota(ctx, userCred, eipPendingUsage) if err != nil { - return nil, httperrors.NewOutOfQuotaError("Out of eip quota: %s", err) + return nil, err } return region.GetDriver().ValidateCreateEipData(ctx, userCred, data) } +func (eip *SElasticip) GetQuotaKeys() (quotas.IQuotaKeys, error) { + region := eip.GetRegion() + if region == nil { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "no valid region") + } + return fetchRegionalQuotaKeys( + rbacutils.ScopeProject, + eip.GetOwnerId(), + region, + eip.GetCloudprovider(), + ), nil +} + func (self *SElasticip) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { self.SVirtualResourceBase.PostCreate(ctx, userCred, ownerId, query, data) - quotaPlatform := self.GetQuotaPlatformID() - eipPendingUsage := &SQuota{Eip: 1} - err := QuotaManager.CancelPendingUsage(ctx, userCred, rbacutils.ScopeProject, ownerId, quotaPlatform, eipPendingUsage, eipPendingUsage) + eipPendingUsage := &SRegionQuota{Eip: 1} + keys, err := self.GetQuotaKeys() if err != nil { - log.Errorf("SElasticip CancelPendingUsage error: %s", err) + log.Errorf("GetQuotaKeys fail %s", err) + } else { + eipPendingUsage.SetKeys(keys) + err := RegionQuotaManager.CancelPendingUsage(ctx, userCred, eipPendingUsage, eipPendingUsage) + if err != nil { + log.Errorf("SElasticip CancelPendingUsage error: %s", err) + } } self.startEipAllocateTask(ctx, userCred, data.(*jsonutils.JSONDict), "") @@ -1069,8 +1095,15 @@ func (manager *SElasticipManager) NewEipForVMOnHost(ctx context.Context, userCre return nil, err } - eipPendingUsage := &SQuota{Eip: 1} - QuotaManager.CancelPendingUsage(ctx, userCred, rbacutils.ScopeProject, vm.GetOwnerId(), vm.GetQuotaPlatformID(), pendingUsage, eipPendingUsage) + eipPendingUsage := &SRegionQuota{Eip: 1} + keys := fetchRegionalQuotaKeys( + rbacutils.ScopeProject, + vm.GetOwnerId(), + region, + host.GetCloudprovider(), + ) + eipPendingUsage.SetKeys(keys) + QuotaManager.CancelPendingUsage(ctx, userCred, pendingUsage, eipPendingUsage) return &eip, nil } @@ -1168,52 +1201,49 @@ func (manager *SElasticipManager) usageQByCloudEnv(q *sqlchemy.SQuery, providers return CloudProviderFilter(q, q.Field("manager_id"), providers, brands, cloudEnv) } -func (manager *SElasticipManager) usageQByRange(q *sqlchemy.SQuery, rangeObj db.IStandaloneModel) *sqlchemy.SQuery { - if rangeObj == nil { - return q +func (manager *SElasticipManager) usageQByRanges(q *sqlchemy.SQuery, rangeObjs []db.IStandaloneModel) *sqlchemy.SQuery { + for _, rangeObj := range rangeObjs { + kw := rangeObj.Keyword() + // log.Debugf("rangeObj keyword: %s", kw) + switch kw { + case "zone": + zone := rangeObj.(*SZone) + q = q.Filter(sqlchemy.Equals(q.Field("cloudregion_id"), zone.CloudregionId)) + case "wire": + wire := rangeObj.(*SWire) + zone := wire.GetZone() + q = q.Filter(sqlchemy.Equals(q.Field("cloudregion_id"), zone.CloudregionId)) + case "host": + host := rangeObj.(*SHost) + zone := host.GetZone() + q = q.Filter(sqlchemy.Equals(q.Field("cloudregion_id"), zone.CloudregionId)) + case "cloudprovider": + q = q.Filter(sqlchemy.Equals(q.Field("manager_id"), rangeObj.GetId())) + case "cloudaccount": + cloudproviders := CloudproviderManager.Query().SubQuery() + subq := cloudproviders.Query(cloudproviders.Field("id")).Equals("cloudaccount_id", rangeObj.GetId()).SubQuery() + q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq)) + case "cloudregion": + q = q.Filter(sqlchemy.Equals(q.Field("cloudregion_id"), rangeObj.GetId())) + } } - - kw := rangeObj.Keyword() - // log.Debugf("rangeObj keyword: %s", kw) - switch kw { - case "zone": - zone := rangeObj.(*SZone) - q = q.Filter(sqlchemy.Equals(q.Field("cloudregion_id"), zone.CloudregionId)) - case "wire": - wire := rangeObj.(*SWire) - zone := wire.GetZone() - q = q.Filter(sqlchemy.Equals(q.Field("cloudregion_id"), zone.CloudregionId)) - case "host": - host := rangeObj.(*SHost) - zone := host.GetZone() - q = q.Filter(sqlchemy.Equals(q.Field("cloudregion_id"), zone.CloudregionId)) - case "cloudprovider": - q = q.Filter(sqlchemy.Equals(q.Field("manager_id"), rangeObj.GetId())) - case "cloudaccount": - cloudproviders := CloudproviderManager.Query().SubQuery() - subq := cloudproviders.Query(cloudproviders.Field("id")).Equals("cloudaccount_id", rangeObj.GetId()).SubQuery() - q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq)) - case "cloudregion": - q = q.Filter(sqlchemy.Equals(q.Field("cloudregion_id"), rangeObj.GetId())) - } - return q } -func (manager *SElasticipManager) usageQ(q *sqlchemy.SQuery, rangeObj db.IStandaloneModel, providers []string, brands []string, cloudEnv string) *sqlchemy.SQuery { - q = manager.usageQByRange(q, rangeObj) +func (manager *SElasticipManager) usageQ(q *sqlchemy.SQuery, rangeObjs []db.IStandaloneModel, providers []string, brands []string, cloudEnv string) *sqlchemy.SQuery { + q = manager.usageQByRanges(q, rangeObjs) q = manager.usageQByCloudEnv(q, providers, brands, cloudEnv) return q } -func (manager *SElasticipManager) TotalCount(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, rangeObj db.IStandaloneModel, providers []string, brands []string, cloudEnv string) EipUsage { +func (manager *SElasticipManager) TotalCount(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, rangeObjs []db.IStandaloneModel, providers []string, brands []string, cloudEnv string) EipUsage { usage := EipUsage{} q1 := manager.Query().Equals("mode", api.EIP_MODE_INSTANCE_PUBLICIP) - q1 = manager.usageQ(q1, rangeObj, providers, brands, cloudEnv) + q1 = manager.usageQ(q1, rangeObjs, providers, brands, cloudEnv) q2 := manager.Query().Equals("mode", api.EIP_MODE_STANDALONE_EIP) - q2 = manager.usageQ(q2, rangeObj, providers, brands, cloudEnv) + q2 = manager.usageQ(q2, rangeObjs, providers, brands, cloudEnv) q3 := manager.Query().Equals("mode", api.EIP_MODE_STANDALONE_EIP).IsNotEmpty("associate_id") - q3 = manager.usageQ(q3, rangeObj, providers, brands, cloudEnv) + q3 = manager.usageQ(q3, rangeObjs, providers, brands, cloudEnv) switch scope { case rbacutils.ScopeSystem: // do nothing diff --git a/pkg/compute/models/filters.go b/pkg/compute/models/filters.go new file mode 100644 index 0000000000..8d04ddf36f --- /dev/null +++ b/pkg/compute/models/filters.go @@ -0,0 +1,86 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "yunion.io/x/sqlchemy" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/util/rbacutils" + "yunion.io/x/onecloud/pkg/mcclient" +) + +func rangeObjectsFilter(q *sqlchemy.SQuery, rangeObjs []db.IStandaloneModel, regionField sqlchemy.IQueryField, zoneField sqlchemy.IQueryField, managerField sqlchemy.IQueryField) *sqlchemy.SQuery { + for _, rangeObj := range rangeObjs { + q = rangeObjFilter(q, rangeObj, regionField, zoneField, managerField) + } + return q +} + +func rangeObjFilter(q *sqlchemy.SQuery, rangeObj db.IStandaloneModel, regionField sqlchemy.IQueryField, zoneField sqlchemy.IQueryField, managerField sqlchemy.IQueryField) *sqlchemy.SQuery { + kw := rangeObj.Keyword() + switch kw { + case "zone": + zone := rangeObj.(*SZone) + if regionField != nil { + q = q.Filter(sqlchemy.Equals(regionField, zone.CloudregionId)) + } else if zoneField != nil { + q = q.Filter(sqlchemy.Equals(zoneField, zone.Id)) + } + case "wire": + wire := rangeObj.(*SWire) + if regionField != nil { + vpc := wire.getVpc() + q = q.Filter(sqlchemy.Equals(regionField, vpc.CloudregionId)) + } else if zoneField != nil { + q = q.Filter(sqlchemy.Equals(zoneField, wire.ZoneId)) + } + case "host": + host := rangeObj.(*SHost) + if regionField != nil { + zone := host.GetZone() + q = q.Filter(sqlchemy.Equals(regionField, zone.CloudregionId)) + } else if zoneField != nil { + q = q.Filter(sqlchemy.Equals(zoneField, host.ZoneId)) + } + case "cloudprovider": + q = q.Filter(sqlchemy.Equals(managerField, rangeObj.GetId())) + case "cloudaccount": + cloudproviders := CloudproviderManager.Query().SubQuery() + subq := cloudproviders.Query(cloudproviders.Field("id")).Equals("cloudaccount_id", rangeObj.GetId()).SubQuery() + q = q.Filter(sqlchemy.In(managerField, subq)) + case "cloudregion": + if regionField != nil { + q = q.Filter(sqlchemy.Equals(regionField, rangeObj.GetId())) + } else if zoneField != nil { + zones := ZoneManager.Query().SubQuery() + subq := zones.Query(zones.Field("id")).Equals("cloudregion_id", rangeObj.GetId()).SubQuery() + q = q.Filter(sqlchemy.In(zoneField, subq)) + } + } + return q +} + +func scopeOwnerIdFilter(q *sqlchemy.SQuery, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) *sqlchemy.SQuery { + switch scope { + case rbacutils.ScopeSystem: + // do nothing + case rbacutils.ScopeDomain: + q = q.Equals("domain_id", ownerId.GetProjectDomainId()) + case rbacutils.ScopeProject: + q = q.Equals("tenant_id", ownerId.GetProjectId()) + } + return q +} diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index 3762b61cac..3ddd216f16 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -57,7 +57,6 @@ import ( "yunion.io/x/onecloud/pkg/util/httputils" "yunion.io/x/onecloud/pkg/util/logclient" "yunion.io/x/onecloud/pkg/util/rand" - "yunion.io/x/onecloud/pkg/util/rbacutils" "yunion.io/x/onecloud/pkg/util/seclib2" ) @@ -498,14 +497,19 @@ func (self *SGuest) PerformClone(ctx context.Context, userCred mcclient.TokenCre db.OpsLog.LogEvent(model, db.ACT_CREATE, model.GetShortDesc(ctx), userCred) logclient.AddActionLogWithContext(ctx, model, logclient.ACT_CREATE, "", userCred, true) - pendingUsage := getGuestResourceRequirements(ctx, userCred, createInput, 1, false) - if task, err := taskman.TaskManager.NewTask(ctx, "GuestCloneTask", model.(db.IStandaloneModel), userCred, - dataDict, "", "", &pendingUsage); err != nil { + pendingUsage, pendingRegionUsage := getGuestResourceRequirements(ctx, userCred, *createInput, userCred, 1, false) + task, err := taskman.TaskManager.NewTask(ctx, + "GuestCloneTask", + model.(db.IStandaloneModel), + userCred, + dataDict, + "", "", + &pendingUsage, &pendingRegionUsage) + if err != nil { log.Errorln(err) return nil, err - } else { - task.ScheduleRun(nil) } + task.ScheduleRun(nil) return nil, nil } @@ -1524,7 +1528,12 @@ func (self *SGuest) PerformCreatedisk(ctx context.Context, userCred mcclient.Tok pendingUsage := &SQuota{ Storage: diskSize, } - err = QuotaManager.CheckSetPendingQuota(ctx, userCred, rbacutils.ScopeProject, self.GetOwnerId(), host.GetQuotaPlatformID(), pendingUsage) + keys, err := self.GetQuotaKeys() + if err != nil { + return nil, err + } + pendingUsage.SetKeys(keys) + err = QuotaManager.CheckSetPendingQuota(ctx, userCred, pendingUsage) if err != nil { logclient.AddActionLogWithContext(ctx, self, logclient.ACT_CREATE, err.Error(), userCred, false) return nil, httperrors.NewOutOfQuotaError(err.Error()) @@ -1535,7 +1544,7 @@ func (self *SGuest) PerformCreatedisk(ctx context.Context, userCred mcclient.Tok err = self.CreateDisksOnHost(ctx, userCred, host, disksConf, pendingUsage, false, false, nil, nil, false) if err != nil { - QuotaManager.CancelPendingUsage(ctx, userCred, rbacutils.ScopeProject, self.GetOwnerId(), host.GetQuotaPlatformID(), nil, pendingUsage) + QuotaManager.CancelPendingUsage(ctx, userCred, pendingUsage, pendingUsage) logclient.AddActionLogWithContext(ctx, self, logclient.ACT_CREATE, err.Error(), userCred, false) return nil, httperrors.NewBadRequestError(err.Error()) } @@ -2089,22 +2098,25 @@ func (self *SGuest) PerformAttachnetwork(ctx context.Context, userCred mcclient. inicCnt = 1 ibw = conf.BwLimit } - pendingUsage := &SQuota{ + pendingUsage := &SRegionQuota{ Port: inicCnt, Eport: enicCnt, Bw: ibw, Ebw: ebw, } - ownerId := self.GetOwnerId() - quotaPlatform := self.GetQuotaPlatformID() - err = QuotaManager.CheckSetPendingQuota(ctx, userCred, rbacutils.ScopeProject, ownerId, quotaPlatform, pendingUsage) + keys, err := self.GetQuotaKeys() + if err != nil { + return nil, err + } + pendingUsage.SetKeys(keys) + err = QuotaManager.CheckSetPendingQuota(ctx, userCred, pendingUsage) if err != nil { return nil, httperrors.NewOutOfQuotaError(err.Error()) } host := self.GetHost() _, err = self.attach2NetworkDesc(ctx, userCred, host, conf, pendingUsage, nil) if err != nil { - QuotaManager.CancelPendingUsage(ctx, userCred, rbacutils.ScopeProject, ownerId, quotaPlatform, nil, pendingUsage) + QuotaManager.CancelPendingUsage(ctx, userCred, nil, pendingUsage) return nil, httperrors.NewBadRequestError(err.Error()) } host.ClearSchedDescCache() @@ -2378,19 +2390,22 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T pendingUsage.Storage = addDisk } - quotaPlatform := self.GetQuotaPlatformID() - + keys, err := self.GetQuotaKeys() + if err != nil { + return nil, err + } + pendingUsage.SetKeys(keys) if !pendingUsage.IsEmpty() { - err := QuotaManager.CheckSetPendingQuota(ctx, userCred, rbacutils.ScopeProject, self.GetOwnerId(), quotaPlatform, pendingUsage) + err := QuotaManager.CheckSetPendingQuota(ctx, userCred, pendingUsage) if err != nil { - return nil, httperrors.NewOutOfQuotaError("Check set pending quota error %s", err) + return nil, err } } if len(newDisks) > 0 { err := self.CreateDisksOnHost(ctx, userCred, host, newDisks, pendingUsage, false, false, nil, nil, false) if err != nil { - QuotaManager.CancelPendingUsage(ctx, userCred, rbacutils.ScopeProject, self.GetOwnerId(), quotaPlatform, nil, pendingUsage) + QuotaManager.CancelPendingUsage(ctx, userCred, nil, pendingUsage) return nil, httperrors.NewBadRequestError("Create disk on host error: %s", err) } confs.Add(jsonutils.Marshal(newDisks), "create") @@ -2408,13 +2423,13 @@ func (self *SGuest) confToSchedDesc(addCpu, addMem, addDisk int) *schedapi.Sched ServerConfig: schedapi.ServerConfig{ ServerConfigs: &api.ServerConfigs{ Hypervisor: self.Hypervisor, - Project: self.ProjectId, - Domain: self.DomainId, PreferHost: self.HostId, Disks: []*api.DiskConfig{diskInfo}, }, - Memory: addMem, - Ncpu: addCpu, + Memory: addMem, + Ncpu: addCpu, + Project: self.ProjectId, + Domain: self.DomainId, }, } return desc @@ -2850,11 +2865,13 @@ func (self *SGuest) PerformCreateEip(ctx context.Context, userCred mcclient.Toke return nil, err } - quotaPlatform := self.GetQuotaPlatformID() - - eipPendingUsage := &SQuota{Eip: 1} - ownerCred := self.GetOwnerId() - err = QuotaManager.CheckSetPendingQuota(ctx, userCred, rbacutils.ScopeProject, ownerCred, quotaPlatform, eipPendingUsage) + eipPendingUsage := &SRegionQuota{Eip: 1} + keys, err := self.GetRegionalQuotaKeys() + if err != nil { + return nil, err + } + eipPendingUsage.SetKeys(keys) + err = RegionQuotaManager.CheckSetPendingQuota(ctx, userCred, eipPendingUsage) if err != nil { return nil, httperrors.NewOutOfQuotaError("Out of eip quota: %s", err) } @@ -3203,10 +3220,13 @@ func (self *SGuest) PerformCreateBackup(ctx context.Context, userCred mcclient.T return nil, httperrors.NewBadRequestError("Cannot create backup with snapshot") } - quotaPlatform := self.GetQuotaPlatformID() - req := self.getGuestBackupResourceRequirements(ctx, userCred) - err = QuotaManager.CheckSetPendingQuota(ctx, userCred, rbacutils.ScopeProject, self.GetOwnerId(), quotaPlatform, &req) + keys, err := self.GetQuotaKeys() + if err != nil { + return nil, err + } + req.SetKeys(keys) + err = QuotaManager.CheckSetPendingQuota(ctx, userCred, &req) if err != nil { return nil, httperrors.NewOutOfQuotaError(err.Error()) } @@ -3215,7 +3235,7 @@ func (self *SGuest) PerformCreateBackup(ctx context.Context, userCred mcclient.T params.Set("guest_status", jsonutils.NewString(self.Status)) task, err := taskman.TaskManager.NewTask(ctx, "GuestCreateBackupTask", self, userCred, params, "", "", &req) if err != nil { - QuotaManager.CancelPendingUsage(ctx, userCred, rbacutils.ScopeProject, self.GetOwnerId(), quotaPlatform, nil, &req) + QuotaManager.CancelPendingUsage(ctx, userCred, nil, &req) log.Errorln(err) return nil, err } else { @@ -4053,8 +4073,11 @@ func (self *SGuest) AllowPerformInstanceSnapshot(ctx context.Context, } func (self *SGuest) validateCreateInstanceSnapshot( - ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject, -) (*SQuota, error) { + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + data jsonutils.JSONObject, +) (*SRegionQuota, error) { if self.Hypervisor != api.HYPERVISOR_KVM { return nil, httperrors.NewBadRequestError("guest hypervisor %s can't create instance snapshot", self.Hypervisor) @@ -4099,9 +4122,13 @@ func (self *SGuest) validateCreateInstanceSnapshot( } } } - quotaPlatform := self.GetQuotaPlatformID() - pendingUsage := &SQuota{Snapshot: len(disks)} - err = QuotaManager.CheckSetPendingQuota(ctx, userCred, rbacutils.ScopeProject, ownerId, quotaPlatform, pendingUsage) + pendingUsage := &SRegionQuota{Snapshot: len(disks)} + keys, err := self.GetRegionalQuotaKeys() + if err != nil { + return nil, err + } + pendingUsage.SetKeys(keys) + err = RegionQuotaManager.CheckSetPendingQuota(ctx, userCred, pendingUsage) if err != nil { return nil, httperrors.NewOutOfQuotaError("Check set pending quota error %s", err) } @@ -4125,22 +4152,25 @@ func (self *SGuest) PerformInstanceSnapshot( instanceSnapshot, err := InstanceSnapshotManager.CreateInstanceSnapshot(ctx, ownerId, self, name, false) if err != nil { QuotaManager.CancelPendingUsage( - ctx, userCred, rbacutils.ScopeProject, ownerId, self.GetQuotaPlatformID(), pendingUsage, pendingUsage) + ctx, userCred, pendingUsage, pendingUsage) return nil, httperrors.NewInternalServerError("create instance snapshot failed: %s", err) } err = self.InstaceCreateSnapshot(ctx, userCred, ownerId, instanceSnapshot, pendingUsage) if err != nil { - QuotaManager.CancelPendingUsage( - ctx, userCred, rbacutils.ScopeProject, ownerId, self.GetQuotaPlatformID(), pendingUsage, pendingUsage) + RegionQuotaManager.CancelPendingUsage( + ctx, userCred, pendingUsage, pendingUsage) return nil, httperrors.NewInternalServerError("start create snapshot task failed: %s", err) } return nil, nil } func (self *SGuest) InstaceCreateSnapshot( - ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, - instanceSnapshot *SInstanceSnapshot, pendingUsage *SQuota) error { - + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + instanceSnapshot *SInstanceSnapshot, + pendingUsage *SRegionQuota, +) error { self.SetStatus(userCred, api.VM_START_INSTANCE_SNAPSHOT, "instance snapshot") return instanceSnapshot.StartCreateInstanceSnapshotTask(ctx, userCred, ownerId, pendingUsage, "") } @@ -4224,44 +4254,52 @@ func (self *SGuest) PerformSnapshotAndClone( return nil, err } // set guest pending usage - pendingUsage, err := self.getGuestUsage(int(count)) + pendingUsage, pendingRegionUsage, err := self.getGuestUsage(int(count)) + keys, err := self.GetQuotaKeys() if err != nil { - QuotaManager.CancelPendingUsage(ctx, userCred, rbacutils.ScopeProject, self.GetOwnerId(), - self.GetQuotaPlatformID(), snapshotUsage, snapshotUsage) + QuotaManager.CancelPendingUsage(ctx, userCred, snapshotUsage, snapshotUsage) return nil, err } - err = QuotaManager.CheckSetPendingQuota(ctx, userCred, - rbacutils.ScopeProject, self.GetOwnerId(), self.GetQuotaPlatformID(), pendingUsage) + pendingUsage.SetKeys(keys) + err = QuotaManager.CheckSetPendingQuota(ctx, userCred, &pendingUsage) if err != nil { - QuotaManager.CancelPendingUsage(ctx, userCred, rbacutils.ScopeProject, self.GetOwnerId(), - self.GetQuotaPlatformID(), snapshotUsage, snapshotUsage) + QuotaManager.CancelPendingUsage(ctx, userCred, snapshotUsage, snapshotUsage) return nil, httperrors.NewOutOfQuotaError("Check set pending quota error %s", err) } - pendingUsage.Snapshot = snapshotUsage.Snapshot + regionKeys, err := self.GetRegionalQuotaKeys() + if err != nil { + QuotaManager.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage) + return nil, err + } + pendingRegionUsage.SetKeys(regionKeys) + err = RegionQuotaManager.CheckSetPendingQuota(ctx, userCred, &pendingRegionUsage) + if err != nil { + QuotaManager.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage) + return nil, err + } + pendingRegionUsage.Snapshot = snapshotUsage.Snapshot instanceSnapshotName, err := db.GenerateName(InstanceSnapshotManager, self.GetOwnerId(), fmt.Sprintf("%s-%s", newlyGuestName, rand.String(8))) if err != nil { - QuotaManager.CancelPendingUsage(ctx, userCred, rbacutils.ScopeProject, self.GetOwnerId(), - self.GetQuotaPlatformID(), pendingUsage, pendingUsage) + QuotaManager.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage) + RegionQuotaManager.CancelPendingUsage(ctx, userCred, &pendingRegionUsage, &pendingRegionUsage) return nil, httperrors.NewInternalServerError("Generate snapshot name failed %s", err) } instanceSnapshot, err := InstanceSnapshotManager.CreateInstanceSnapshot( ctx, self.GetOwnerId(), self, instanceSnapshotName, jsonutils.QueryBoolean(data, "auto_delete_instance_snapshot", false)) if err != nil { - QuotaManager.CancelPendingUsage( - ctx, userCred, rbacutils.ScopeProject, self.GetOwnerId(), - self.GetQuotaPlatformID(), pendingUsage, pendingUsage) + QuotaManager.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage) + RegionQuotaManager.CancelPendingUsage(ctx, userCred, &pendingRegionUsage, &pendingRegionUsage) return nil, httperrors.NewInternalServerError("create instance snapshot failed: %s", err) } err = self.StartInstanceSnapshotAndCloneTask( - ctx, userCred, newlyGuestName, pendingUsage, instanceSnapshot, data.(*jsonutils.JSONDict)) + ctx, userCred, newlyGuestName, &pendingUsage, &pendingRegionUsage, instanceSnapshot, data.(*jsonutils.JSONDict)) if err != nil { - QuotaManager.CancelPendingUsage( - ctx, userCred, rbacutils.ScopeProject, self.GetOwnerId(), - self.GetQuotaPlatformID(), pendingUsage, pendingUsage) + QuotaManager.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage) + RegionQuotaManager.CancelPendingUsage(ctx, userCred, &pendingRegionUsage, &pendingRegionUsage) return nil, err } return nil, nil @@ -4269,12 +4307,12 @@ func (self *SGuest) PerformSnapshotAndClone( func (self *SGuest) StartInstanceSnapshotAndCloneTask( ctx context.Context, userCred mcclient.TokenCredential, newlyGuestName string, - pendingUsage *SQuota, instanceSnapshot *SInstanceSnapshot, data *jsonutils.JSONDict) error { + pendingUsage *SQuota, pendingRegionUsage *SRegionQuota, instanceSnapshot *SInstanceSnapshot, data *jsonutils.JSONDict) error { params := jsonutils.NewDict() params.Set("guest_params", data) if task, err := taskman.TaskManager.NewTask( - ctx, "InstanceSnapshotAndCloneTask", instanceSnapshot, userCred, params, "", "", pendingUsage); err != nil { + ctx, "InstanceSnapshotAndCloneTask", instanceSnapshot, userCred, params, "", "", pendingUsage, pendingRegionUsage); err != nil { return err } else { self.SetStatus(userCred, api.VM_START_INSTANCE_SNAPSHOT, "instance snapshot") diff --git a/pkg/compute/models/guestdrivers.go b/pkg/compute/models/guestdrivers.go index 6e3b261007..e3a9d116ca 100644 --- a/pkg/compute/models/guestdrivers.go +++ b/pkg/compute/models/guestdrivers.go @@ -27,6 +27,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/billing" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) type IGuestScheduleDriver interface { @@ -41,7 +42,7 @@ type IGuestDriver interface { GetHypervisor() string GetProvider() string - GetQuotaPlatformID() []string + GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) SComputeResourceKeys GetMaxVCpuCount() int GetMaxVMemSizeGB() int diff --git a/pkg/compute/models/guestnetworks.go b/pkg/compute/models/guestnetworks.go index 9ca4832b15..2d9efe5360 100644 --- a/pkg/compute/models/guestnetworks.go +++ b/pkg/compute/models/guestnetworks.go @@ -472,26 +472,24 @@ func (self *SGuestnetwork) Detach(ctx context.Context, userCred mcclient.TokenCr return db.DetachJoint(ctx, userCred, self) } -func totalGuestNicCount(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, rangeObj db.IStandaloneModel, includeSystem bool) GuestnicsCount { +func totalGuestNicCount( + scope rbacutils.TRbacScope, + ownerId mcclient.IIdentityProvider, + rangeObjs []db.IStandaloneModel, + includeSystem bool, + providers []string, + brands []string, + cloudEnv string, +) GuestnicsCount { guests := GuestManager.Query().SubQuery() + hosts := HostManager.Query().SubQuery() guestnics := GuestnetworkManager.Query().SubQuery() - q := guestnics.Query().Join(guests, sqlchemy.Equals(guests.Field("id"), guestnics.Field("guest_id"))) - if rangeObj != nil { - if rangeObj.Keyword() == "zone" { - hosts := HostManager.Query().SubQuery() - q = q.Join(hosts, sqlchemy.Equals(guests.Field("host_id"), hosts.Field("id"))).Filter(sqlchemy.Equals(hosts.Field("zone_id"), rangeObj.GetId())) - } else if rangeObj.Keyword() == "wire" { - hosts := HostManager.Query().SubQuery() - hostwires := HostwireManager.Query().SubQuery() - q = q.Join(hosts, sqlchemy.Equals(guests.Field("host_id"), hosts.Field("id"))).Join(hostwires, sqlchemy.Equals(hosts.Field("id"), hostwires.Field("host_id"))).Filter(sqlchemy.Equals(hostwires.Field("wire_id"), rangeObj.GetId())) - } else if rangeObj.Keyword() == "host" { + q := guestnics.Query() + q = q.Join(guests, sqlchemy.Equals(guests.Field("id"), guestnics.Field("guest_id"))) + q = q.Join(hosts, sqlchemy.Equals(guests.Field("host_id"), hosts.Field("id"))) - } else if rangeObj.Keyword() == "vcenter" { - - } else if rangeObj.Keyword() == "schedtag" { - - } - } + q = CloudProviderFilter(q, hosts.Field("manager_id"), providers, brands, cloudEnv) + q = rangeObjectsFilter(q, rangeObjs, nil, hosts.Field("zone_id"), hosts.Field("manager_id")) switch scope { case rbacutils.ScopeSystem: diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index f1ff8585bc..4830fbd96a 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -29,7 +29,7 @@ import ( "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/compare" - errors_aggr "yunion.io/x/pkg/util/errors" + // errors_aggr "yunion.io/x/pkg/util/errors" "yunion.io/x/pkg/util/netutils" "yunion.io/x/pkg/util/osprofile" "yunion.io/x/pkg/util/regutils" @@ -845,6 +845,37 @@ func (self *SGuest) ValidateUpdateData(ctx context.Context, userCred mcclient.To return self.SVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data) } +func serverCreateInput2ComputeQuotaKeys(input api.ServerCreateInput, ownerId mcclient.IIdentityProvider) SComputeResourceKeys { + // input.Hypervisor must be set + keys := GetDriver(input.Hypervisor).GetComputeQuotaKeys( + rbacutils.ScopeProject, + ownerId, + "", + ) + if len(input.PreferHost) > 0 { + hostObj, _ := HostManager.FetchById(input.PreferHost) + host := hostObj.(*SHost) + zone := host.GetZone() + keys.ZoneId = zone.Id + keys.RegionId = zone.CloudregionId + } else if len(input.PreferZone) > 0 { + zoneObj, _ := ZoneManager.FetchById(input.PreferZone) + zone := zoneObj.(*SZone) + keys.ZoneId = zone.Id + keys.RegionId = zone.CloudregionId + } else if len(input.PreferWire) > 0 { + wireObj, _ := WireManager.FetchById(input.PreferWire) + wire := wireObj.(*SWire) + zone := wire.GetZone() + keys.ZoneId = zone.Id + keys.RegionId = zone.CloudregionId + } else if len(input.PreferRegion) > 0 { + regionObj, _ := CloudregionManager.FetchById(input.PreferRegion) + keys.RegionId = regionObj.GetId() + } + return keys +} + func (manager *SGuestManager) BatchPreValidate( ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict, count int, @@ -853,8 +884,8 @@ func (manager *SGuestManager) BatchPreValidate( if err != nil { return nil, err } - if !input.IsSystem { - reqQuota, err := manager.checkCreateQuota(ctx, userCred, ownerId, input, input.Backup, count) + if input.IsSystem == nil || *input.IsSystem == false { + reqQuota, reqRegionQuota, err := manager.checkCreateQuota(ctx, userCred, ownerId, *input, input.Backup, count) if err != nil { return nil, err } @@ -862,20 +893,22 @@ func (manager *SGuestManager) BatchPreValidate( Cpu: reqQuota.Cpu / count, Memory: reqQuota.Memory / count, Storage: reqQuota.Storage / count, - Port: reqQuota.Port / count, - Eport: reqQuota.Eport / count, - Bw: reqQuota.Bw / count, - Ebw: reqQuota.Ebw / count, IsolatedDevice: reqQuota.IsolatedDevice / count, - Eip: reqQuota.Eip / count, } - quotaPlatform := make([]string, 0) - if len(input.Hypervisor) > 0 { - quotaPlatform = GetDriver(input.Hypervisor).GetQuotaPlatformID() + regionQuota := &SRegionQuota{ + Port: reqRegionQuota.Port / count, + Eport: reqRegionQuota.Eport / count, + Bw: reqRegionQuota.Bw / count, + Ebw: reqRegionQuota.Ebw / count, + Eip: reqRegionQuota.Eip / count, } + keys := serverCreateInput2ComputeQuotaKeys(*input, ownerId) + regionKeys := keys.SRegionalCloudResourceKeys + quota.SetKeys(keys) + regionQuota.SetKeys(regionKeys) return func() { - QuotaManager.CancelPendingUsage( - ctx, userCred, rbacutils.ScopeProject, ownerId, quotaPlatform, quota, quota) + QuotaManager.CancelPendingUsage(ctx, userCred, quota, quota) + RegionQuotaManager.CancelPendingUsage(ctx, userCred, regionQuota, regionQuota) }, nil } return nil, nil @@ -1243,15 +1276,10 @@ func (manager *SGuestManager) validateCreateData( } } - data, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.JSON(input)) + input.VirtualResourceCreateInput, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.VirtualResourceCreateInput) if err != nil { return nil, err } - if err := data.Unmarshal(input); err != nil { - return nil, err - } - - log.Debugf("Create data: %s", data) if err := userdata.ValidateUserdata(input.UserData); err != nil { return nil, httperrors.NewInputParameterError("Invalid userdata: %v", err) @@ -1276,8 +1304,8 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m if err != nil { return nil, err } - if !input.IsSystem { - _, err = manager.checkCreateQuota(ctx, userCred, ownerId, input, input.Backup, 1) + if input.IsSystem == nil || !(*input.IsSystem) { + _, _, err = manager.checkCreateQuota(ctx, userCred, ownerId, *input, input.Backup, 1) if err != nil { return nil, err } @@ -1331,20 +1359,23 @@ func (self *SGuest) PostUpdate(ctx context.Context, userCred mcclient.TokenCrede } func (manager *SGuestManager) checkCreateQuota( - ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, - input *api.ServerCreateInput, hasBackup bool, count int) (*SQuota, error) { - - req := getGuestResourceRequirements(ctx, userCred, input, count, hasBackup) - quotaPlatform := make([]string, 0) - if len(input.Hypervisor) > 0 { - quotaPlatform = GetDriver(input.Hypervisor).GetQuotaPlatformID() - } - err := QuotaManager.CheckSetPendingQuota(ctx, userCred, rbacutils.ScopeProject, ownerId, quotaPlatform, &req) + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + input api.ServerCreateInput, + hasBackup bool, + count int, +) (*SQuota, *SRegionQuota, error) { + req, regionReq := getGuestResourceRequirements(ctx, userCred, input, ownerId, count, hasBackup) + err := QuotaManager.CheckSetPendingQuota(ctx, userCred, &req) if err != nil { - return nil, httperrors.NewOutOfQuotaError(err.Error()) - } else { - return &req, nil + return nil, nil, err } + err = RegionQuotaManager.CheckSetPendingQuota(ctx, userCred, ®ionReq) + if err != nil { + return nil, nil, err + } + return &req, ®ionReq, nil } func (self *SGuest) checkUpdateQuota(ctx context.Context, userCred mcclient.TokenCredential, vcpuCount int, vmemSize int) error { @@ -1358,13 +1389,24 @@ func (self *SGuest) checkUpdateQuota(ctx context.Context, userCred mcclient.Toke req.Memory = vmemSize - self.VmemSize } - quotaPlatform := self.GetQuotaPlatformID() - _, err := QuotaManager.CheckQuota(ctx, userCred, rbacutils.ScopeProject, self.GetOwnerId(), quotaPlatform, &req) + keys, err := self.GetQuotaKeys() + if err != nil { + return errors.Wrap(err, "self.GetQuotaKeys") + } + req.SetKeys(keys) + err = QuotaManager.CheckQuota(ctx, &req) return err } -func getGuestResourceRequirements(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput, count int, hasBackup bool) SQuota { +func getGuestResourceRequirements( + ctx context.Context, + userCred mcclient.TokenCredential, + input api.ServerCreateInput, + ownerId mcclient.IIdentityProvider, + count int, + hasBackup bool, +) (SQuota, SRegionQuota) { vcpuCount := input.VcpuCount if vcpuCount == 0 { vcpuCount = 1 @@ -1405,17 +1447,23 @@ func getGuestResourceRequirements(ctx context.Context, userCred mcclient.TokenCr eipCnt = 1 } - return SQuota{ + req := SQuota{ Cpu: int(vcpuCount) * count, Memory: int(vmemSize) * count, Storage: diskSize * count, - Port: iNicCnt * count, - Eport: eNicCnt * count, - Bw: iBw * count, - Ebw: eBw * count, IsolatedDevice: devCount * count, - Eip: eipCnt * count, } + regionReq := SRegionQuota{ + Port: iNicCnt * count, + Eport: eNicCnt * count, + Bw: iBw * count, + Ebw: eBw * count, + Eip: eipCnt * count, + } + keys := serverCreateInput2ComputeQuotaKeys(input, ownerId) + req.SetKeys(keys) + regionReq.SetKeys(keys.SRegionalCloudResourceKeys) + return req, regionReq } func (guest *SGuest) getGuestBackupResourceRequirements(ctx context.Context, userCred mcclient.TokenCredential) SQuota { @@ -1503,14 +1551,14 @@ func (manager *SGuestManager) SetPropertiesWithInstanceSnapshot( } } -func (manager *SGuestManager) OnCreateComplete(ctx context.Context, items []db.IModel, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) { - input := new(api.ServerCreateInput) - data.Unmarshal(input) +func (manager *SGuestManager) OnCreateComplete(ctx context.Context, items []db.IModel, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { + input := api.ServerCreateInput{} + data.Unmarshal(&input) if len(input.InstanceSnapshotId) > 0 { manager.SetPropertiesWithInstanceSnapshot(ctx, userCred, input.InstanceSnapshotId, items) } - pendingUsage := getGuestResourceRequirements(ctx, userCred, input, len(items), input.Backup) - RunBatchCreateTask(ctx, items, userCred, data, pendingUsage, "GuestBatchCreateTask", input.ParentTaskId) + pendingUsage, pendingRegionUsage := getGuestResourceRequirements(ctx, userCred, input, ownerId, len(items), input.Backup) + RunBatchCreateTask(ctx, items, userCred, data, pendingUsage, pendingRegionUsage, "GuestBatchCreateTask", input.ParentTaskId) } func (guest *SGuest) GetGroups() []SGroupguest { @@ -2383,12 +2431,12 @@ func (manager *SGuestManager) newCloudVM(ctx context.Context, userCred mcclient. func (manager *SGuestManager) TotalCount( scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, - rangeObj db.IStandaloneModel, + rangeObjs []db.IStandaloneModel, status []string, hypervisors []string, includeSystem bool, pendingDelete bool, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string, ) SGuestCountStat { - return totalGuestResourceCount(scope, ownerId, rangeObj, status, hypervisors, includeSystem, pendingDelete, hostTypes, resourceTypes, providers, brands, cloudEnv) + return totalGuestResourceCount(scope, ownerId, rangeObjs, status, hypervisors, includeSystem, pendingDelete, hostTypes, resourceTypes, providers, brands, cloudEnv) } func (self *SGuest) detachNetworks(ctx context.Context, userCred mcclient.TokenCredential, gns []SGuestnetwork, reserve bool, deploy bool) error { @@ -2499,7 +2547,7 @@ func (self *SGuest) attach2NetworkOnce(ctx context.Context, userCred mcclient.To network.updateGuestNetmap(guestnic) bwLimit = guestnic.getBandwidth() if pendingUsage != nil && len(teamWithMac) == 0 { - cancelUsage := SQuota{} + cancelUsage := SRegionQuota{} if network.IsExitNetwork() { cancelUsage.Eport = 1 cancelUsage.Ebw = bwLimit @@ -2507,8 +2555,12 @@ func (self *SGuest) attach2NetworkOnce(ctx context.Context, userCred mcclient.To cancelUsage.Port = 1 cancelUsage.Bw = bwLimit } - quotaPlatform := self.GetQuotaPlatformID() - err = QuotaManager.CancelPendingUsage(ctx, userCred, rbacutils.ScopeProject, self.GetOwnerId(), quotaPlatform, pendingUsage, &cancelUsage) + keys, err := self.GetRegionalQuotaKeys() + if err != nil { + log.Warningf("self.GetRegionalQuotaKeys fail %s", err) + } + cancelUsage.SetKeys(keys) + err = QuotaManager.CancelPendingUsage(ctx, userCred, pendingUsage, &cancelUsage) if err != nil { log.Warningf("QuotaManager.CancelPendingUsage fail %s", err) } @@ -2814,14 +2866,14 @@ func (self *SGuest) SyncVMDisks(ctx context.Context, userCred mcclient.TokenCred return result } -func filterGuestByRange(q *sqlchemy.SQuery, rangeObj db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string) *sqlchemy.SQuery { +func filterGuestByRange(q *sqlchemy.SQuery, rangeObjs []db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string) *sqlchemy.SQuery { hosts := HostManager.Query().SubQuery() q = q.Join(hosts, sqlchemy.Equals(hosts.Field("id"), q.Field("host_id"))) //q = q.Filter(sqlchemy.IsTrue(hosts.Field("enabled"))) // q = q.Filter(sqlchemy.Equals(hosts.Field("host_status"), HOST_ONLINE)) - q = AttachUsageQuery(q, hosts, hostTypes, resourceTypes, providers, brands, cloudEnv, rangeObj) + q = AttachUsageQuery(q, hosts, hostTypes, resourceTypes, providers, brands, cloudEnv, rangeObjs) return q } @@ -2840,7 +2892,7 @@ type SGuestCountStat struct { func totalGuestResourceCount( scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, - rangeObj db.IStandaloneModel, + rangeObjs []db.IStandaloneModel, status []string, hypervisors []string, includeSystem bool, @@ -2899,7 +2951,7 @@ func totalGuestResourceCount( q = q.LeftJoin(isoDevSubQuery, sqlchemy.Equals(isoDevSubQuery.Field("guest_id"), guests.Field("id"))) - q = filterGuestByRange(q, rangeObj, hostTypes, resourceTypes, providers, brands, cloudEnv) + q = filterGuestByRange(q, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv) switch scope { case rbacutils.ScopeSystem: @@ -3007,7 +3059,7 @@ func (self *SGuest) attach2NetworkDesc( } errs = append(errs, err) } - return nil, errors_aggr.NewAggregate(errs) + return nil, errors.NewAggregate(errs) } else { netConfig.Network = "" return self.attach2RandomNetwork(ctx, userCred, host, netConfig, pendingUsage) @@ -3101,11 +3153,14 @@ func (self *SGuest) createDiskOnStorage(ctx context.Context, userCred mcclient.T return nil, err } - quotaPlatform := self.GetQuotaPlatformID() - cancelUsage := SQuota{} cancelUsage.Storage = disk.DiskSize - err = QuotaManager.CancelPendingUsage(ctx, userCred, rbacutils.ScopeProject, self.GetOwnerId(), quotaPlatform, pendingUsage, &cancelUsage) + keys, err := self.GetQuotaKeys() + if err != nil { + return nil, err + } + cancelUsage.SetKeys(keys) + err = QuotaManager.CancelPendingUsage(ctx, userCred, pendingUsage, &cancelUsage) return disk, nil } @@ -3188,10 +3243,13 @@ func (self *SGuest) createIsolatedDeviceOnHost(ctx context.Context, userCred mcc return err } - quotaPlatform := self.GetQuotaPlatformID() - cancelUsage := SQuota{IsolatedDevice: 1} - err = QuotaManager.CancelPendingUsage(ctx, userCred, rbacutils.ScopeProject, self.GetOwnerId(), quotaPlatform, pendingUsage, &cancelUsage) + keys, err := self.GetQuotaKeys() + if err != nil { + return err + } + cancelUsage.SetKeys(keys) + err = QuotaManager.CancelPendingUsage(ctx, userCred, pendingUsage, &cancelUsage) return err } @@ -4619,7 +4677,9 @@ func (self *SGuest) ToCreateInput(userCred mcclient.TokenCredential) *api.Server userInput.KeypairId = genInput.KeypairId userInput.EipBw = genInput.EipBw userInput.EipChargeType = genInput.EipChargeType - userInput.Project = genInput.Project + // cloned server should belongs to the project creating it + userInput.Project = userCred.GetProjectId() + userInput.Domain = userCred.GetProjectDomainId() userInput.Secgroups = []string{} secgroups := self.GetSecgroups() for _, secgroup := range secgroups { @@ -4658,7 +4718,7 @@ func (self *SGuest) toCreateInput() *api.ServerCreateInput { *r.DisableDelete = self.DisableDelete.Bool() r.ShutdownBehavior = self.ShutdownBehavior // ignore r.DeployConfigs - r.IsSystem = self.IsSystem + r.IsSystem = &self.IsSystem r.SecgroupId = self.SecgrpId r.ServerConfigs = new(api.ServerConfigs) @@ -4834,22 +4894,30 @@ func (self *SGuest) GetDiskSnapshotsNotInInstanceSnapshots() ([]SSnapshot, error return snapshots, nil } -func (self *SGuest) getGuestUsage(guestCount int) (*SQuota, error) { - usage := new(SQuota) +func (self *SGuest) getGuestUsage(guestCount int) (SQuota, SRegionQuota, error) { + usage := SQuota{} + regionUsage := SRegionQuota{} usage.Cpu = int(self.VcpuCount) * guestCount usage.Memory = int(self.VmemSize * guestCount) diskSize := self.getDiskSize() if diskSize < 0 { - return nil, httperrors.NewInternalServerError("fetch disk size failed") + return usage, regionUsage, httperrors.NewInternalServerError("fetch disk size failed") } usage.Storage = self.getDiskSize() * guestCount netCount, err := self.NetworkCount() - if err != nil && err != sql.ErrNoRows { - return nil, err + if err != nil && errors.Cause(err) != sql.ErrNoRows { + return usage, regionUsage, err } - usage.Port = netCount - usage.Bw = self.getBandwidth(false) - return usage, err + regionUsage.Port = netCount + regionUsage.Bw = self.getBandwidth(false) + eip, err := self.GetEip() + if err != nil && errors.Cause(err) != sql.ErrNoRows { + return usage, regionUsage, err + } + if eip != nil { + regionUsage.Eip = 1 + } + return usage, regionUsage, nil } func (self *SGuestManager) checkGuestImage(ctx context.Context, input *api.ServerCreateInput) error { @@ -4918,3 +4986,45 @@ func (self *SGuest) GetDiskIndex(diskId string) int8 { } return -1 } + +func (guest *SGuest) GetRegionalQuotaKeys() (quotas.IQuotaKeys, error) { + host := guest.GetHost() + if host == nil { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "no valid host") + } + provider := host.GetCloudprovider() + if provider == nil { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "no valid manager") + } + region := host.GetRegion() + if region == nil { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "no valid region") + } + return fetchRegionalQuotaKeys(rbacutils.ScopeProject, guest.GetOwnerId(), region, provider), nil +} + +func (guest *SGuest) GetQuotaKeys() (quotas.IQuotaKeys, error) { + host := guest.GetHost() + if host == nil { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "no valid host") + } + provider := host.GetCloudprovider() + if provider == nil { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "no valid manager") + } + zone := host.GetZone() + if zone == nil { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "no valid zone") + } + hypervisor := guest.Hypervisor + if !utils.IsInStringArray(hypervisor, api.ONECLOUD_HYPERVISORS) { + hypervisor = "" + } + return fetchComputeQuotaKeys( + rbacutils.ScopeProject, + guest.GetOwnerId(), + zone, + provider, + hypervisor, + ), nil +} diff --git a/pkg/compute/models/helper.go b/pkg/compute/models/helper.go index 9144950f59..0c317f1ecc 100644 --- a/pkg/compute/models/helper.go +++ b/pkg/compute/models/helper.go @@ -38,6 +38,7 @@ func RunBatchCreateTask( userCred mcclient.TokenCredential, data jsonutils.JSONObject, pendingUsage SQuota, + pendingRegionUsage SRegionQuota, taskName string, parentTaskId string, ) { @@ -46,7 +47,7 @@ func RunBatchCreateTask( taskItems[i] = t.(db.IStandaloneModel) } params := data.(*jsonutils.JSONDict) - task, err := taskman.TaskManager.NewParallelTask(ctx, taskName, taskItems, userCred, params, parentTaskId, "", &pendingUsage) + task, err := taskman.TaskManager.NewParallelTask(ctx, taskName, taskItems, userCred, params, parentTaskId, "", &pendingUsage, &pendingRegionUsage) if err != nil { log.Errorf("%s newTask error %s", taskName, err) } else { diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 5deb992fab..05c0fb4ca2 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -36,6 +36,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" billing_api "yunion.io/x/onecloud/pkg/apis/billing" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/appsrv" @@ -2027,7 +2028,7 @@ func (manager *SHostManager) FetchHostById(hostId string) *SHost { func (manager *SHostManager) totalCountQ( userCred mcclient.IIdentityProvider, - rangeObj db.IStandaloneModel, + rangeObjs []db.IStandaloneModel, hostStatus, status string, hostTypes []string, resourceTypes []string, @@ -2035,15 +2036,6 @@ func (manager *SHostManager) totalCountQ( enabled, isBaremetal tristate.TriState, ) *sqlchemy.SQuery { hosts := manager.Query().SubQuery() - /* - MemSize int - MemReserved int - MemCmtbound float32 - CpuCount int8 - CpuReserved int8 - CpuCmtbound float32 - StorageSize int - */ q := hosts.Query( hosts.Field("mem_size"), hosts.Field("mem_reserved"), @@ -2073,7 +2065,7 @@ func (manager *SHostManager) totalCountQ( } q = q.Filter(cond(hosts.Field("is_baremetal"))) } - q = AttachUsageQuery(q, hosts, hostTypes, resourceTypes, providers, brands, cloudEnv, rangeObj) + q = AttachUsageQuery(q, hosts, hostTypes, resourceTypes, providers, brands, cloudEnv, rangeObjs) return q } @@ -2152,14 +2144,28 @@ func (manager *SHostManager) calculateCount(q *sqlchemy.SQuery) HostsCountStat { func (manager *SHostManager) TotalCount( userCred mcclient.IIdentityProvider, - rangeObj db.IStandaloneModel, + rangeObjs []db.IStandaloneModel, hostStatus, status string, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string, enabled, isBaremetal tristate.TriState, ) HostsCountStat { - return manager.calculateCount(manager.totalCountQ(userCred, rangeObj, hostStatus, status, hostTypes, resourceTypes, providers, brands, cloudEnv, enabled, isBaremetal)) + return manager.calculateCount( + manager.totalCountQ( + userCred, + rangeObjs, + hostStatus, + status, + hostTypes, + resourceTypes, + providers, + brands, + cloudEnv, + enabled, + isBaremetal, + ), + ) } /* @@ -2779,7 +2785,17 @@ func (manager *SHostManager) ValidateCreateData(ctx context.Context, userCred mc } } - return manager.SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + input := apis.EnabledStatusStandaloneResourceCreateInput{} + err = data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal EnabledStatusStandaloneCreateInput fail %s", err) + } + input, err = manager.SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { + return nil, err + } + data.Update(jsonutils.Marshal(input)) + return data, nil } func (self *SHost) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { diff --git a/pkg/compute/models/hoststorages.go b/pkg/compute/models/hoststorages.go index 1dff80dc8c..a499a3484d 100644 --- a/pkg/compute/models/hoststorages.go +++ b/pkg/compute/models/hoststorages.go @@ -22,6 +22,7 @@ import ( "yunion.io/x/pkg/tristate" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" @@ -135,7 +136,17 @@ func (manager *SHoststorageManager) ValidateCreateData(ctx context.Context, user return nil, err } - return manager.SJointResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + input := apis.JoinResourceBaseCreateInput{} + err := data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal JoinResourceBaseCreateInput fail %s", err) + } + input, err = manager.SJointResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { + return nil, err + } + data.Update(jsonutils.Marshal(input)) + return data, nil } func (self *SHoststorage) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { diff --git a/pkg/compute/models/instance_snapshots.go b/pkg/compute/models/instance_snapshots.go index 69de2fe9ac..66b1f99118 100644 --- a/pkg/compute/models/instance_snapshots.go +++ b/pkg/compute/models/instance_snapshots.go @@ -133,10 +133,14 @@ func (self *SInstanceSnapshot) GetExtraDetails(ctx context.Context, userCred mcc extra = self.getMoreDetails(userCred, extra) return extra, nil } -func (self *SInstanceSnapshot) StartCreateInstanceSnapshotTask( - ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, - pendingUsage quotas.IQuota, parentTaskId string) error { +func (self *SInstanceSnapshot) StartCreateInstanceSnapshotTask( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + pendingUsage quotas.IQuota, + parentTaskId string, +) error { if task, err := taskman.TaskManager.NewTask( ctx, "InstanceSnapshotCreateTask", self, userCred, nil, parentTaskId, "", pendingUsage); err != nil { return err diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go index 2d9b1c90b9..873019c443 100644 --- a/pkg/compute/models/isolated_devices.go +++ b/pkg/compute/models/isolated_devices.go @@ -26,6 +26,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" @@ -117,7 +118,18 @@ func (manager *SIsolatedDeviceManager) ValidateCreateData(ctx context.Context, u name = fmt.Sprintf("dev_%s_%d", host.GetName(), time.Now().UnixNano()) data.Set("name", jsonutils.NewString(name)) } - return manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + + input := apis.StandaloneResourceCreateInput{} + err := data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal StandaloneRes ourceCreateInput fail %s", err) + } + input, err = manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { + return nil, err + } + data.Update(jsonutils.Marshal(input)) + return data, nil } func (self *SIsolatedDevice) AllowUpdateItem(ctx context.Context, userCred mcclient.TokenCredential) bool { @@ -437,8 +449,9 @@ func (manager *SIsolatedDeviceManager) ReleaseDevicesOfGuest(ctx context.Context func (manager *SIsolatedDeviceManager) totalCountQ( devType []string, hostTypes []string, - resourceTypes []string, providers []string, brands []string, cloudEnv string, - rangeObj db.IStandaloneModel, + resourceTypes []string, + providers []string, brands []string, cloudEnv string, + rangeObjs []db.IStandaloneModel, ) *sqlchemy.SQuery { hosts := HostManager.Query().SubQuery() devs := manager.Query().SubQuery() @@ -447,7 +460,7 @@ func (manager *SIsolatedDeviceManager) totalCountQ( if len(devType) != 0 { q = q.Filter(sqlchemy.In(devs.Field("dev_type"), devType)) } - return AttachUsageQuery(q, hosts, hostTypes, resourceTypes, providers, brands, cloudEnv, rangeObj) + return AttachUsageQuery(q, hosts, hostTypes, resourceTypes, providers, brands, cloudEnv, rangeObjs) } type IsolatedDeviceCountStat struct { @@ -455,17 +468,46 @@ type IsolatedDeviceCountStat struct { Gpus int } -func (manager *SIsolatedDeviceManager) totalCount(devType, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string, rangeObj db.IStandaloneModel) (int, error) { - return manager.totalCountQ(devType, hostTypes, resourceTypes, providers, brands, cloudEnv, rangeObj).CountWithError() +func (manager *SIsolatedDeviceManager) totalCount( + devType, + hostTypes []string, + resourceTypes []string, + providers []string, + brands []string, + cloudEnv string, + rangeObjs []db.IStandaloneModel, +) (int, error) { + return manager.totalCountQ( + devType, + hostTypes, + resourceTypes, + providers, + brands, + cloudEnv, + rangeObjs, + ).CountWithError() } -func (manager *SIsolatedDeviceManager) TotalCount(hostType []string, resourceTypes []string, providers []string, brands []string, cloudEnv string, rangeObj db.IStandaloneModel) (IsolatedDeviceCountStat, error) { +func (manager *SIsolatedDeviceManager) TotalCount( + hostType []string, + resourceTypes []string, + providers []string, + brands []string, + cloudEnv string, + rangeObjs []db.IStandaloneModel, +) (IsolatedDeviceCountStat, error) { stat := IsolatedDeviceCountStat{} - devCnt, err := manager.totalCount(nil, hostType, resourceTypes, providers, brands, cloudEnv, rangeObj) + devCnt, err := manager.totalCount( + nil, hostType, resourceTypes, + providers, brands, cloudEnv, + rangeObjs) if err != nil { return stat, err } - gpuCnt, err := manager.totalCount(VALID_GPU_TYPES, hostType, resourceTypes, providers, brands, cloudEnv, rangeObj) + gpuCnt, err := manager.totalCount( + VALID_GPU_TYPES, hostType, resourceTypes, + providers, brands, cloudEnv, + rangeObjs) if err != nil { return stat, err } diff --git a/pkg/compute/models/keypairs.go b/pkg/compute/models/keypairs.go index b12d4da2b9..da33bf8331 100644 --- a/pkg/compute/models/keypairs.go +++ b/pkg/compute/models/keypairs.go @@ -24,6 +24,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" @@ -185,7 +186,17 @@ func (manager *SKeypairManager) ValidateCreateData(ctx context.Context, userCred data.Set("scheme", jsonutils.NewString(scheme)) data.Set("owner_id", jsonutils.NewString(userCred.GetUserId())) - return manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + input := apis.StandaloneResourceCreateInput{} + err = data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal StandaloneRes ourceCreateInput fail %s", err) + } + input, err = manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { + return nil, err + } + data.Update(jsonutils.Marshal(input)) + return data, nil } func (self *SKeypair) ValidateDeleteCondition(ctx context.Context) error { diff --git a/pkg/compute/models/loadbalanceracls.go b/pkg/compute/models/loadbalanceracls.go index 41500b990f..50acf485b7 100644 --- a/pkg/compute/models/loadbalanceracls.go +++ b/pkg/compute/models/loadbalanceracls.go @@ -30,6 +30,7 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" @@ -180,9 +181,17 @@ func (man *SLoadbalancerAclManager) ValidateCreateData(ctx context.Context, user if err != nil { return nil, err } - if _, err := man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data); err != nil { + + input := apis.VirtualResourceCreateInput{} + err = data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal VirtualResourceCreateInput fail %s", err) + } + input, err = man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { return nil, err } + data.Update(jsonutils.Marshal(input)) managerIdV := validators.NewModelIdOrNameValidator("manager", "cloudprovider", ownerId) managerIdV.Optional(true) diff --git a/pkg/compute/models/loadbalanceragents.go b/pkg/compute/models/loadbalanceragents.go index a44b8ffc4e..2a05698d03 100644 --- a/pkg/compute/models/loadbalanceragents.go +++ b/pkg/compute/models/loadbalanceragents.go @@ -28,6 +28,7 @@ import ( "yunion.io/x/pkg/gotypes" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/validators" @@ -364,7 +365,18 @@ func (man *SLoadbalancerAgentManager) ValidateCreateData(ctx context.Context, us otherCluster.Name, otherCluster.Id, vrrpRouterId) } } - return man.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + + input := apis.StandaloneResourceCreateInput{} + err := data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal StandaloneResourceCreateInput fail %s", err) + } + input, err = man.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { + return nil, err + } + data.Update(jsonutils.Marshal(input)) + return data, nil } func (man *SLoadbalancerAgentManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { diff --git a/pkg/compute/models/loadbalancerbackendgroups.go b/pkg/compute/models/loadbalancerbackendgroups.go index 72e784bf5c..365b319b40 100644 --- a/pkg/compute/models/loadbalancerbackendgroups.go +++ b/pkg/compute/models/loadbalancerbackendgroups.go @@ -25,6 +25,7 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" @@ -106,9 +107,18 @@ func (man *SLoadbalancerBackendGroupManager) ValidateCreateData(ctx context.Cont if err != nil { return nil, err } - if _, err := man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data); err != nil { + + input := apis.VirtualResourceCreateInput{} + err = data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal VirtualResourceCreateInput fail %s", err) + } + input, err = man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { return nil, err } + data.Update(jsonutils.Marshal(input)) + lb := lbV.Model.(*SLoadbalancer) data.Set("manager_id", jsonutils.NewString(lb.ManagerId)) data.Set("cloudregion_id", jsonutils.NewString(lb.CloudregionId)) diff --git a/pkg/compute/models/loadbalancerbackends.go b/pkg/compute/models/loadbalancerbackends.go index 1f181bfe41..e0d8727246 100644 --- a/pkg/compute/models/loadbalancerbackends.go +++ b/pkg/compute/models/loadbalancerbackends.go @@ -26,6 +26,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" @@ -158,9 +159,18 @@ func (man *SLoadbalancerBackendManager) ValidateCreateData(ctx context.Context, backendGroup := backendGroupV.Model.(*SLoadbalancerBackendGroup) lb := backendGroup.GetLoadbalancer() var backendModel db.IModel - if _, err := man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data); err != nil { + + input := apis.VirtualResourceCreateInput{} + err := data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal VirtualResourceCreateInput fail %s", err) + } + input, err = man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { return nil, err } + data.Update(jsonutils.Marshal(input)) + region := lb.GetRegion() if region == nil { return nil, httperrors.NewResourceNotFoundError("failed to find region for loadbalancer %s", lb.Name) diff --git a/pkg/compute/models/loadbalancercachedacls.go b/pkg/compute/models/loadbalancercachedacls.go index f5fbfa9119..f0a6a601a7 100644 --- a/pkg/compute/models/loadbalancercachedacls.go +++ b/pkg/compute/models/loadbalancercachedacls.go @@ -24,6 +24,7 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" @@ -134,9 +135,17 @@ func (man *SCachedLoadbalancerAclManager) ValidateCreateData(ctx context.Context data.Set("manager_id", jsonutils.NewString(provider.Id)) name, _ := db.GenerateName(man, ownerId, aclV.Model.GetName()) data.Set("name", jsonutils.NewString(name)) - if _, err := man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data); err != nil { + + input := apis.VirtualResourceCreateInput{} + err = data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal VirtualResourceCreateInput fail %s", err) + } + input, err = man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { return nil, err } + data.Update(jsonutils.Marshal(input)) return data, nil } diff --git a/pkg/compute/models/loadbalancercachedcertificates.go b/pkg/compute/models/loadbalancercachedcertificates.go index 5358b1e8c8..185e93416e 100644 --- a/pkg/compute/models/loadbalancercachedcertificates.go +++ b/pkg/compute/models/loadbalancercachedcertificates.go @@ -25,6 +25,7 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" @@ -165,9 +166,17 @@ func (man *SCachedLoadbalancerCertificateManager) ValidateCreateData(ctx context data.Set("manager_id", jsonutils.NewString(provider.Id)) name, _ := db.GenerateName(man, ownerId, certificateV.Model.GetName()) data.Set("name", jsonutils.NewString(name)) - if _, err := man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data); err != nil { + + input := apis.VirtualResourceCreateInput{} + err = data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal VirtualResourceCreateInput fail %s", err) + } + input, err = man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { return nil, err } + data.Update(jsonutils.Marshal(input)) return data, nil } diff --git a/pkg/compute/models/loadbalancercertificates.go b/pkg/compute/models/loadbalancercertificates.go index 959e907fed..6ab000a78b 100644 --- a/pkg/compute/models/loadbalancercertificates.go +++ b/pkg/compute/models/loadbalancercertificates.go @@ -29,6 +29,7 @@ import ( "yunion.io/x/log" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/validators" @@ -265,9 +266,17 @@ func (man *SLoadbalancerCertificateManager) ValidateCreateData(ctx context.Conte if err != nil { return nil, err } - if _, err := man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data); err != nil { + + input := apis.VirtualResourceCreateInput{} + err = data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal VirtualResourceCreateInput fail %s", err) + } + input, err = man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { return nil, err } + data.Update(jsonutils.Marshal(input)) data.Remove("cloudregion_id") data.Remove("manager_id") diff --git a/pkg/compute/models/loadbalancerclusters.go b/pkg/compute/models/loadbalancerclusters.go index a764679e76..bb191b5f56 100644 --- a/pkg/compute/models/loadbalancerclusters.go +++ b/pkg/compute/models/loadbalancerclusters.go @@ -17,12 +17,12 @@ package models import ( "context" - "github.com/pkg/errors" - "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/validators" "yunion.io/x/onecloud/pkg/httperrors" @@ -112,7 +112,18 @@ func (man *SLoadbalancerClusterManager) ValidateCreateData(ctx context.Context, wire.ZoneId, zone.Name, zone.Id) } } - return man.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + + input := apis.StandaloneResourceCreateInput{} + err := data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal StandaloneResourceCreateInput fail %s", err) + } + input, err = man.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { + return nil, err + } + data.Update(jsonutils.Marshal(input)) + return data, nil } func (lbc *SLoadbalancerCluster) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { @@ -177,19 +188,19 @@ func (lbc *SLoadbalancerCluster) CustomizeDelete(ctx context.Context, userCred m lbagents := []SLoadbalancerAgent{} q := LoadbalancerAgentManager.Query().Equals("cluster_id", lbc.Id) if err := db.FetchModelObjects(LoadbalancerAgentManager, q, &lbagents); err != nil { - return errors.WithMessagef(err, "lbcluster %s(%s): find lbagents", lbc.Name, lbc.Id) + return errors.Wrapf(err, "lbcluster %s(%s): find lbagents", lbc.Name, lbc.Id) } for i := range lbagents { lbagent := &lbagents[i] if err := lbagent.ValidateDeleteCondition(ctx); err != nil { - return errors.WithMessagef(err, "lbagent %s(%s): validate delete", lbagent.Name, lbagent.Id) + return errors.Wrapf(err, "lbagent %s(%s): validate delete", lbagent.Name, lbagent.Id) } if err := lbagent.CustomizeDelete(ctx, userCred, query, data); err != nil { - return errors.WithMessagef(err, "lbagent %s(%s): customize delete", lbagent.Name, lbagent.Id) + return errors.Wrapf(err, "lbagent %s(%s): customize delete", lbagent.Name, lbagent.Id) } lbagent.PreDelete(ctx, userCred) if err := lbagent.Delete(ctx, userCred); err != nil { - return errors.WithMessagef(err, "lbagent %s(%s): delete", lbagent.Name, lbagent.Id) + return errors.Wrapf(err, "lbagent %s(%s): delete", lbagent.Name, lbagent.Id) } lbagent.PostDelete(ctx, userCred) } @@ -248,7 +259,7 @@ func (man *SLoadbalancerClusterManager) InitializeData() error { IsNullOrEmpty("manager_id"). IsNullOrEmpty("cluster_id") if err := db.FetchModelObjects(LoadbalancerManager, lbQ, &lbs); err != nil { - return errors.WithMessage(err, "find lb with empty cluster_id") + return errors.Wrap(err, "find lb with empty cluster_id") } // create 1 cluster for each zone @@ -267,13 +278,13 @@ func (man *SLoadbalancerClusterManager) InitializeData() error { if len(lbcs) == 0 { m, err := db.NewModelObject(man) if err != nil { - return errors.WithMessage(err, "new model object") + return errors.Wrap(err, "new model object") } lbc = m.(*SLoadbalancerCluster) lbc.Name = "auto-lbc-" + zoneId lbc.ZoneId = zoneId if err := man.TableSpec().Insert(lbc); err != nil { - return errors.WithMessage(err, "insert lbcluster model") + return errors.Wrap(err, "insert lbcluster model") } } else { if len(lbcs) > 1 { @@ -287,7 +298,7 @@ func (man *SLoadbalancerClusterManager) InitializeData() error { lb.ClusterId = lbc.Id return nil }); err != nil { - return errors.WithMessagef(err, "lb %s(%s): assign cluster: %s(%s)", lb.Name, lb.Name, lbc.Name, lbc.Id) + return errors.Wrapf(err, "lb %s(%s): assign cluster: %s(%s)", lb.Name, lb.Name, lbc.Name, lbc.Id) } } @@ -301,7 +312,7 @@ func (man *SLoadbalancerClusterManager) InitializeData() error { q := LoadbalancerAgentManager.Query(). IsNullOrEmpty("cluster_id") if err := db.FetchModelObjects(LoadbalancerAgentManager, q, &lbagents); err != nil { - return errors.WithMessage(err, "find lbagents with empty cluster_id") + return errors.Wrap(err, "find lbagents with empty cluster_id") } for i := range lbagents { lbagent := &lbagents[i] @@ -309,7 +320,7 @@ func (man *SLoadbalancerClusterManager) InitializeData() error { lbagent.ClusterId = lbc.Id return nil }); err != nil { - return errors.WithMessagef(err, "lbagent %s(%s): assign cluster: %s(%s)", + return errors.Wrapf(err, "lbagent %s(%s): assign cluster: %s(%s)", lbagent.Name, lbagent.Id, lbc.Name, lbc.Id) } } diff --git a/pkg/compute/models/loadbalancerlistenerrules.go b/pkg/compute/models/loadbalancerlistenerrules.go index c4b80d71b2..6fadb58133 100644 --- a/pkg/compute/models/loadbalancerlistenerrules.go +++ b/pkg/compute/models/loadbalancerlistenerrules.go @@ -25,6 +25,7 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" @@ -396,9 +397,16 @@ func (man *SLoadbalancerListenerRuleManager) ListItemFilter(ctx context.Context, } func (man *SLoadbalancerListenerRuleManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - if _, err := man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data); err != nil { + input := apis.VirtualResourceCreateInput{} + err := data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal StandaloneResourceCreateInput fail %s", err) + } + input, err = man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { return nil, err } + data.Update(jsonutils.Marshal(input)) listenerV := validators.NewModelIdOrNameValidator("listener", "loadbalancerlistener", ownerId) backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", ownerId) diff --git a/pkg/compute/models/loadbalancerlisteners.go b/pkg/compute/models/loadbalancerlisteners.go index e1f7dbe301..81a56e38dc 100644 --- a/pkg/compute/models/loadbalancerlisteners.go +++ b/pkg/compute/models/loadbalancerlisteners.go @@ -26,6 +26,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" @@ -225,9 +226,16 @@ func (man *SLoadbalancerListenerManager) ValidateCreateData(ctx context.Context, return nil, err } - if _, err := man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data); err != nil { + input := apis.VirtualResourceCreateInput{} + err := data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal VirtualResourceCreateInput fail %s", err) + } + input, err = man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { return nil, err } + data.Update(jsonutils.Marshal(input)) lb := lbV.Model.(*SLoadbalancer) region := lb.GetRegion() diff --git a/pkg/compute/models/loadbalancernetworks.go b/pkg/compute/models/loadbalancernetworks.go index eb3d90c195..3a7cce0925 100644 --- a/pkg/compute/models/loadbalancernetworks.go +++ b/pkg/compute/models/loadbalancernetworks.go @@ -218,10 +218,18 @@ func (ln *SLoadbalancerNetwork) GetCustomizeColumns(ctx context.Context, userCre return db.JointModelExtra(ln, extra) } -func totalLBNicCount(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) (int, error) { +func totalLBNicCount( + scope rbacutils.TRbacScope, + ownerId mcclient.IIdentityProvider, + rangeObjs []db.IStandaloneModel, + providers []string, + brands []string, + cloudEnv string, +) (int, error) { lbs := LoadbalancerManager.Query().SubQuery() lbnics := LoadbalancernetworkManager.Query().SubQuery() - q := lbnics.Query().Join(lbs, sqlchemy.Equals(lbs.Field("id"), lbnics.Field("loadbalancer_id"))) + q := lbnics.Query() + q = q.Join(lbs, sqlchemy.Equals(lbs.Field("id"), lbnics.Field("loadbalancer_id"))) switch scope { case rbacutils.ScopeSystem: // do nothing @@ -230,5 +238,7 @@ func totalLBNicCount(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvi case rbacutils.ScopeProject: q = q.Filter(sqlchemy.Equals(lbs.Field("tenant_id"), ownerId.GetProjectId())) } + q = rangeObjectsFilter(q, rangeObjs, nil, lbs.Field("zone_id"), lbs.Field("manager_id")) + q = CloudProviderFilter(q, lbs.Field("manager_id"), providers, brands, cloudEnv) return q.CountWithError() } diff --git a/pkg/compute/models/loadbalancers.go b/pkg/compute/models/loadbalancers.go index ffd3beff70..998a869477 100644 --- a/pkg/compute/models/loadbalancers.go +++ b/pkg/compute/models/loadbalancers.go @@ -26,6 +26,7 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" @@ -34,6 +35,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) type SLoadbalancerManager struct { @@ -144,9 +146,16 @@ func (man *SLoadbalancerManager) ValidateCreateData(ctx context.Context, userCre return nil, httperrors.NewBadRequestError("cannot find region info") } - if _, err := man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data); err != nil { + input := apis.VirtualResourceCreateInput{} + err := data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal VirtualResourceCreateInput fail %s", err) + } + input, err = man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { return nil, err } + data.Update(jsonutils.Marshal(input)) ctx = context.WithValue(ctx, "ownerId", ownerId) return region.GetDriver().ValidateCreateLoadbalancerData(ctx, userCred, data) @@ -861,3 +870,16 @@ func (man *SLoadbalancerManager) getLoadbalancer(lbId string) (*SLoadbalancer, e } return lb, nil } + +func (man *SLoadbalancerManager) TotalCount( + scope rbacutils.TRbacScope, + ownerId mcclient.IIdentityProvider, + rangeObjs []db.IStandaloneModel, + providers []string, brands []string, cloudEnv string, +) (int, error) { + q := man.Query() + q = scopeOwnerIdFilter(q, scope, ownerId) + q = CloudProviderFilter(q, q.Field("manager_id"), providers, brands, cloudEnv) + q = rangeObjectsFilter(q, rangeObjs, nil, q.Field("zone_id"), q.Field("manager_id")) + return q.CountWithError() +} \ No newline at end of file diff --git a/pkg/compute/models/networks.go b/pkg/compute/models/networks.go index 97701b0365..c8260cb16d 100644 --- a/pkg/compute/models/networks.go +++ b/pkg/compute/models/networks.go @@ -737,7 +737,7 @@ func (manager *SNetworkManager) GetOnPremiseNetworkOfIP(ipAddr string, serverTyp return nil, sql.ErrNoRows } -func (manager *SNetworkManager) allNetworksQ(providers []string, brands []string, cloudEnv string, rangeObj db.IStandaloneModel) *sqlchemy.SQuery { +func (manager *SNetworkManager) allNetworksQ(providers []string, brands []string, cloudEnv string, rangeObjs []db.IStandaloneModel) *sqlchemy.SQuery { networks := manager.Query().SubQuery() hostwires := HostwireManager.Query().SubQuery() hosts := HostManager.Query().SubQuery() @@ -748,11 +748,18 @@ func (manager *SNetworkManager) allNetworksQ(providers []string, brands []string q = q.Filter(sqlchemy.OR( sqlchemy.Equals(hosts.Field("host_type"), api.HOST_TYPE_BAREMETAL), sqlchemy.Equals(hosts.Field("host_status"), api.HOST_ONLINE))) - return AttachUsageQuery(q, hosts, nil, nil, providers, brands, cloudEnv, rangeObj) + return AttachUsageQuery(q, hosts, nil, nil, providers, brands, cloudEnv, rangeObjs) } -func (manager *SNetworkManager) totalPortCountQ(scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, providers []string, brands []string, cloudEnv string, rangeObj db.IStandaloneModel) *sqlchemy.SQuery { - q := manager.allNetworksQ(providers, brands, cloudEnv, rangeObj) +func (manager *SNetworkManager) totalPortCountQ( + scope rbacutils.TRbacScope, + userCred mcclient.IIdentityProvider, + providers []string, + brands []string, + cloudEnv string, + rangeObjs []db.IStandaloneModel, +) *sqlchemy.SQuery { + q := manager.allNetworksQ(providers, brands, cloudEnv, rangeObjs) switch scope { case rbacutils.ScopeSystem: case rbacutils.ScopeDomain: @@ -772,10 +779,15 @@ func (manager *SNetworkManager) TotalPortCount( scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, providers []string, brands []string, cloudEnv string, - rangeObj db.IStandaloneModel, + rangeObjs []db.IStandaloneModel, ) NetworkPortStat { nets := make([]SNetwork, 0) - err := manager.totalPortCountQ(scope, userCred, providers, brands, cloudEnv, rangeObj).All(&nets) + err := manager.totalPortCountQ( + scope, + userCred, + providers, brands, cloudEnv, + rangeObjs, + ).All(&nets) if err != nil { log.Errorf("TotalPortCount: %v", err) } @@ -1169,13 +1181,13 @@ func (manager *SNetworkManager) newIfnameHint(hint string) (string, error) { return r, nil } -func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.NetworkCreateInput) (*jsonutils.JSONDict, error) { +func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.NetworkCreateInput) (api.NetworkCreateInput, error) { var err error var startIp, endIp netutils.IPV4Addr if len(input.GuestIpPrefix) > 0 { prefix, err := netutils.NewIPV4Prefix(input.GuestIpPrefix) if err != nil { - return nil, httperrors.NewInputParameterError("ip_prefix error: %s", err) + return input, httperrors.NewInputParameterError("ip_prefix error: %s", err) } iprange := prefix.ToIPRange() startIp = iprange.StartIp().StepUp() @@ -1184,11 +1196,11 @@ func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred } else { startIp, err = netutils.NewIPV4Addr(input.GuestIpStart) if err != nil { - return nil, httperrors.NewInputParameterError("Invalid start ip: %s %s", input.GuestIpStart, err) + return input, httperrors.NewInputParameterError("Invalid start ip: %s %s", input.GuestIpStart, err) } endIp, err = netutils.NewIPV4Addr(input.GuestIpEnd) if err != nil { - return nil, httperrors.NewInputParameterError("invalid end ip: %s %s", input.GuestIpEnd, err) + return input, httperrors.NewInputParameterError("invalid end ip: %s %s", input.GuestIpEnd, err) } if startIp > endIp { tmp := startIp @@ -1200,7 +1212,7 @@ func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred input.GuestIpEnd = endIp.String() if !isValidMaskLen(input.GuestIpMask) { - return nil, httperrors.NewInputParameterError("Invalid masklen %d", input.GuestIpMask) + return input, httperrors.NewInputParameterError("Invalid masklen %d", input.GuestIpMask) } { @@ -1209,7 +1221,7 @@ func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred } input.IfnameHint, err = manager.newIfnameHint(input.IfnameHint) if err != nil { - return nil, httperrors.NewBadRequestError("cannot derive valid ifname hint: %v", err) + return input, httperrors.NewBadRequestError("cannot derive valid ifname hint: %v", err) } } @@ -1219,22 +1231,22 @@ func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred ipList := strings.Split(ipStr, ",") for _, ipstr := range ipList { if !regutils.MatchIPAddr(ipstr) { - return nil, httperrors.NewInputParameterError("%s: Invalid IP address %s", key, ipstr) + return input, httperrors.NewInputParameterError("%s: Invalid IP address %s", key, ipstr) } } } else if !regutils.MatchIPAddr(ipStr) { - return nil, httperrors.NewInputParameterError("%s: Invalid IP address %s", key, ipStr) + return input, httperrors.NewInputParameterError("%s: Invalid IP address %s", key, ipStr) } } } nets := manager.getAllNetworks("") if nets == nil { - return nil, httperrors.NewInternalServerError("query all networks fail") + return input, httperrors.NewInternalServerError("query all networks fail") } if isOverlapNetworks(nets, startIp, endIp) { - return nil, httperrors.NewInputParameterError("Conflict address space with existing networks") + return input, httperrors.NewInputParameterError("Conflict address space with existing networks") } if len(input.WireId) > 0 { @@ -1245,9 +1257,9 @@ func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred wireObj, err := WireManager.FetchByIdOrName(userCred, input.Wire) if err != nil { if err == sql.ErrNoRows { - return nil, httperrors.NewNotFoundError("wire %s not found", input.Wire) + return input, httperrors.NewNotFoundError("wire %s not found", input.Wire) } else { - return nil, httperrors.NewInternalServerError("query wire %s error %s", input.Wire, err) + return input, httperrors.NewInternalServerError("query wire %s error %s", input.Wire, err) } } input.WireId = wireObj.GetId() @@ -1257,24 +1269,24 @@ func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred zoneObj, err := ZoneManager.FetchByIdOrName(userCred, input.Zone) if err != nil { if err == sql.ErrNoRows { - return nil, httperrors.NewNotFoundError("zone %s not found", input.Zone) + return input, httperrors.NewNotFoundError("zone %s not found", input.Zone) } else { - return nil, httperrors.NewInternalServerError("query zone %s error %s", input.Zone, err) + return input, httperrors.NewInternalServerError("query zone %s error %s", input.Zone, err) } } vpcObj, err := VpcManager.FetchByIdOrName(userCred, input.Vpc) if err != nil { if err == sql.ErrNoRows { - return nil, httperrors.NewNotFoundError("vpc %s not found", input.Vpc) + return input, httperrors.NewNotFoundError("vpc %s not found", input.Vpc) } else { - return nil, httperrors.NewInternalServerError("query vpc %s error %s", input.Vpc, err) + return input, httperrors.NewInternalServerError("query vpc %s error %s", input.Vpc, err) } } vpc := vpcObj.(*SVpc) zone := zoneObj.(*SZone) region := zone.GetRegion() if region == nil { - return nil, httperrors.NewInternalServerError("zone %s related region not found", zone.Id) + return input, httperrors.NewInternalServerError("zone %s related region not found", zone.Id) } // 华为云,ucloud wire zone_id 为空 @@ -1286,37 +1298,37 @@ func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred } if err != nil { - return nil, httperrors.NewInternalServerError("query wire for zone %s and vpc %s: %v", input.Zone, input.Vpc, err) + return input, httperrors.NewInternalServerError("query wire for zone %s and vpc %s: %v", input.Zone, input.Vpc, err) } if len(wires) == 0 { - return nil, httperrors.NewNotFoundError("wire not found for zone %s and vpc %s", input.Zone, input.Vpc) + return input, httperrors.NewNotFoundError("wire not found for zone %s and vpc %s", input.Zone, input.Vpc) } else if len(wires) > 1 { - return nil, httperrors.NewConflictError("found %d wires for zone %s and vpc %s", len(wires), input.Zone, input.Vpc) + return input, httperrors.NewConflictError("found %d wires for zone %s and vpc %s", len(wires), input.Zone, input.Vpc) } else { input.WireId = wires[0].Id } } else { - return nil, httperrors.NewInputParameterError("No either wire or vpc provided") + return input, httperrors.NewInputParameterError("No either wire or vpc provided") } } else { - return nil, httperrors.NewInvalidStatusError("No either wire or zone provided") + return input, httperrors.NewInvalidStatusError("No either wire or zone provided") } } if len(input.WireId) == 0 { - return nil, httperrors.NewMissingParameterError("wire_id") + return input, httperrors.NewMissingParameterError("wire_id") } wire := WireManager.FetchWireById(input.WireId) if wire == nil { - return nil, httperrors.NewResourceNotFoundError("wire %s not found", input.WireId) + return input, httperrors.NewResourceNotFoundError("wire %s not found", input.WireId) } vpc := wire.getVpc() if vpc == nil { - return nil, httperrors.NewInputParameterError("no valid vpc ???") + return input, httperrors.NewInputParameterError("no valid vpc ???") } if vpc.Status != api.VPC_STATUS_AVAILABLE { - return nil, httperrors.NewInvalidStatusError("VPC not ready") + return input, httperrors.NewInvalidStatusError("VPC not ready") } vpcRanges := vpc.getIPRanges() @@ -1332,16 +1344,20 @@ func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred } if !inRange { - return nil, httperrors.NewInputParameterError("Network not in range of VPC cidrblock %s", vpc.CidrBlock) + return input, httperrors.NewInputParameterError("Network not in range of VPC cidrblock %s", vpc.CidrBlock) } if len(input.ServerType) == 0 { input.ServerType = api.NETWORK_TYPE_GUEST } else if !utils.IsInStringArray(input.ServerType, ALL_NETWORK_TYPES) { - return nil, httperrors.NewInputParameterError("Invalid server_type: %s", input.ServerType) + return input, httperrors.NewInputParameterError("Invalid server_type: %s", input.ServerType) } - return manager.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.JSON(input)) + input.SharableVirtualResourceCreateInput, err = manager.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.SharableVirtualResourceCreateInput) + if err != nil { + return input, err + } + return input, nil } func (self *SNetwork) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { diff --git a/pkg/compute/models/projectquota.go b/pkg/compute/models/projectquota.go new file mode 100644 index 0000000000..4d627117ef --- /dev/null +++ b/pkg/compute/models/projectquota.go @@ -0,0 +1,157 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + + "yunion.io/x/jsonutils" + + identityapi "yunion.io/x/onecloud/pkg/apis/identity" + "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" + commonOptions "yunion.io/x/onecloud/pkg/cloudcommon/options" + "yunion.io/x/onecloud/pkg/compute/options" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/util/rbacutils" +) + +var ( + ProjectQuota SProjectQuota + ProjectQuotaManager *SQuotaManager + ProjectUsageManager *SQuotaManager + ProjectPendingUsageManager *SQuotaManager +) + +func init() { + ProjectQuota = SProjectQuota{} + + ProjectUsageManager = &SQuotaManager{ + SQuotaBaseManager: quotas.NewQuotaUsageManager(RegionQuota, "project_quota_usage_tbl"), + } + ProjectUsageManager.SetVirtualObject(ProjectUsageManager) + ProjectPendingUsageManager = &SQuotaManager{ + SQuotaBaseManager: quotas.NewQuotaUsageManager(RegionQuota, "project_quota_pending_usage_tbl"), + } + ProjectPendingUsageManager.SetVirtualObject(ProjectPendingUsageManager) + ProjectQuotaManager = &SQuotaManager{ + SQuotaBaseManager: quotas.NewQuotaBaseManager(RegionQuota, + "project_quota_tbl", + ProjectPendingUsageManager, + ProjectUsageManager, + "project_quota", + "project_quotas", + ), + } + ProjectQuotaManager.SetVirtualObject(ProjectQuotaManager) +} + +type SProjectQuota struct { + quotas.SQuotaBase + + quotas.SBaseQuotaKeys + + Secgroup int +} + +func (self *SProjectQuota) GetKeys() quotas.IQuotaKeys { + return self.SBaseQuotaKeys +} + +func (self *SProjectQuota) SetKeys(keys quotas.IQuotaKeys) { + self.SBaseQuotaKeys = keys.(quotas.SBaseQuotaKeys) +} + +func (self *SProjectQuota) FetchSystemQuota() { + keys := self.SBaseQuotaKeys + base := 0 + switch options.Options.DefaultQuotaValue { + case commonOptions.DefaultQuotaUnlimit: + base = -1 + case commonOptions.DefaultQuotaZero: + base = 0 + if keys.Scope() == rbacutils.ScopeDomain { // domain level quota + base = 10 + } else if keys.DomainId == identityapi.DEFAULT_DOMAIN_ID && keys.ProjectId == auth.AdminCredential().GetProjectId() { + base = 1 + } + case commonOptions.DefaultQuotaDefault: + base = 1 + if keys.Scope() == rbacutils.ScopeDomain { + base = 10 + } + } + defaultValue := func(def int) int { + if base < 0 { + return -1 + } else { + return def * base + } + } + self.Secgroup = defaultValue(options.Options.DefaultSecgroupQuota) +} + +func (self *SProjectQuota) FetchUsage(ctx context.Context) error { + regionKeys := self.SBaseQuotaKeys + + scope := regionKeys.Scope() + ownerId := regionKeys.OwnerId() + + self.Secgroup, _ = totalSecurityGroupCount(scope, ownerId) + return nil +} + +func (self *SProjectQuota) IsEmpty() bool { + if self.Secgroup > 0 { + return false + } + return true +} + +func (self *SProjectQuota) Add(quota quotas.IQuota) { + squota := quota.(*SProjectQuota) + self.Secgroup = self.Secgroup + quotas.NonNegative(squota.Secgroup) +} + +func (self *SProjectQuota) Sub(quota quotas.IQuota) { + squota := quota.(*SProjectQuota) + self.Secgroup = nonNegative(self.Secgroup - squota.Secgroup) +} + +func (self *SProjectQuota) Update(quota quotas.IQuota) { + squota := quota.(*SProjectQuota) + if squota.Secgroup > 0 { + self.Secgroup = squota.Secgroup + } +} + +func (self *SProjectQuota) Exceed(request quotas.IQuota, quota quotas.IQuota) error { + err := quotas.NewOutOfQuotaError() + sreq := request.(*SProjectQuota) + squota := quota.(*SProjectQuota) + if sreq.Secgroup > 0 && self.Secgroup > squota.Secgroup { + err.Add("secgroup", squota.Secgroup, self.Secgroup) + } + if err.IsError() { + return err + } else { + return nil + } +} + +func (self *SProjectQuota) ToJSON(prefix string) jsonutils.JSONObject { + ret := jsonutils.NewDict() + ret.Add(jsonutils.NewInt(int64(self.Secgroup)), keyName(prefix, "secgroup")) + return ret +} diff --git a/pkg/compute/models/quotametrics.go b/pkg/compute/models/quotametrics.go deleted file mode 100644 index fdf6879dfc..0000000000 --- a/pkg/compute/models/quotametrics.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package models - -import ( - api "yunion.io/x/onecloud/pkg/apis/compute" -) - -func (account *SCloudaccount) GetQuotaPlatformID() []string { - return []string{ - account.getCloudEnv(), - account.Provider, - } -} - -func (provider *SCloudprovider) GetQuotaPlatformID() []string { - return provider.GetCloudaccount().GetQuotaPlatformID() -} - -func (base *SManagedResourceBase) GetQuotaPlatformID() []string { - account := base.GetCloudaccount() - if account != nil { - return account.GetQuotaPlatformID() - } else { - return []string{ - api.CLOUD_ENV_ON_PREMISE, api.CLOUD_PROVIDER_ONECLOUD, - } - } -} - -func (host *SHost) GetQuotaPlatformID() []string { - ids := host.SManagedResourceBase.GetQuotaPlatformID() - switch host.HostType { - case api.HOST_TYPE_HYPERVISOR: - ids = append(ids, api.HYPERVISOR_KVM) - case api.HOST_TYPE_BAREMETAL: - ids = append(ids, api.HYPERVISOR_BAREMETAL) - case api.HOST_TYPE_KUBELET: - ids = append(ids, api.HYPERVISOR_CONTAINER) - } - return ids -} - -func (storage *SStorage) getQuotaPlatformID() []string { - hosts := storage.GetAttachedHosts() - if len(hosts) > 0 { - return hosts[0].GetQuotaPlatformID() - } - return storage.SManagedResourceBase.GetQuotaPlatformID() -} - -func (guest *SGuest) GetQuotaPlatformID() []string { - host := guest.GetHost() - if host != nil { - return host.GetQuotaPlatformID() - } - return GetDriver(guest.GetHypervisor()).GetQuotaPlatformID() -} - -func (disk *SDisk) GetQuotaPlatformID() []string { - storage := disk.GetStorage() - if storage != nil { - return storage.getQuotaPlatformID() - } - return []string{} -} - -func GetQuotaPlatformID(hypervisor string) []string { - if len(hypervisor) > 0 { - return GetDriver(hypervisor).GetQuotaPlatformID() - } - return []string{} -} diff --git a/pkg/compute/models/quotas.go b/pkg/compute/models/quotas.go index d6deccbd59..67f00390d7 100644 --- a/pkg/compute/models/quotas.go +++ b/pkg/compute/models/quotas.go @@ -16,17 +16,22 @@ package models import ( "context" + "database/sql" "fmt" "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" "yunion.io/x/pkg/tristate" - "yunion.io/x/pkg/util/sets" api "yunion.io/x/onecloud/pkg/apis/compute" identityapi "yunion.io/x/onecloud/pkg/apis/identity" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" + commonOptions "yunion.io/x/onecloud/pkg/cloudcommon/options" "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/utils" "yunion.io/x/onecloud/pkg/util/rbacutils" ) @@ -34,103 +39,149 @@ type SQuotaManager struct { quotas.SQuotaBaseManager } -var QuotaManager *SQuotaManager -var QuotaUsageManager *SQuotaManager +var ( + Quota SQuota + QuotaManager *SQuotaManager + QuotaUsageManager *SQuotaManager + QuotaPendingUsageManager *SQuotaManager +) func init() { - pendingStore := quotas.NewMemoryQuotaStore() + Quota = SQuota{} QuotaUsageManager = &SQuotaManager{ - SQuotaBaseManager: quotas.NewQuotaUsageManager(SQuota{}, "quota_usage_tbl"), + SQuotaBaseManager: quotas.NewQuotaUsageManager(Quota, "quota_usage_tbl"), } - + QuotaUsageManager.SetVirtualObject(QuotaUsageManager) + QuotaPendingUsageManager = &SQuotaManager{ + SQuotaBaseManager: quotas.NewQuotaUsageManager(Quota, "quota_pending_usage_tbl"), + } + QuotaPendingUsageManager.SetVirtualObject(QuotaPendingUsageManager) QuotaManager = &SQuotaManager{ - SQuotaBaseManager: quotas.NewQuotaBaseManager(SQuota{}, "quota_tbl", pendingStore, QuotaUsageManager), + SQuotaBaseManager: quotas.NewQuotaBaseManager(Quota, + "quota_tbl", + QuotaPendingUsageManager, + QuotaUsageManager, + "quota", + "quotas", + ), } + QuotaManager.SetVirtualObject(QuotaManager) } type SQuota struct { quotas.SQuotaBase + SComputeResourceKeys + + Count int Cpu int Memory int Storage int - Port int - Eip int - Eport int - Bw int - Ebw int - // Keypair int - // Image int + Group int - Secgroup int IsolatedDevice int - Snapshot int - - Bucket int - ObjectGB int - ObjectCnt int } -func (self *SQuota) FetchSystemQuota(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) { +func (self *SQuota) GetKeys() quotas.IQuotaKeys { + return self.SComputeResourceKeys +} + +func (self *SQuota) SetKeys(keys quotas.IQuotaKeys) { + self.SComputeResourceKeys = keys.(SComputeResourceKeys) +} + +func (self *SQuota) FetchSystemQuota() { + keys := self.SComputeResourceKeys base := 0 - if scope == rbacutils.ScopeDomain { - base = 10 - } else if ownerId.GetProjectDomainId() == identityapi.DEFAULT_DOMAIN_ID && ownerId.GetProjectName() == identityapi.SystemAdminProject { + switch options.Options.DefaultQuotaValue { + case commonOptions.DefaultQuotaUnlimit: + base = -1 + case commonOptions.DefaultQuotaZero: + base = 0 + if keys.Scope() == rbacutils.ScopeDomain { // domain level quota + base = 10 + } else if keys.DomainId == identityapi.DEFAULT_DOMAIN_ID && keys.ProjectId == auth.AdminCredential().GetProjectId() { + base = 1 + } + case commonOptions.DefaultQuotaDefault: base = 1 + if keys.Scope() == rbacutils.ScopeDomain { + base = 10 + } } - self.Cpu = options.Options.DefaultCpuQuota * base - self.Memory = options.Options.DefaultMemoryQuota * base - self.Storage = options.Options.DefaultStorageQuota * base - self.Port = options.Options.DefaultPortQuota * base - self.Eip = options.Options.DefaultEipQuota * base - self.Eport = options.Options.DefaultEportQuota * base - self.Bw = options.Options.DefaultBwQuota * base - self.Ebw = options.Options.DefaultEbwQuota * base - self.Group = options.Options.DefaultGroupQuota * base - self.Secgroup = options.Options.DefaultSecgroupQuota * base - self.IsolatedDevice = options.Options.DefaultIsolatedDeviceQuota * base - self.Snapshot = options.Options.DefaultSnapshotQuota * base - self.Bucket = options.Options.DefaultBucketQuota * base - self.ObjectGB = options.Options.DefaultObjectGBQuota * base - self.ObjectCnt = options.Options.DefaultObjectCntQuota * base + defaultValue := func(def int) int { + if base < 0 { + return -1 + } else { + return def * base + } + } + self.Count = defaultValue(options.Options.DefaultServerQuota) + self.Cpu = defaultValue(options.Options.DefaultCpuQuota) + self.Memory = defaultValue(options.Options.DefaultMemoryQuota) + self.Storage = defaultValue(options.Options.DefaultStorageQuota) + self.Group = defaultValue(options.Options.DefaultGroupQuota) + self.IsolatedDevice = defaultValue(options.Options.DefaultIsolatedDeviceQuota) } -func (self *SQuota) FetchUsage(ctx context.Context, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, name []string) error { - diskSize := totalDiskSize(scope, ownerId, tristate.None, tristate.None, false, false) - net := totalGuestNicCount(scope, ownerId, nil, false) - lbnic, _ := totalLBNicCount(scope, ownerId) - // net := WireManager.TotalCount(nil, nil, nil, "", scope, ownerId) - hypervisors := sets.NewString(api.HYPERVISORS...) - hypervisors.Delete(api.HYPERVISOR_CONTAINER) - guest := totalGuestResourceCount(scope, ownerId, nil, nil, hypervisors.List(), false, false, nil, nil, nil, nil, "") - eipUsage := ElasticipManager.TotalCount(scope, ownerId, nil, nil, nil, "") - snapshotCount, _ := TotalSnapshotCount(scope, ownerId, nil, nil, nil, "") - bucketUsage := BucketManager.TotalCount(scope, ownerId, nil, nil, nil, "") - // XXX - // keypair belongs to user - // keypair := totalKeypairCount(projectId) +func (self *SQuota) FetchUsage(ctx context.Context) error { + keys := self.SComputeResourceKeys + scope := keys.Scope() + ownerId := keys.OwnerId() + + rangeObjs := make([]db.IStandaloneModel, 0) + if len(keys.ManagerId) > 0 { + obj, err := CloudproviderManager.FetchById(keys.ManagerId) + if err != nil { + return errors.Wrap(err, "CloudproviderManager.FetchById") + } + rangeObjs = append(rangeObjs, obj.(db.IStandaloneModel)) + } else if len(keys.AccountId) > 0 { + obj, err := CloudaccountManager.FetchById(keys.AccountId) + if err != nil { + return errors.Wrap(err, "CloudaccountManager.FetchById") + } + rangeObjs = append(rangeObjs, obj.(db.IStandaloneModel)) + } + + if len(keys.ZoneId) > 0 { + obj, err := ZoneManager.FetchById(keys.ZoneId) + if err != nil { + return errors.Wrap(err, "ZoneManager.FetchById") + } + rangeObjs = append(rangeObjs, obj.(db.IStandaloneModel)) + } else if len(keys.RegionId) > 0 { + obj, err := CloudregionManager.FetchById(keys.RegionId) + if err != nil { + return errors.Wrap(err, "CloudregionManager.FetchById") + } + rangeObjs = append(rangeObjs, obj.(db.IStandaloneModel)) + } + var hypervisors []string + if len(keys.Hypervisor) > 0 { + hypervisors = []string{keys.Hypervisor} + } + var providers []string + if len(keys.Provider) > 0 { + providers = []string{keys.Provider} + } + var brands []string + if len(keys.Brand) > 0 { + brands = []string{keys.Brand} + } + + diskSize := totalDiskSize(scope, ownerId, tristate.None, tristate.None, false, false, rangeObjs, providers, brands, keys.CloudEnv, hypervisors) + + guest := totalGuestResourceCount(scope, ownerId, rangeObjs, nil, hypervisors, false, false, nil, nil, providers, brands, keys.CloudEnv) + + self.Count = guest.TotalGuestCount self.Cpu = guest.TotalCpuCount self.Memory = guest.TotalMemSize self.Storage = diskSize - self.Eip = eipUsage.Total() - // log.Debugf("%d %d %d\n", net.InternalNicCount, net.InternalVirtualNicCount, lbnic) - self.Port = net.InternalNicCount + net.InternalVirtualNicCount + lbnic - self.Eport = net.ExternalNicCount + net.ExternalVirtualNicCount - self.Bw = net.InternalBandwidth - self.Ebw = net.ExternalBandwidth - // self.Port = net.GuestNicCount + net.GroupNicCount + net.LbNicCount - // if scope == rbacutils.ScopeSystem { - // self.Port += net.HostNicCount + net.ReservedCount - // } self.Group = 0 - self.Secgroup, _ = totalSecurityGroupCount(scope, ownerId) self.IsolatedDevice = guest.TotalIsolatedCount - self.Snapshot = snapshotCount - self.Bucket = bucketUsage.Buckets - self.ObjectGB = int(bucketUsage.Bytes / 1000 / 1000 / 1000) - self.ObjectCnt = bucketUsage.Objects return nil } @@ -144,62 +195,22 @@ func (self *SQuota) IsEmpty() bool { if self.Storage > 0 { return false } - if self.Port > 0 { - return false - } - if self.Eip > 0 { - return false - } - if self.Eport > 0 { - return false - } - if self.Bw > 0 { - return false - } - if self.Ebw > 0 { - return false - } if self.Group > 0 { return false } - if self.Secgroup > 0 { - return false - } if self.IsolatedDevice > 0 { return false } - if self.Snapshot > 0 { - return false - } - if self.Bucket > 0 { - return false - } - if self.ObjectGB > 0 { - return false - } - if self.ObjectCnt > 0 { - return false - } return true } func (self *SQuota) Add(quota quotas.IQuota) { squota := quota.(*SQuota) - self.Cpu = self.Cpu + squota.Cpu - self.Memory = self.Memory + squota.Memory - self.Storage = self.Storage + squota.Storage - self.Port = self.Port + squota.Port - self.Eip = self.Eip + squota.Eip - self.Eport = self.Eport + squota.Eport - self.Bw = self.Bw + squota.Bw - self.Ebw = self.Ebw + squota.Ebw - self.Group = self.Group + squota.Group - self.Secgroup = self.Secgroup + squota.Secgroup - self.IsolatedDevice = self.IsolatedDevice + squota.IsolatedDevice - self.Snapshot = self.Snapshot + squota.Snapshot - self.Bucket = self.Bucket + squota.Bucket - self.ObjectGB = self.ObjectGB + squota.ObjectGB - self.ObjectCnt = self.ObjectCnt + squota.ObjectCnt + self.Cpu = self.Cpu + quotas.NonNegative(squota.Cpu) + self.Memory = self.Memory + quotas.NonNegative(squota.Memory) + self.Storage = self.Storage + quotas.NonNegative(squota.Storage) + self.Group = self.Group + quotas.NonNegative(squota.Group) + self.IsolatedDevice = self.IsolatedDevice + quotas.NonNegative(squota.IsolatedDevice) } func nonNegative(val int) int { @@ -211,18 +222,8 @@ func (self *SQuota) Sub(quota quotas.IQuota) { self.Cpu = nonNegative(self.Cpu - squota.Cpu) self.Memory = nonNegative(self.Memory - squota.Memory) self.Storage = nonNegative(self.Storage - squota.Storage) - self.Port = nonNegative(self.Port - squota.Port) - self.Eip = nonNegative(self.Eip - squota.Eip) - self.Eport = nonNegative(self.Eport - squota.Eport) - self.Bw = nonNegative(self.Bw - squota.Bw) - self.Ebw = nonNegative(self.Ebw - squota.Ebw) self.Group = nonNegative(self.Group - squota.Group) - self.Secgroup = nonNegative(self.Secgroup - squota.Secgroup) self.IsolatedDevice = nonNegative(self.IsolatedDevice - squota.IsolatedDevice) - self.Snapshot = nonNegative(self.Snapshot - squota.Snapshot) - self.Bucket = nonNegative(self.Bucket - squota.Bucket) - self.ObjectGB = nonNegative(self.ObjectGB - squota.ObjectGB) - self.ObjectCnt = nonNegative(self.ObjectCnt - squota.ObjectCnt) } func (self *SQuota) Update(quota quotas.IQuota) { @@ -236,42 +237,12 @@ func (self *SQuota) Update(quota quotas.IQuota) { if squota.Storage > 0 { self.Storage = squota.Storage } - if squota.Port > 0 { - self.Port = squota.Port - } - if squota.Eip > 0 { - self.Eip = squota.Eip - } - if squota.Eport > 0 { - self.Eport = squota.Eport - } - if squota.Bw > 0 { - self.Bw = squota.Bw - } - if squota.Ebw > 0 { - self.Ebw = squota.Ebw - } if squota.Group > 0 { self.Group = squota.Group } - if squota.Secgroup > 0 { - self.Secgroup = squota.Secgroup - } if squota.IsolatedDevice > 0 { self.IsolatedDevice = squota.IsolatedDevice } - if squota.Snapshot > 0 { - self.Snapshot = squota.Snapshot - } - if squota.Bucket > 0 { - self.Bucket = squota.Bucket - } - if squota.ObjectGB > 0 { - self.ObjectGB = squota.ObjectGB - } - if squota.ObjectCnt > 0 { - self.ObjectCnt = squota.ObjectCnt - } } func (self *SQuota) Exceed(request quotas.IQuota, quota quotas.IQuota) error { @@ -287,42 +258,12 @@ func (self *SQuota) Exceed(request quotas.IQuota, quota quotas.IQuota) error { if sreq.Storage > 0 && self.Storage > squota.Storage { err.Add("storage", squota.Storage, self.Storage) } - if sreq.Port > 0 && self.Port > squota.Port { - err.Add("port", squota.Port, self.Port) - } - if sreq.Eip > 0 && self.Eip > squota.Eip { - err.Add("eip", squota.Eip, self.Eip) - } - if sreq.Eport > 0 && self.Eport > squota.Eport { - err.Add("eport", squota.Eport, self.Eport) - } - if sreq.Bw > 0 && self.Bw > squota.Bw { - err.Add("bw", squota.Bw, self.Bw) - } - if sreq.Ebw > 0 && self.Ebw > squota.Ebw { - err.Add("ebw", squota.Ebw, self.Ebw) - } if sreq.Group > 0 && self.Group > squota.Group { err.Add("group", squota.Group, self.Group) } - if sreq.Secgroup > 0 && self.Secgroup > squota.Secgroup { - err.Add("secgroup", squota.Secgroup, self.Secgroup) - } if sreq.IsolatedDevice > 0 && self.IsolatedDevice > squota.IsolatedDevice { err.Add("isolated_device", squota.IsolatedDevice, self.IsolatedDevice) } - if sreq.Snapshot > 0 && self.Snapshot > squota.Snapshot { - err.Add("snapshot", squota.Snapshot, self.Snapshot) - } - if sreq.Bucket > 0 && self.Bucket > squota.Bucket { - err.Add("bucket", squota.Bucket, self.Bucket) - } - if sreq.ObjectGB > 0 && self.ObjectGB > squota.ObjectGB { - err.Add("object_gb", squota.ObjectGB, self.ObjectGB) - } - if sreq.ObjectCnt > 0 && self.ObjectCnt > squota.ObjectCnt { - err.Add("object_cnt", squota.ObjectCnt, self.ObjectCnt) - } if err.IsError() { return err } else { @@ -343,17 +284,158 @@ func (self *SQuota) ToJSON(prefix string) jsonutils.JSONObject { ret.Add(jsonutils.NewInt(int64(self.Cpu)), keyName(prefix, "cpu")) ret.Add(jsonutils.NewInt(int64(self.Memory)), keyName(prefix, "memory")) ret.Add(jsonutils.NewInt(int64(self.Storage)), keyName(prefix, "storage")) - ret.Add(jsonutils.NewInt(int64(self.Port)), keyName(prefix, "port")) - ret.Add(jsonutils.NewInt(int64(self.Eip)), keyName(prefix, "eip")) - ret.Add(jsonutils.NewInt(int64(self.Eport)), keyName(prefix, "eport")) - ret.Add(jsonutils.NewInt(int64(self.Bw)), keyName(prefix, "bw")) - ret.Add(jsonutils.NewInt(int64(self.Ebw)), keyName(prefix, "ebw")) ret.Add(jsonutils.NewInt(int64(self.Group)), keyName(prefix, "group")) - ret.Add(jsonutils.NewInt(int64(self.Secgroup)), keyName(prefix, "secgroup")) ret.Add(jsonutils.NewInt(int64(self.IsolatedDevice)), keyName(prefix, "isolated_device")) - ret.Add(jsonutils.NewInt(int64(self.Snapshot)), keyName(prefix, "snapshot")) - ret.Add(jsonutils.NewInt(int64(self.Bucket)), keyName(prefix, "bucket")) - ret.Add(jsonutils.NewInt(int64(self.ObjectGB)), keyName(prefix, "object_gb")) - ret.Add(jsonutils.NewInt(int64(self.ObjectCnt)), keyName(prefix, "object_cnt")) return ret } + +func (manager *SQuotaManager) FetchIdMap(ctx context.Context, idMap map[string]map[string]string) (map[string]map[string]string, error) { + for field := range idMap { + switch field { + case "domain_id": + fieldIdMap, err := utils.FetchDomainNames(ctx, idMap[field]) + if err != nil { + return nil, errors.Wrap(err, "utils.FetchDomainNames") + } + idMap[field] = fieldIdMap + case "tenant_id": + fieldIdMap, err := utils.FetchTenantNames(ctx, idMap[field]) + if err != nil { + return nil, errors.Wrap(err, "utils.FetchTenantNames") + } + idMap[field] = fieldIdMap + case "region_id": + fieldIdMap, err := fetchRegionNames(idMap[field]) + if err != nil { + return nil, errors.Wrap(err, "fetchRegionNames") + } + idMap[field] = fieldIdMap + case "zone_id": + fieldIdMap, err := fetchZoneNames(idMap[field]) + if err != nil { + return nil, errors.Wrap(err, "fetchZoneNames") + } + idMap[field] = fieldIdMap + case "account_id": + fieldIdMap, err := fetchAccountNames(idMap[field]) + if err != nil { + return nil, errors.Wrap(err, "fetchAccountNames") + } + idMap[field] = fieldIdMap + case "manager_id": + fieldIdMap, err := fetchManagerNames(idMap[field]) + if err != nil { + return nil, errors.Wrap(err, "fetchManagerNames") + } + idMap[field] = fieldIdMap + } + } + return idMap, nil +} + +func dbFetchIdNameMap(manager db.IStandaloneModelManager, idMap map[string]string) (map[string]string, error) { + q := manager.Query("id", "name").In("id", utils.MapKeys(idMap)) + rows, err := q.Rows() + if err != nil { + if errors.Cause(err) == sql.ErrNoRows { + return idMap, nil + } else { + return idMap, errors.Wrap(err, "Query") + } + } + defer rows.Close() + for rows.Next() { + var id string + var name string + err := rows.Scan(&id, &name) + if err != nil { + return idMap, errors.Wrap(err, "rows.Scan") + } + idMap[id] = name + } + return idMap, nil +} + +func fetchRegionNames(idMap map[string]string) (map[string]string, error) { + return dbFetchIdNameMap(CloudregionManager, idMap) +} + +func fetchZoneNames(idMap map[string]string) (map[string]string, error) { + return dbFetchIdNameMap(ZoneManager, idMap) +} + +func fetchAccountNames(idMap map[string]string) (map[string]string, error) { + return dbFetchIdNameMap(CloudaccountManager, idMap) +} + +func fetchManagerNames(idMap map[string]string) (map[string]string, error) { + return dbFetchIdNameMap(CloudproviderManager, idMap) +} + +type SComputeResourceKeys struct { + quotas.SZonalCloudResourceKeys + + Hypervisor string `width:"16" charset:"ascii" nullable:"false" primary:"true" list:"user"` +} + +func (k SComputeResourceKeys) Fields() []string { + return append(k.SZonalCloudResourceKeys.Fields(), "hypervisor") +} + +func (k SComputeResourceKeys) Values() []string { + return append(k.SZonalCloudResourceKeys.Values(), k.Hypervisor) +} + +func (k1 SComputeResourceKeys) Compare(ik quotas.IQuotaKeys) int { + k2 := ik.(SComputeResourceKeys) + r := k1.SZonalCloudResourceKeys.Compare(k2.SZonalCloudResourceKeys) + if r != 0 { + return r + } + if k1.Hypervisor < k2.Hypervisor { + return -1 + } else if k1.Hypervisor > k2.Hypervisor { + return 1 + } + return 0 +} + +func fetchCloudQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, manager *SCloudprovider) quotas.SCloudResourceKeys { + keys := quotas.SCloudResourceKeys{} + keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId) + if manager != nil { + account := manager.GetCloudaccount() + keys.Provider = account.Provider + keys.Brand = account.Brand + keys.CloudEnv = account.getCloudEnv() + keys.AccountId = account.Id + keys.ManagerId = manager.Id + } else { + keys.Provider = api.CLOUD_PROVIDER_ONECLOUD + keys.Brand = api.ONECLOUD_BRAND_ONECLOUD + keys.CloudEnv = api.CLOUD_ENV_ON_PREMISE + } + return keys +} + +func fetchRegionalQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, region *SCloudregion, manager *SCloudprovider) quotas.SRegionalCloudResourceKeys { + keys := quotas.SRegionalCloudResourceKeys{} + keys.SCloudResourceKeys = fetchCloudQuotaKeys(scope, ownerId, manager) + keys.RegionId = region.Id + return keys +} + +func fetchZonalQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, zone *SZone, manager *SCloudprovider) quotas.SZonalCloudResourceKeys { + keys := quotas.SZonalCloudResourceKeys{} + keys.SCloudResourceKeys = fetchCloudQuotaKeys(scope, ownerId, manager) + keys.RegionId = zone.CloudregionId + keys.ZoneId = zone.Id + return keys +} + +func fetchComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, zone *SZone, manager *SCloudprovider, hypervisor string) SComputeResourceKeys { + keys := SComputeResourceKeys{} + keys.SZonalCloudResourceKeys = fetchZonalQuotaKeys(scope, ownerId, zone, manager) + keys.Hypervisor = hypervisor + return keys +} diff --git a/pkg/compute/models/regionquota.go b/pkg/compute/models/regionquota.go new file mode 100644 index 0000000000..727ac32022 --- /dev/null +++ b/pkg/compute/models/regionquota.go @@ -0,0 +1,355 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + identityapi "yunion.io/x/onecloud/pkg/apis/identity" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" + commonOptions "yunion.io/x/onecloud/pkg/cloudcommon/options" + "yunion.io/x/onecloud/pkg/compute/options" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/util/rbacutils" +) + +var ( + RegionQuota SRegionQuota + RegionQuotaManager *SQuotaManager + RegionUsageManager *SQuotaManager + RegionPendingUsageManager *SQuotaManager +) + +func init() { + RegionQuota = SRegionQuota{} + + RegionUsageManager = &SQuotaManager{ + SQuotaBaseManager: quotas.NewQuotaUsageManager(RegionQuota, "region_quota_usage_tbl"), + } + RegionUsageManager.SetVirtualObject(RegionUsageManager) + RegionPendingUsageManager = &SQuotaManager{ + SQuotaBaseManager: quotas.NewQuotaUsageManager(RegionQuota, "region_quota_pending_usage_tbl"), + } + RegionPendingUsageManager.SetVirtualObject(RegionPendingUsageManager) + RegionQuotaManager = &SQuotaManager{ + SQuotaBaseManager: quotas.NewQuotaBaseManager(RegionQuota, + "region_quota_tbl", + RegionPendingUsageManager, + RegionUsageManager, + "region_quota", + "region_quotas", + ), + } + RegionQuotaManager.SetVirtualObject(RegionQuotaManager) +} + +type SRegionQuota struct { + quotas.SQuotaBase + + quotas.SRegionalCloudResourceKeys + + Eip int + Port int + Eport int + Bw int + Ebw int + + Snapshot int + + Bucket int + ObjectGB int + ObjectCnt int + + Rds int + Cache int +} + +func (self *SRegionQuota) GetKeys() quotas.IQuotaKeys { + return self.SRegionalCloudResourceKeys +} + +func (self *SRegionQuota) SetKeys(keys quotas.IQuotaKeys) { + self.SRegionalCloudResourceKeys = keys.(quotas.SRegionalCloudResourceKeys) +} + +func (self *SRegionQuota) FetchSystemQuota() { + keys := self.SRegionalCloudResourceKeys + base := 0 + switch options.Options.DefaultQuotaValue { + case commonOptions.DefaultQuotaUnlimit: + base = -1 + case commonOptions.DefaultQuotaZero: + base = 0 + if keys.Scope() == rbacutils.ScopeDomain { // domain level quota + base = 10 + } else if keys.DomainId == identityapi.DEFAULT_DOMAIN_ID && keys.ProjectId == auth.AdminCredential().GetProjectId() { + base = 1 + } + case commonOptions.DefaultQuotaDefault: + base = 1 + if keys.Scope() == rbacutils.ScopeDomain { + base = 10 + } + } + defaultValue := func(def int) int { + if base < 0 { + return -1 + } else { + return def * base + } + } + self.Eip = defaultValue(options.Options.DefaultEipQuota) + self.Port = defaultValue(options.Options.DefaultPortQuota) + self.Eport = defaultValue(options.Options.DefaultEportQuota) + self.Bw = defaultValue(options.Options.DefaultBwQuota) + self.Ebw = defaultValue(options.Options.DefaultEbwQuota) + self.Snapshot = defaultValue(options.Options.DefaultSnapshotQuota) + self.Bucket = defaultValue(options.Options.DefaultBucketQuota) + self.ObjectGB = defaultValue(options.Options.DefaultObjectGBQuota) + self.ObjectCnt = defaultValue(options.Options.DefaultObjectCntQuota) + self.Rds = defaultValue(options.Options.DefaultRdsQuota) + self.Cache = defaultValue(options.Options.DefaultCacheQuota) +} + +func (self *SRegionQuota) FetchUsage(ctx context.Context) error { + regionKeys := self.SRegionalCloudResourceKeys + + scope := regionKeys.Scope() + ownerId := regionKeys.OwnerId() + + var rangeObjs []db.IStandaloneModel + if len(regionKeys.RegionId) > 0 { + obj, err := CloudregionManager.FetchById(regionKeys.RegionId) + if err != nil { + return errors.Wrap(err, "CloudregionManager.FetchById") + } + rangeObjs = append(rangeObjs, obj.(db.IStandaloneModel)) + } + if len(regionKeys.ManagerId) > 0 { + obj, err := CloudproviderManager.FetchById(regionKeys.ManagerId) + if err != nil { + return errors.Wrap(err, "CloudproviderManager.FetchById") + } + rangeObjs = append(rangeObjs, obj.(db.IStandaloneModel)) + } else if len(regionKeys.AccountId) > 0 { + obj, err := CloudaccountManager.FetchById(regionKeys.AccountId) + if err != nil { + return errors.Wrap(err, "CloudaccountManager.FetchById") + } + rangeObjs = append(rangeObjs, obj.(db.IStandaloneModel)) + } + + var providers []string + if len(regionKeys.Provider) > 0 { + providers = []string{regionKeys.Provider} + } + var brands []string + if len(regionKeys.Brand) > 0 { + brands = []string{regionKeys.Brand} + } + + net := totalGuestNicCount(scope, ownerId, rangeObjs, false, providers, brands, regionKeys.CloudEnv) + + lbnic, _ := totalLBNicCount(scope, ownerId, rangeObjs, providers, brands, regionKeys.CloudEnv) + + eipUsage := ElasticipManager.TotalCount(scope, ownerId, rangeObjs, providers, brands, regionKeys.CloudEnv) + + self.Eip = eipUsage.Total() + self.Port = net.InternalNicCount + net.InternalVirtualNicCount + lbnic + self.Eport = net.ExternalNicCount + net.ExternalVirtualNicCount + self.Bw = net.InternalBandwidth + self.Ebw = net.ExternalBandwidth + + snapshotCount, _ := TotalSnapshotCount(scope, ownerId, rangeObjs, providers, brands, regionKeys.CloudEnv) + self.Snapshot = snapshotCount + + bucketUsage := BucketManager.TotalCount(scope, ownerId, rangeObjs, providers, brands, regionKeys.CloudEnv) + self.Bucket = bucketUsage.Buckets + self.ObjectGB = int(bucketUsage.Bytes / 1000 / 1000 / 1000) + self.ObjectCnt = bucketUsage.Objects + + self.Rds, _ = DBInstanceManager.TotalCount(scope, ownerId, rangeObjs, providers, brands, regionKeys.CloudEnv) + self.Cache, _ = ElasticcacheManager.TotalCount(scope, ownerId, rangeObjs, providers, brands, regionKeys.CloudEnv) + + return nil +} + +func (self *SRegionQuota) IsEmpty() bool { + if self.Port > 0 { + return false + } + if self.Eip > 0 { + return false + } + if self.Eport > 0 { + return false + } + if self.Bw > 0 { + return false + } + if self.Ebw > 0 { + return false + } + if self.Snapshot > 0 { + return false + } + if self.Bucket > 0 { + return false + } + if self.ObjectGB > 0 { + return false + } + if self.ObjectCnt > 0 { + return false + } + if self.Rds > 0 { + return false + } + if self.Cache > 0 { + return false + } + return true +} + +func (self *SRegionQuota) Add(quota quotas.IQuota) { + squota := quota.(*SRegionQuota) + self.Port = self.Port + quotas.NonNegative(squota.Port) + self.Eip = self.Eip + quotas.NonNegative(squota.Eip) + self.Eport = self.Eport + quotas.NonNegative(squota.Eport) + self.Bw = self.Bw + quotas.NonNegative(squota.Bw) + self.Ebw = self.Ebw + quotas.NonNegative(squota.Ebw) + self.Snapshot = self.Snapshot + quotas.NonNegative(squota.Snapshot) + self.Bucket = self.Bucket + quotas.NonNegative(squota.Bucket) + self.ObjectGB = self.ObjectGB + quotas.NonNegative(squota.ObjectGB) + self.ObjectCnt = self.ObjectCnt + quotas.NonNegative(squota.ObjectCnt) + self.Rds = self.Rds + quotas.NonNegative(squota.Rds) + self.Cache = self.Cache + quotas.NonNegative(squota.Cache) +} + +func (self *SRegionQuota) Sub(quota quotas.IQuota) { + squota := quota.(*SRegionQuota) + self.Port = nonNegative(self.Port - squota.Port) + self.Eip = nonNegative(self.Eip - squota.Eip) + self.Eport = nonNegative(self.Eport - squota.Eport) + self.Bw = nonNegative(self.Bw - squota.Bw) + self.Ebw = nonNegative(self.Ebw - squota.Ebw) + self.Snapshot = nonNegative(self.Snapshot - squota.Snapshot) + self.Bucket = nonNegative(self.Bucket - squota.Bucket) + self.ObjectGB = nonNegative(self.ObjectGB - squota.ObjectGB) + self.ObjectCnt = nonNegative(self.ObjectCnt - squota.ObjectCnt) + self.Rds = nonNegative(self.Rds - squota.Rds) + self.Cache = nonNegative(self.Cache - squota.Cache) +} + +func (self *SRegionQuota) Update(quota quotas.IQuota) { + squota := quota.(*SRegionQuota) + if squota.Port > 0 { + self.Port = squota.Port + } + if squota.Eip > 0 { + self.Eip = squota.Eip + } + if squota.Eport > 0 { + self.Eport = squota.Eport + } + if squota.Bw > 0 { + self.Bw = squota.Bw + } + if squota.Ebw > 0 { + self.Ebw = squota.Ebw + } + if squota.Snapshot > 0 { + self.Snapshot = squota.Snapshot + } + if squota.Bucket > 0 { + self.Bucket = squota.Bucket + } + if squota.ObjectGB > 0 { + self.ObjectGB = squota.ObjectGB + } + if squota.ObjectCnt > 0 { + self.ObjectCnt = squota.ObjectCnt + } + if squota.Rds > 0 { + self.Rds = squota.Rds + } + if squota.Cache > 0 { + self.Cache = squota.Cache + } +} + +func (self *SRegionQuota) Exceed(request quotas.IQuota, quota quotas.IQuota) error { + err := quotas.NewOutOfQuotaError() + sreq := request.(*SRegionQuota) + squota := quota.(*SRegionQuota) + if sreq.Port > 0 && self.Port > squota.Port { + err.Add("port", squota.Port, self.Port) + } + if sreq.Eip > 0 && self.Eip > squota.Eip { + err.Add("eip", squota.Eip, self.Eip) + } + if sreq.Eport > 0 && self.Eport > squota.Eport { + err.Add("eport", squota.Eport, self.Eport) + } + if sreq.Bw > 0 && self.Bw > squota.Bw { + err.Add("bw", squota.Bw, self.Bw) + } + if sreq.Ebw > 0 && self.Ebw > squota.Ebw { + err.Add("ebw", squota.Ebw, self.Ebw) + } + if sreq.Snapshot > 0 && self.Snapshot > squota.Snapshot { + err.Add("snapshot", squota.Snapshot, self.Snapshot) + } + if sreq.Bucket > 0 && self.Bucket > squota.Bucket { + err.Add("bucket", squota.Bucket, self.Bucket) + } + if sreq.ObjectGB > 0 && self.ObjectGB > squota.ObjectGB { + err.Add("object_gb", squota.ObjectGB, self.ObjectGB) + } + if sreq.ObjectCnt > 0 && self.ObjectCnt > squota.ObjectCnt { + err.Add("object_cnt", squota.ObjectCnt, self.ObjectCnt) + } + if sreq.Rds > 0 && self.Rds > squota.Rds { + err.Add("rds", squota.Rds, self.Rds) + } + if sreq.Cache > 0 && self.Cache > squota.Cache { + err.Add("cache", squota.Cache, self.Cache) + } + if err.IsError() { + return err + } else { + return nil + } +} + +func (self *SRegionQuota) ToJSON(prefix string) jsonutils.JSONObject { + ret := jsonutils.NewDict() + ret.Add(jsonutils.NewInt(int64(self.Port)), keyName(prefix, "port")) + ret.Add(jsonutils.NewInt(int64(self.Eip)), keyName(prefix, "eip")) + ret.Add(jsonutils.NewInt(int64(self.Eport)), keyName(prefix, "eport")) + ret.Add(jsonutils.NewInt(int64(self.Bw)), keyName(prefix, "bw")) + ret.Add(jsonutils.NewInt(int64(self.Ebw)), keyName(prefix, "ebw")) + ret.Add(jsonutils.NewInt(int64(self.Snapshot)), keyName(prefix, "snapshot")) + ret.Add(jsonutils.NewInt(int64(self.Bucket)), keyName(prefix, "bucket")) + ret.Add(jsonutils.NewInt(int64(self.ObjectGB)), keyName(prefix, "object_gb")) + ret.Add(jsonutils.NewInt(int64(self.ObjectCnt)), keyName(prefix, "object_cnt")) + ret.Add(jsonutils.NewInt(int64(self.Rds)), keyName(prefix, "rds")) + ret.Add(jsonutils.NewInt(int64(self.Cache)), keyName(prefix, "cache")) + return ret +} diff --git a/pkg/compute/models/routetables.go b/pkg/compute/models/routetables.go index cd6a4c9301..1caa2d7930 100644 --- a/pkg/compute/models/routetables.go +++ b/pkg/compute/models/routetables.go @@ -26,6 +26,7 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/validators" @@ -221,7 +222,18 @@ func (man *SRouteTableManager) ValidateCreateData(ctx context.Context, userCred return nil, httperrors.NewConflictError("failed getting region of vpc %s(%s)", vpc.Name, vpc.Id) } data.Set("cloudregion_id", jsonutils.NewString(cloudregion.Id)) - return man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + + input := apis.VirtualResourceCreateInput{} + err = data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal VirtualResourceCreateInput fail %s", err) + } + input, err = man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { + return nil, err + } + data.Update(jsonutils.Marshal(input)) + return data, nil } func (manager *SRouteTableManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) { diff --git a/pkg/compute/models/schedpolicies.go b/pkg/compute/models/schedpolicies.go index 53299a25a8..48a2df0b36 100644 --- a/pkg/compute/models/schedpolicies.go +++ b/pkg/compute/models/schedpolicies.go @@ -23,6 +23,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" "yunion.io/x/onecloud/pkg/cloudcommon/db" @@ -104,7 +105,17 @@ func (manager *SSchedpolicyManager) ValidateCreateData(ctx context.Context, user return nil, err } - return manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + input := apis.StandaloneResourceCreateInput{} + err = data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal StandaloneResourceCreateInput fail %s", err) + } + input, err = manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { + return nil, err + } + data.Update(jsonutils.Marshal(input)) + return data, nil } func (self *SSchedpolicy) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { diff --git a/pkg/compute/models/schedtagjoint.go b/pkg/compute/models/schedtagjoint.go index 997652b1f0..17112769f4 100644 --- a/pkg/compute/models/schedtagjoint.go +++ b/pkg/compute/models/schedtagjoint.go @@ -21,6 +21,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/apis" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" @@ -92,7 +93,18 @@ func (man *SSchedtagJointsManager) ValidateCreateData(ctx context.Context, userC if resourceType != schedtag.ResourceType { return nil, httperrors.NewInputParameterError("Schedtag %s resource_type mismatch: %s != %s", schedtag.GetName(), schedtag.ResourceType, resourceType) } - return man.SJointResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + + input := apis.JoinResourceBaseCreateInput{} + err = data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal JointResourceCreateInput fail %s", err) + } + input, err = man.SJointResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { + return nil, err + } + data.Update(jsonutils.Marshal(input)) + return data, nil } func (man *SSchedtagJointsManager) AllowListDescendent(ctx context.Context, userCred mcclient.TokenCredential, model db.IStandaloneModel, query jsonutils.JSONObject) bool { diff --git a/pkg/compute/models/schedtags.go b/pkg/compute/models/schedtags.go index 56dc192e6a..3eb184157b 100644 --- a/pkg/compute/models/schedtags.go +++ b/pkg/compute/models/schedtags.go @@ -25,6 +25,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" @@ -207,7 +208,18 @@ func (manager *SSchedtagManager) ValidateCreateData(ctx context.Context, userCre if err != nil { return nil, err } - return manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + + input := apis.StandaloneResourceCreateInput{} + err = data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal StandaloneResourceCreateInput fail %s", err) + } + input, err = manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { + return nil, err + } + data.Update(jsonutils.Marshal(input)) + return data, nil } func (manager *SSchedtagManager) GetResourceSchedtags(resType string) ([]SSchedtag, error) { diff --git a/pkg/compute/models/secgrouprules.go b/pkg/compute/models/secgrouprules.go index 7647b34e17..6b16989e99 100644 --- a/pkg/compute/models/secgrouprules.go +++ b/pkg/compute/models/secgrouprules.go @@ -188,42 +188,43 @@ func (self *SSecurityGroupRule) BeforeInsert() { } } -func (manager *SSecurityGroupRuleManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { +func (manager *SSecurityGroupRuleManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.SSecgroupRuleCreateInput) (api.SSecgroupRuleCreateInput, error) { + data := jsonutils.Marshal(input).(*jsonutils.JSONDict) + priorityV := validators.NewRangeValidator("priority", 1, 100) priorityV.Optional(true) err := priorityV.Validate(data) if err != nil { - return nil, err + return input, err } secgroupV := validators.NewModelIdOrNameValidator("secgroup", "secgroup", ownerId) err = secgroupV.Validate(data) if err != nil { - return nil, err + return input, err } secgroup := secgroupV.Model.(*SSecurityGroup) if !secgroup.IsOwner(userCred) && !userCred.HasSystemAdminPrivilege() { - return nil, httperrors.NewForbiddenError("not enough privilege") + return input, httperrors.NewForbiddenError("not enough privilege") } - input := &api.SSecgroupRuleCreateInput{} - err = data.Unmarshal(input) + err = data.Unmarshal(&input) if err != nil { - return nil, httperrors.NewInputParameterError("Failed to unmarshal input: %v", err) + return input, httperrors.NewInputParameterError("Failed to unmarshal input: %v", err) } err = input.Check() if err != nil { - return nil, err + return input, err } - data, err = manager.SResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + input.ResourceBaseCreateInput, err = manager.SResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.ResourceBaseCreateInput) if err != nil { - return nil, err + return input, err } - return input.JSON(input), nil + return input, nil } func (self *SSecurityGroupRule) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { diff --git a/pkg/compute/models/secgroups.go b/pkg/compute/models/secgroups.go index 531f0f6d9f..629c7659ed 100644 --- a/pkg/compute/models/secgroups.go +++ b/pkg/compute/models/secgroups.go @@ -242,31 +242,26 @@ func (manager *SSecurityGroupManager) ValidateCreateData( userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, - data *jsonutils.JSONDict, -) (*jsonutils.JSONDict, error) { + input api.SSecgroupCreateInput, +) (api.SSecgroupCreateInput, error) { + var err error + // TODO: check set pending quota - - input := &api.SSecgroupCreateInput{} - - err := data.Unmarshal(input) - if err != nil { - return nil, httperrors.NewInputParameterError("Failed to unmarshal input: %v", err) - } input.Status = api.SECGROUP_STATUS_READY for i, rule := range input.Rules { err = rule.Check() if err != nil { - return nil, httperrors.NewInputParameterError("rule %d is invalid: %s", i, err) + return input, httperrors.NewInputParameterError("rule %d is invalid: %s", i, err) } } - data, err = manager.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + input.SharableVirtualResourceCreateInput, err = manager.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.SharableVirtualResourceCreateInput) if err != nil { - return nil, err + return input, err } - return input.JSON(input), nil + return input, nil } func (self *SSecurityGroup) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { diff --git a/pkg/compute/models/skus.go b/pkg/compute/models/skus.go index bdfe520330..85513f6591 100644 --- a/pkg/compute/models/skus.go +++ b/pkg/compute/models/skus.go @@ -25,15 +25,15 @@ import ( "strings" "time" - "github.com/pkg/errors" - "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/compare" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" @@ -381,7 +381,17 @@ func (self *SServerSkuManager) ValidateCreateData(ctx context.Context, userCred data.Set("name", jsonutils.NewString(name)) } - return self.SStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + input := apis.StatusStandaloneResourceCreateInput{} + err := data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal StatusStandaloneResourceCreateInput fail %s", err) + } + input, err = self.SStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { + return nil, err + } + data.Update(jsonutils.Marshal(input)) + return data, nil } func (self *SServerSku) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { diff --git a/pkg/compute/models/snapshots.go b/pkg/compute/models/snapshots.go index 650e0746f5..53447c7af8 100644 --- a/pkg/compute/models/snapshots.go +++ b/pkg/compute/models/snapshots.go @@ -216,24 +216,30 @@ func (self *SSnapshot) GetShortDesc(ctx context.Context) *jsonutils.JSONDict { return res } -func (manager *SSnapshotManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { +func (manager *SSnapshotManager) ValidateCreateData( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + query jsonutils.JSONObject, + input api.SSnapshotCreateInput, +) (api.SSnapshotCreateInput, error) { + data := jsonutils.Marshal(input).(*jsonutils.JSONDict) + diskV := validators.NewModelIdOrNameValidator("disk", "disk", ownerId) err := diskV.Validate(data) if err != nil { - return nil, err - } - disk := diskV.Model.(*SDisk) - - input := &api.SSnapshotCreateInput{ - DiskType: disk.DiskType, - Size: disk.DiskSize, + return input, err } err = data.Unmarshal(input) if err != nil { - return nil, httperrors.NewInputParameterError("failed to unmarshal input params: %v", err) + return input, httperrors.NewInputParameterError("failed to unmarshal input params: %v", err) } + disk := diskV.Model.(*SDisk) + input.DiskType = disk.DiskType + input.Size = disk.DiskSize + storage := disk.GetStorage() if len(disk.ExternalId) == 0 { input.StorageId = disk.StorageId @@ -241,36 +247,40 @@ func (manager *SSnapshotManager) ValidateCreateData(ctx context.Context, userCre input.ManagerId = storage.ManagerId region := storage.GetRegion() if region == nil { - return nil, httperrors.NewInputParameterError("failed to found region for disk's storage %s(%s)", storage.Name, storage.Id) + return input, httperrors.NewInputParameterError("failed to found region for disk's storage %s(%s)", storage.Name, storage.Id) } input.CloudregionId = region.Id driver, err := storage.GetRegionDriver() if err != nil { - return nil, err + return input, err } input.OutOfChain = driver.SnapshotIsOutOfChain(disk) - err = driver.ValidateCreateSnapshotData(ctx, userCred, disk, storage, input) + err = driver.ValidateCreateSnapshotData(ctx, userCred, disk, storage, &input) if err != nil { - return nil, err + return input, err } - quotaPlatform := disk.GetQuotaPlatformID() - pendingUsage := &SQuota{Snapshot: 1} - _, err = QuotaManager.CheckQuota(ctx, userCred, rbacutils.ScopeProject, ownerId, quotaPlatform, pendingUsage) + pendingUsage := &SRegionQuota{Snapshot: 1} + keys, err := disk.GetQuotaKeys() if err != nil { - return nil, httperrors.NewOutOfQuotaError("Check set pending quota error %s", err) + return input, err + } + pendingUsage.SetKeys(keys.(SComputeResourceKeys).SRegionalCloudResourceKeys) + err = QuotaManager.CheckQuota(ctx, pendingUsage) + if err != nil { + return input, err } - return input.JSON(input), nil + return input, nil } func (self *SSnapshot) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error { return self.SVirtualResourceBase.CustomizeCreate(ctx, userCred, ownerId, query, data) } -func (manager *SSnapshotManager) OnCreateComplete(ctx context.Context, items []db.IModel, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) { +func (manager *SSnapshotManager) OnCreateComplete(ctx context.Context, items []db.IModel, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { snapshot := items[0].(*SSnapshot) snapshot.StartSnapshotCreateTask(ctx, userCred, nil, "") } @@ -640,7 +650,7 @@ func (self *SSnapshotManager) DeleteDiskSnapshots(ctx context.Context, userCred return nil } -func TotalSnapshotCount(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, rangeObj db.IStandaloneModel, providers []string, brands []string, cloudEnv string) (int, error) { +func TotalSnapshotCount(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, rangeObjs []db.IStandaloneModel, providers []string, brands []string, cloudEnv string) (int, error) { q := SnapshotManager.Query() switch scope { @@ -651,19 +661,7 @@ func TotalSnapshotCount(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityPr q = q.Equals("tenant_id", ownerId.GetProjectId()) } - if rangeObj != nil { - switch rangeObj.Keyword() { - case "cloudprovider": - q = q.Filter(sqlchemy.Equals(q.Field("manager_id"), rangeObj.GetId())) - case "cloudaccount": - cloudproviders := CloudproviderManager.Query().SubQuery() - subq := cloudproviders.Query(cloudproviders.Field("id")).Equals("cloudaccount_id", rangeObj.GetId()).SubQuery() - q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq)) - case "cloudregion": - q = q.Filter(sqlchemy.Equals(q.Field("cloudregion_id"), rangeObj.GetId())) - } - } - + q = rangeObjectsFilter(q, rangeObjs, q.Field("cloudregion_id"), nil, q.Field("manager_id")) q = CloudProviderFilter(q, q.Field("manager_id"), providers, brands, cloudEnv) q = q.Equals("created_by", api.SNAPSHOT_MANUAL) q = q.Equals("fake_deleted", false) diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index 1a79aa837f..6c71e6bfa7 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -28,6 +28,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" @@ -202,7 +203,17 @@ func (manager *SStorageManager) ValidateCreateData(ctx context.Context, userCred return nil, err } - return manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + input := apis.StandaloneResourceCreateInput{} + err = data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal StandaloneResourceCreateInput fail %s", err) + } + input, err = manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { + return nil, err + } + data.Update(jsonutils.Marshal(input)) + return data, nil } func (self *SStorage) ValidateDeleteCondition(ctx context.Context) error { @@ -804,7 +815,7 @@ func (manager *SStorageManager) disksFailedQ(scope rbacutils.TRbacScope, ownerId } func (manager *SStorageManager) totalCapacityQ( - rangeObj db.IStandaloneModel, hostTypes []string, + rangeObjs []db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, @@ -828,7 +839,7 @@ func (manager *SStorageManager) totalCapacityQ( q = q.LeftJoin(attachedDisks, sqlchemy.Equals(attachedDisks.Field("storage_id"), storages.Field("id"))) q = q.LeftJoin(detachedDisks, sqlchemy.Equals(detachedDisks.Field("storage_id"), storages.Field("id"))) - if len(hostTypes) > 0 || len(resourceTypes) > 0 || rangeObj != nil { + if len(hostTypes) > 0 || len(resourceTypes) > 0 || len(rangeObjs) > 0 { hosts := HostManager.Query().SubQuery() hostStorages := HoststorageManager.Query().SubQuery() @@ -837,7 +848,7 @@ func (manager *SStorageManager) totalCapacityQ( q = q.Filter(sqlchemy.IsTrue(hosts.Field("enabled"))) q = q.Filter(sqlchemy.Equals(hosts.Field("host_status"), api.HOST_ONLINE)) - q = AttachUsageQuery(q, hosts, hostTypes, resourceTypes, nil, nil, "", rangeObj) + q = AttachUsageQuery(q, hosts, hostTypes, resourceTypes, nil, nil, "", rangeObjs) } q = CloudProviderFilter(q, storages.Field("manager_id"), providers, brands, cloudEnv) @@ -899,8 +910,23 @@ func (manager *SStorageManager) calculateCapacity(q *sqlchemy.SQuery) StoragesCa } } -func (manager *SStorageManager) TotalCapacity(rangeObj db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) StoragesCapacityStat { - res1 := manager.calculateCapacity(manager.totalCapacityQ(rangeObj, hostTypes, resourceTypes, providers, brands, cloudEnv, scope, ownerId)) +func (manager *SStorageManager) TotalCapacity( + rangeObjs []db.IStandaloneModel, + hostTypes []string, + resourceTypes []string, + providers []string, brands []string, cloudEnv string, + scope rbacutils.TRbacScope, + ownerId mcclient.IIdentityProvider, +) StoragesCapacityStat { + res1 := manager.calculateCapacity( + manager.totalCapacityQ( + rangeObjs, + hostTypes, + resourceTypes, + providers, brands, cloudEnv, + scope, ownerId, + ), + ) return res1 } diff --git a/pkg/compute/models/usage.go b/pkg/compute/models/usage.go index f94b2dcb82..1d2c1aacfb 100644 --- a/pkg/compute/models/usage.go +++ b/pkg/compute/models/usage.go @@ -28,7 +28,7 @@ func AttachUsageQuery( hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string, - rangeObj db.IStandaloneModel, + rangeObjs []db.IStandaloneModel, ) *sqlchemy.SQuery { if len(hostTypes) > 0 { q = q.Filter(sqlchemy.In(hosts.Field("host_type"), hostTypes)) @@ -43,42 +43,7 @@ func AttachUsageQuery( q = q.Filter(sqlchemy.In(hosts.Field("resource_type"), resourceTypes)) } } - q = CloudProviderFilter(q, hosts.Field("manager_id"), providers, brands, cloudEnv) - - if rangeObj == nil { - return q - } - //rangeObjId := rangeObj.GetId() - kw := rangeObj.Keyword() - // log.Debugf("rangeObj keyword: %s", kw) - switch kw { - case "zone": - zone := rangeObj.(*SZone) - q = q.Filter(sqlchemy.Equals(hosts.Field("zone_id"), zone.Id)) - case "wire": - wire := rangeObj.(*SWire) - hostWires := HostwireManager.Query().SubQuery() - subq := hostWires.Query(hostWires.Field("host_id")).Equals("wire_id", wire.Id).SubQuery() - q = q.Filter(sqlchemy.In(hosts.Field("id"), subq)) - case "host": - q = q.Filter(sqlchemy.Equals(hosts.Field("id"), rangeObj.GetId())) - case "cloudprovider": - q = q.Filter(sqlchemy.Equals(hosts.Field("manager_id"), rangeObj.GetId())) - case "cloudaccount": - cloudproviders := CloudproviderManager.Query().SubQuery() - subq := cloudproviders.Query(cloudproviders.Field("id")).Equals("cloudaccount_id", rangeObj.GetId()).SubQuery() - q = q.Filter(sqlchemy.In(hosts.Field("manager_id"), subq)) - case "schedtag": - aggHosts := HostschedtagManager.Query().SubQuery() - q = q.Join(aggHosts, sqlchemy.AND( - sqlchemy.Equals(hosts.Field("id"), aggHosts.Field("host_id")), - sqlchemy.IsFalse(aggHosts.Field("deleted")))). - Filter(sqlchemy.Equals(aggHosts.Field("schedtag_id"), rangeObj.GetId())) - case "cloudregion": - zones := ZoneManager.Query().SubQuery() - q = q.Join(zones, sqlchemy.Equals(hosts.Field("zone_id"), zones.Field("id"))) - q = q.Filter(sqlchemy.Equals(zones.Field("cloudregion_id"), rangeObj.GetId())) - } + q = rangeObjectsFilter(q, rangeObjs, nil, hosts.Field("zone_id"), hosts.Field("manager_id")) return q } diff --git a/pkg/compute/models/vpcs.go b/pkg/compute/models/vpcs.go index 477b678e57..d8d55ef6cb 100644 --- a/pkg/compute/models/vpcs.go +++ b/pkg/compute/models/vpcs.go @@ -27,6 +27,7 @@ import ( "yunion.io/x/pkg/util/netutils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" @@ -589,10 +590,18 @@ func (manager *SVpcManager) ValidateCreateData(ctx context.Context, userCred mcc } } } - data, err = manager.SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + + input := apis.EnabledStatusStandaloneResourceCreateInput{} + err = data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal EnabledStatusStandaloneResourceCreateInput fail %s", err) + } + input, err = manager.SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) if err != nil { return nil, err } + data.Update(jsonutils.Marshal(input)) + return region.GetDriver().ValidateCreateVpcData(ctx, userCred, data) } diff --git a/pkg/compute/models/wires.go b/pkg/compute/models/wires.go index af779f3592..302e91de41 100644 --- a/pkg/compute/models/wires.go +++ b/pkg/compute/models/wires.go @@ -27,6 +27,7 @@ import ( "yunion.io/x/pkg/util/netutils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" @@ -124,7 +125,17 @@ func (manager *SWireManager) ValidateCreateData(ctx context.Context, userCred mc data.Add(jsonutils.NewString(vpcObj.GetId()), "vpc_id") } - return manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + input := apis.StandaloneResourceCreateInput{} + err := data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal StandaloneResourceCreateInput fail %s", err) + } + input, err = manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { + return nil, err + } + data.Update(jsonutils.Marshal(input)) + return data, nil } func (wire *SWire) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { @@ -358,7 +369,13 @@ func filterByScopeOwnerId(q *sqlchemy.SQuery, scope rbacutils.TRbacScope, ownerI return q } -func (manager *SWireManager) totalCountQ(rangeObj db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) *sqlchemy.SQuery { +func (manager *SWireManager) totalCountQ( + rangeObjs []db.IStandaloneModel, + hostTypes []string, + providers []string, brands []string, cloudEnv string, + scope rbacutils.TRbacScope, + ownerId mcclient.IIdentityProvider, +) *sqlchemy.SQuery { guests := filterByScopeOwnerId(GuestManager.Query(), scope, ownerId).SubQuery() hosts := HostManager.Query().SubQuery() groups := filterByScopeOwnerId(GroupManager.Query(), scope, ownerId).SubQuery() @@ -437,12 +454,12 @@ func (manager *SWireManager) totalCountQ(rangeObj db.IStandaloneModel, hostTypes ) q = q.LeftJoin(netSQ, sqlchemy.Equals(wires.Field("id"), netSQ.Field("wire_id"))) - if rangeObj != nil || len(hostTypes) > 0 { + if len(rangeObjs) > 0 || len(hostTypes) > 0 { hostwires := HostwireManager.Query().SubQuery() sq := hostwires.Query(hostwires.Field("wire_id")) sq = sq.Join(hosts, sqlchemy.Equals(hosts.Field("id"), hostwires.Field("host_id"))) sq = sq.Filter(sqlchemy.IsTrue(hosts.Field("enabled"))) - sq = AttachUsageQuery(sq, hosts, hostTypes, nil, nil, nil, "", rangeObj) + sq = AttachUsageQuery(sq, hosts, hostTypes, nil, nil, nil, "", rangeObjs) q = q.Filter(sqlchemy.In(wires.Field("id"), sq.Distinct())) } @@ -471,9 +488,20 @@ func (wstat WiresCountStat) NicCount() int { return wstat.GuestNicCount + wstat.HostNicCount + wstat.ReservedCount + wstat.GroupNicCount + wstat.LbNicCount } -func (manager *SWireManager) TotalCount(rangeObj db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) WiresCountStat { +func (manager *SWireManager) TotalCount( + rangeObjs []db.IStandaloneModel, + hostTypes []string, + providers []string, brands []string, cloudEnv string, + scope rbacutils.TRbacScope, + ownerId mcclient.IIdentityProvider, +) WiresCountStat { stat := WiresCountStat{} - err := manager.totalCountQ(rangeObj, hostTypes, providers, brands, cloudEnv, scope, ownerId).First(&stat) + err := manager.totalCountQ( + rangeObjs, + hostTypes, + providers, brands, cloudEnv, + scope, ownerId, + ).First(&stat) if err != nil { log.Errorf("Wire total count: %v", err) } diff --git a/pkg/compute/models/zonequota.go b/pkg/compute/models/zonequota.go new file mode 100644 index 0000000000..6834ff167c --- /dev/null +++ b/pkg/compute/models/zonequota.go @@ -0,0 +1,197 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + + "yunion.io/x/jsonutils" + + identityapi "yunion.io/x/onecloud/pkg/apis/identity" + "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" + commonOptions "yunion.io/x/onecloud/pkg/cloudcommon/options" + "yunion.io/x/onecloud/pkg/compute/options" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/util/rbacutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/onecloud/pkg/cloudcommon/db" +) + +var ( + ZoneQuota SZoneQuota + ZoneQuotaManager *SQuotaManager + ZoneUsageManager *SQuotaManager + ZonePendingUsageManager *SQuotaManager +) + +func init() { + ZoneQuota = SZoneQuota{} + + ZoneUsageManager = &SQuotaManager{ + SQuotaBaseManager: quotas.NewQuotaUsageManager(RegionQuota, "zone_quota_usage_tbl"), + } + ZoneUsageManager.SetVirtualObject(ZoneUsageManager) + ZonePendingUsageManager = &SQuotaManager{ + SQuotaBaseManager: quotas.NewQuotaUsageManager(RegionQuota, "zone_quota_pending_usage_tbl"), + } + ZonePendingUsageManager.SetVirtualObject(ZonePendingUsageManager) + ZoneQuotaManager = &SQuotaManager{ + SQuotaBaseManager: quotas.NewQuotaBaseManager(RegionQuota, + "zone_quota_tbl", + ZonePendingUsageManager, + ZoneUsageManager, + "zone_quota", + "zone_quotas", + ), + } + ZoneQuotaManager.SetVirtualObject(ZoneQuotaManager) +} + +type SZoneQuota struct { + quotas.SQuotaBase + + quotas.SZonalCloudResourceKeys + + Loadbalancer int +} + +func (self *SZoneQuota) GetKeys() quotas.IQuotaKeys { + return self.SZonalCloudResourceKeys +} + +func (self *SZoneQuota) SetKeys(keys quotas.IQuotaKeys) { + self.SZonalCloudResourceKeys = keys.(quotas.SZonalCloudResourceKeys) +} + +func (self *SZoneQuota) FetchSystemQuota() { + keys := self.SBaseQuotaKeys + base := 0 + switch options.Options.DefaultQuotaValue { + case commonOptions.DefaultQuotaUnlimit: + base = -1 + case commonOptions.DefaultQuotaZero: + base = 0 + if keys.Scope() == rbacutils.ScopeDomain { // domain level quota + base = 10 + } else if keys.DomainId == identityapi.DEFAULT_DOMAIN_ID && keys.ProjectId == auth.AdminCredential().GetProjectId() { + base = 1 + } + case commonOptions.DefaultQuotaDefault: + base = 1 + if keys.Scope() == rbacutils.ScopeDomain { + base = 10 + } + } + defaultValue := func(def int) int { + if base < 0 { + return -1 + } else { + return def * base + } + } + self.Loadbalancer = defaultValue(options.Options.DefaultLoadbalancerQuota) +} + +func (self *SZoneQuota) FetchUsage(ctx context.Context) error { + keys := self.SZonalCloudResourceKeys + + scope := keys.Scope() + ownerId := keys.OwnerId() + + rangeObjs := make([]db.IStandaloneModel, 0) + if len(keys.ManagerId) > 0 { + obj, err := CloudproviderManager.FetchById(keys.ManagerId) + if err != nil { + return errors.Wrap(err, "CloudproviderManager.FetchById") + } + rangeObjs = append(rangeObjs, obj.(db.IStandaloneModel)) + } else if len(keys.AccountId) > 0 { + obj, err := CloudaccountManager.FetchById(keys.AccountId) + if err != nil { + return errors.Wrap(err, "CloudaccountManager.FetchById") + } + rangeObjs = append(rangeObjs, obj.(db.IStandaloneModel)) + } + + if len(keys.ZoneId) > 0 { + obj, err := ZoneManager.FetchById(keys.ZoneId) + if err != nil { + return errors.Wrap(err, "ZoneManager.FetchById") + } + rangeObjs = append(rangeObjs, obj.(db.IStandaloneModel)) + } else if len(keys.RegionId) > 0 { + obj, err := CloudregionManager.FetchById(keys.RegionId) + if err != nil { + return errors.Wrap(err, "CloudregionManager.FetchById") + } + rangeObjs = append(rangeObjs, obj.(db.IStandaloneModel)) + } + var providers []string + if len(keys.Provider) > 0 { + providers = []string{keys.Provider} + } + var brands []string + if len(keys.Brand) > 0 { + brands = []string{keys.Brand} + } + + self.Loadbalancer, _ = LoadbalancerManager.TotalCount(scope, ownerId, rangeObjs, providers, brands, keys.CloudEnv) + + return nil +} + +func (self *SZoneQuota) IsEmpty() bool { + if self.Loadbalancer > 0 { + return false + } + return true +} + +func (self *SZoneQuota) Add(quota quotas.IQuota) { + squota := quota.(*SZoneQuota) + self.Loadbalancer = self.Loadbalancer + quotas.NonNegative(squota.Loadbalancer) +} + +func (self *SZoneQuota) Sub(quota quotas.IQuota) { + squota := quota.(*SZoneQuota) + self.Loadbalancer = nonNegative(self.Loadbalancer - squota.Loadbalancer) +} + +func (self *SZoneQuota) Update(quota quotas.IQuota) { + squota := quota.(*SZoneQuota) + if squota.Loadbalancer > 0 { + self.Loadbalancer = squota.Loadbalancer + } +} + +func (self *SZoneQuota) Exceed(request quotas.IQuota, quota quotas.IQuota) error { + err := quotas.NewOutOfQuotaError() + sreq := request.(*SZoneQuota) + squota := quota.(*SZoneQuota) + if sreq.Loadbalancer > 0 && self.Loadbalancer > squota.Loadbalancer { + err.Add("loadbalancer", squota.Loadbalancer, self.Loadbalancer) + } + if err.IsError() { + return err + } else { + return nil + } +} + +func (self *SZoneQuota) ToJSON(prefix string) jsonutils.JSONObject { + ret := jsonutils.NewDict() + ret.Add(jsonutils.NewInt(int64(self.Loadbalancer)), keyName(prefix, "loadbalancer")) + return ret +} diff --git a/pkg/compute/models/zones.go b/pkg/compute/models/zones.go index dbd2d2343e..af24bb4ce1 100644 --- a/pkg/compute/models/zones.go +++ b/pkg/compute/models/zones.go @@ -24,6 +24,7 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" @@ -675,5 +676,16 @@ func (manager *SZoneManager) ValidateCreateData(ctx context.Context, userCred mc } data.Add(jsonutils.NewString(regionId), "cloudregion_id") data.Set("status", jsonutils.NewString(api.ZONE_ENABLE)) - return manager.SStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + + input := apis.StatusStandaloneResourceCreateInput{} + err := data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal StatusStandaloneResourceCreateInput fail %s", err) + } + input, err = manager.SStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) + if err != nil { + return nil, err + } + data.Update(jsonutils.Marshal(input)) + return data, nil } diff --git a/pkg/compute/options/options.go b/pkg/compute/options/options.go index fc3873e402..c7f9a5b3b6 100644 --- a/pkg/compute/options/options.go +++ b/pkg/compute/options/options.go @@ -50,7 +50,8 @@ type ComputeOptions struct { DefaultBandwidth int `default:"1000" help:"Default bandwidth"` DefaultMtu int `default:"1500" help:"Default network mtu"` - DefaultCpuQuota int `help:"Common CPU quota per tenant, default 200" default:"200"` + DefaultServerQuota int `default:"50" help:"Common Server quota per tenant, default 50"` + DefaultCpuQuota int `default:"200" help:"Common CPU quota per tenant, default 200"` DefaultMemoryQuota int `default:"204800" help:"Common memory quota per tenant in MB, default 200G"` DefaultStorageQuota int `default:"12288000" help:"Common storage quota per tenant in MB, default 12T"` DefaultPortQuota int `default:"200" help:"Common network port quota per tenant, default 200"` @@ -63,9 +64,14 @@ type ComputeOptions struct { DefaultSecgroupQuota int `default:"50" help:"Common security group quota per tenant, default 50"` DefaultIsolatedDeviceQuota int `default:"200" help:"Common isolated device quota per tenant, default 200"` DefaultSnapshotQuota int `default:"10" help:"Common snapshot quota per tenant, default 10"` - DefaultBucketQuota int `default:"100" help:"Common bucket quota per tenant, default 100"` - DefaultObjectGBQuota int `default:"100" help:"Common object size quota per tenant in GB, default 100GB"` - DefaultObjectCntQuota int `default:"500" help:"Common object count quota per tenant, default 500"` + + DefaultBucketQuota int `default:"100" help:"Common bucket quota per tenant, default 100"` + DefaultObjectGBQuota int `default:"100" help:"Common object size quota per tenant in GB, default 100GB"` + DefaultObjectCntQuota int `default:"500" help:"Common object count quota per tenant, default 500"` + + DefaultLoadbalancerQuota int `default:"10" help:"Common loadbalancer quota per tenant, default 10"` + DefaultRdsQuota int `default:"10" help:"Common RDS quota per tenant, default 10"` + DefaultCacheQuota int `default:"10" help:"Common ElasticCache quota per tenant, default 10"` SystemAdminQuotaCheck bool `help:"Enable quota check for system admin, default False" default:"false"` diff --git a/pkg/compute/service/handlers.go b/pkg/compute/service/handlers.go index 4f6c8b5cd4..09a5d80ee1 100644 --- a/pkg/compute/service/handlers.go +++ b/pkg/compute/service/handlers.go @@ -36,6 +36,9 @@ func InitHandlers(app *appsrv.Application) { db.AddProjectResourceCountHandler("", app) quotas.AddQuotaHandler(&models.QuotaManager.SQuotaBaseManager, "", app) + quotas.AddQuotaHandler(&models.RegionQuotaManager.SQuotaBaseManager, "", app) + quotas.AddQuotaHandler(&models.ZoneQuotaManager.SQuotaBaseManager, "", app) + quotas.AddQuotaHandler(&models.ProjectQuotaManager.SQuotaBaseManager, "", app) usages.AddUsageHandler("", app) capabilities.AddCapabilityHandler("", app) @@ -57,6 +60,16 @@ func InitHandlers(app *appsrv.Application) { models.QuotaManager, models.QuotaUsageManager, + models.QuotaPendingUsageManager, + models.ZoneQuotaManager, + models.ZoneUsageManager, + models.ZonePendingUsageManager, + models.RegionQuotaManager, + models.RegionUsageManager, + models.RegionPendingUsageManager, + models.ProjectQuotaManager, + models.ProjectUsageManager, + models.ProjectPendingUsageManager, models.GroupguestManager, } { diff --git a/pkg/compute/tasks/baremetal_convert_hypervisor_task.go b/pkg/compute/tasks/baremetal_convert_hypervisor_task.go index 959913f8a4..9c9a04ec04 100644 --- a/pkg/compute/tasks/baremetal_convert_hypervisor_task.go +++ b/pkg/compute/tasks/baremetal_convert_hypervisor_task.go @@ -64,7 +64,7 @@ func (self *BaremetalConvertHypervisorTask) OnInit(ctx context.Context, obj db.I return } input.ParentTaskId = self.GetTaskId() - models.GuestManager.OnCreateComplete(ctx, []db.IModel{guest}, self.UserCred, nil, jsonutils.Marshal(input)) + models.GuestManager.OnCreateComplete(ctx, []db.IModel{guest}, self.UserCred, self.UserCred, nil, jsonutils.Marshal(input)) } func (self *BaremetalConvertHypervisorTask) OnGuestDeployComplete(ctx context.Context, baremetal *models.SHost, body jsonutils.JSONObject) { diff --git a/pkg/compute/tasks/disk_base_task.go b/pkg/compute/tasks/disk_base_task.go index 9e33a77542..8a94022652 100644 --- a/pkg/compute/tasks/disk_base_task.go +++ b/pkg/compute/tasks/disk_base_task.go @@ -19,7 +19,6 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/onecloud/pkg/util/rbacutils" ) type SDiskBaseTask struct { @@ -38,11 +37,9 @@ func (self *SDiskBaseTask) SetStageFailed(ctx context.Context, reason string) { func (self *SDiskBaseTask) finalReleasePendingUsage(ctx context.Context) { pendingUsage := models.SQuota{} - err := self.GetPendingUsage(&pendingUsage) + err := self.GetPendingUsage(&pendingUsage, 0) if err == nil && !pendingUsage.IsEmpty() { - disk := self.getDisk() - quotaPlatform := disk.GetQuotaPlatformID() - models.QuotaManager.CancelPendingUsage(ctx, self.UserCred, rbacutils.ScopeProject, disk.GetOwnerId(), quotaPlatform, &pendingUsage, &pendingUsage) + models.QuotaManager.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &pendingUsage) } } diff --git a/pkg/compute/tasks/disk_batch_create_task.go b/pkg/compute/tasks/disk_batch_create_task.go index 269d2cd82b..7936c1dd1f 100644 --- a/pkg/compute/tasks/disk_batch_create_task.go +++ b/pkg/compute/tasks/disk_batch_create_task.go @@ -26,7 +26,6 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/onecloud/pkg/util/rbacutils" ) type DiskBatchCreateTask struct { @@ -49,9 +48,7 @@ func (self *DiskBatchCreateTask) getNeedScheduleDisks(objs []db.IStandaloneModel } func (self *DiskBatchCreateTask) clearPendingUsage(ctx context.Context, disk *models.SDisk) { - input, _ := self.GetCreateInput() - quotaPlatform := models.GetQuotaPlatformID(input.Hypervisor) - ClearTaskPendingUsage(ctx, self, rbacutils.ScopeProject, disk.GetOwnerId(), quotaPlatform) + ClearTaskPendingUsage(ctx, self) } func (self *DiskBatchCreateTask) OnInit(ctx context.Context, objs []db.IStandaloneModel, body jsonutils.JSONObject) { @@ -94,11 +91,11 @@ func (self *DiskBatchCreateTask) OnScheduleFailCallback(ctx context.Context, obj func (self *DiskBatchCreateTask) SaveScheduleResult(ctx context.Context, obj IScheduleModel, candidate *schedapi.CandidateResource) { var err error disk := obj.(*models.SDisk) - pendingUsage := models.SQuota{} - err = self.GetPendingUsage(&pendingUsage) - if err != nil { - log.Errorf("GetPendingUsage fail %s", err) - } + // pendingUsage := models.SQuota{} + // err = self.GetPendingUsage(&pendingUsage, 0) + // if err != nil { + // log.Errorf("GetPendingUsage fail %s", err) + // } // input, _ := self.GetCreateInput() // quotaPlatform := models.GetQuotaPlatformID(input.Hypervisor) @@ -142,17 +139,19 @@ func (self *DiskBatchCreateTask) SaveScheduleResult(ctx context.Context, obj ISc func (self *DiskBatchCreateTask) startCreateDisk(ctx context.Context, disk *models.SDisk) { pendingUsage := models.SQuota{} - err := self.GetPendingUsage(&pendingUsage) + err := self.GetPendingUsage(&pendingUsage, 0) if err != nil { - log.Errorf("GetPendingUsage fail %s", err) + log.Warningf("GetPendingUsage fail %s", err) } - input, _ := self.GetCreateInput() - quotaPlatform := models.GetQuotaPlatformID(input.Hypervisor) - quotaStorage := models.SQuota{Storage: disk.DiskSize} - models.QuotaManager.CancelPendingUsage(ctx, self.UserCred, rbacutils.ScopeProject, disk.GetOwnerId(), quotaPlatform, &pendingUsage, "aStorage) - self.SetPendingUsage(&pendingUsage) + keys, err := disk.GetQuotaKeys() + if err != nil { + log.Warningf("disk.GetQuotaKeys fail %s", err) + } + quotaStorage.SetKeys(keys) + models.QuotaManager.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, "aStorage) + self.SetPendingUsage(&pendingUsage, 0) disk.StartDiskCreateTask(ctx, self.GetUserCred(), false, "", self.GetTaskId()) } diff --git a/pkg/compute/tasks/guest_base_task.go b/pkg/compute/tasks/guest_base_task.go index 7c9a0258a1..9f3fd16722 100644 --- a/pkg/compute/tasks/guest_base_task.go +++ b/pkg/compute/tasks/guest_base_task.go @@ -19,7 +19,6 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/onecloud/pkg/util/rbacutils" ) type SGuestBaseTask struct { @@ -38,10 +37,13 @@ func (self *SGuestBaseTask) SetStageFailed(ctx context.Context, reason string) { func (self *SGuestBaseTask) finalReleasePendingUsage(ctx context.Context) { pendingUsage := models.SQuota{} - err := self.GetPendingUsage(&pendingUsage) + err := self.GetPendingUsage(&pendingUsage, 0) if err == nil && !pendingUsage.IsEmpty() { - guest := self.getGuest() - quotaPlatform := guest.GetQuotaPlatformID() - models.QuotaManager.CancelPendingUsage(ctx, self.UserCred, rbacutils.ScopeProject, guest.GetOwnerId(), quotaPlatform, &pendingUsage, &pendingUsage) + models.QuotaManager.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &pendingUsage) + } + pendingRegionUsage := models.SRegionQuota{} + err = self.GetPendingUsage(&pendingRegionUsage, 1) + if err == nil && !pendingRegionUsage.IsEmpty() { + models.RegionQuotaManager.CancelPendingUsage(ctx, self.UserCred, &pendingRegionUsage, &pendingRegionUsage) } } diff --git a/pkg/compute/tasks/guest_batch_create_task.go b/pkg/compute/tasks/guest_batch_create_task.go index 73bab36bba..63b6322be7 100644 --- a/pkg/compute/tasks/guest_batch_create_task.go +++ b/pkg/compute/tasks/guest_batch_create_task.go @@ -27,7 +27,6 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" - "yunion.io/x/onecloud/pkg/util/rbacutils" ) type GuestBatchCreateTask struct { @@ -45,12 +44,8 @@ func (self *GuestBatchCreateTask) GetCreateInput() (*api.ServerCreateInput, erro } func (self *GuestBatchCreateTask) clearPendingUsage(ctx context.Context, guest *models.SGuest) { - platform := make([]string, 0) - input, _ := self.GetCreateInput() - if len(input.Hypervisor) > 0 { - platform = models.GetDriver(input.Hypervisor).GetQuotaPlatformID() - } - ClearTaskPendingUsage(ctx, self, rbacutils.ScopeProject, guest.GetOwnerId(), platform) + ClearTaskPendingUsage(ctx, self) + ClearTaskPendingRegionUsage(ctx, self) } func (self *GuestBatchCreateTask) OnInit(ctx context.Context, objs []db.IStandaloneModel, body jsonutils.JSONObject) { @@ -74,18 +69,21 @@ func (self *GuestBatchCreateTask) SaveScheduleResultWithBackup(ctx context.Conte func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest *models.SGuest, candidate *schedapi.CandidateResource) error { pendingUsage := models.SQuota{} - err := self.GetPendingUsage(&pendingUsage) + err := self.GetPendingUsage(&pendingUsage, 0) if err != nil { log.Errorf("GetPendingUsage fail %s", err) } host := guest.GetHost() - quotaPlatform := host.GetQuotaPlatformID() - quotaCpuMem := models.SQuota{Cpu: int(guest.VcpuCount), Memory: guest.VmemSize} - err = models.QuotaManager.CancelPendingUsage(ctx, self.UserCred, rbacutils.ScopeProject, guest.GetOwnerId(), quotaPlatform, &pendingUsage, "aCpuMem) - self.SetPendingUsage(&pendingUsage) + keys, err := guest.GetQuotaKeys() + if err != nil { + log.Errorf("guest.GetQuotaKeys fail %s", err) + } + quotaCpuMem.SetKeys(keys) + err = models.QuotaManager.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, "aCpuMem) + self.SetPendingUsage(&pendingUsage, 0) input, err := self.GetCreateInput() if err != nil { @@ -109,9 +107,11 @@ func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest return err } + pendingRegionUsage := models.SRegionQuota{} + self.GetPendingUsage(&pendingRegionUsage, 1) // allocate networks - err = guest.CreateNetworksOnHost(ctx, self.UserCred, host, input.Networks, &pendingUsage, candidate.Nets) - self.SetPendingUsage(&pendingUsage) + err = guest.CreateNetworksOnHost(ctx, self.UserCred, host, input.Networks, &pendingRegionUsage, candidate.Nets) + self.SetPendingUsage(&pendingRegionUsage, 1) if err != nil { log.Errorf("Network failed: %s", err) guest.SetStatus(self.UserCred, api.VM_NETWORK_FAILED, err.Error()) @@ -120,8 +120,8 @@ func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest // allocate eips if input.EipBw > 0 { - eip, err := models.ElasticipManager.NewEipForVMOnHost(ctx, self.UserCred, guest, host, input.EipBw, input.EipChargeType, &pendingUsage) - self.SetPendingUsage(&pendingUsage) + eip, err := models.ElasticipManager.NewEipForVMOnHost(ctx, self.UserCred, guest, host, input.EipBw, input.EipChargeType, &pendingRegionUsage) + self.SetPendingUsage(&pendingRegionUsage, 1) if err != nil { log.Errorf("guest.CreateElasticipOnHost failed: %s", err) guest.SetStatus(self.UserCred, api.VM_NETWORK_FAILED, err.Error()) @@ -147,7 +147,7 @@ func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest } // 纳管的云需要有关联关系后,在做deploy时才有磁盘的信息 err = guest.CreateDisksOnHost(ctx, self.UserCred, host, input.Disks, &pendingUsage, true, true, candidate.Disks, backupCandidateDisks, true) - self.SetPendingUsage(&pendingUsage) + self.SetPendingUsage(&pendingUsage, 0) if err != nil { log.Errorf("Disk create failed: %s", err) @@ -157,7 +157,7 @@ func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest // allocate GPUs err = guest.CreateIsolatedDeviceOnHost(ctx, self.UserCred, host, input.IsolatedDevices, &pendingUsage) - self.SetPendingUsage(&pendingUsage) + self.SetPendingUsage(&pendingUsage, 0) if err != nil { log.Errorf("IsolatedDevices create failed: %s", err) guest.SetStatus(self.UserCred, api.VM_DEVICE_FAILED, err.Error()) diff --git a/pkg/compute/tasks/guest_change_config_task.go b/pkg/compute/tasks/guest_change_config_task.go index 8470f87b97..6cd023e852 100644 --- a/pkg/compute/tasks/guest_change_config_task.go +++ b/pkg/compute/tasks/guest_change_config_task.go @@ -27,7 +27,6 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" - "yunion.io/x/onecloud/pkg/util/rbacutils" ) type GuestChangeConfigTask struct { @@ -93,7 +92,7 @@ func (self *GuestChangeConfigTask) OnDisksResizeComplete(ctx context.Context, ob disk := iDisk.(*models.SDisk) if disk.DiskSize < int(size) { var pendingUsage models.SQuota - err = self.GetPendingUsage(&pendingUsage) + err = self.GetPendingUsage(&pendingUsage, 0) if err != nil { self.markStageFailed(ctx, guest, fmt.Sprintf("self.GetPendingUsage(&pendingUsage) fail %s", err)) return @@ -212,7 +211,7 @@ func (self *GuestChangeConfigTask) OnGuestChangeCpuMemSpecComplete(ctx context.C db.OpsLog.LogEvent(guest, db.ACT_CHANGE_FLAVOR, changeConfigSpec.String(), self.UserCred) var pendingUsage models.SQuota - err = self.GetPendingUsage(&pendingUsage) + err = self.GetPendingUsage(&pendingUsage, 0) if err != nil { self.markStageFailed(ctx, guest, fmt.Sprintf("GetPendingUsage %s", err)) return @@ -225,15 +224,22 @@ func (self *GuestChangeConfigTask) OnGuestChangeCpuMemSpecComplete(ctx context.C cancelUsage.Memory = addMem } + keys, err := guest.GetQuotaKeys() + if err != nil { + self.markStageFailed(ctx, guest, fmt.Sprintf("guest.GetQuotaKeys %s", err)) + return + } + cancelUsage.SetKeys(keys) + lockman.LockClass(ctx, guest.GetModelManager(), guest.ProjectId) defer lockman.ReleaseClass(ctx, guest.GetModelManager(), guest.ProjectId) - err = models.QuotaManager.CancelPendingUsage(ctx, self.UserCred, rbacutils.ScopeProject, guest.GetOwnerId(), guest.GetQuotaPlatformID(), &pendingUsage, &cancelUsage) + err = models.QuotaManager.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &cancelUsage) if err != nil { self.markStageFailed(ctx, guest, fmt.Sprintf("CancelPendingUsage fail %s", err)) return } - err = self.SetPendingUsage(&pendingUsage) + err = self.SetPendingUsage(&pendingUsage, 0) if err != nil { self.markStageFailed(ctx, guest, fmt.Sprintf("SetPendingUsage fail %s", err)) return diff --git a/pkg/compute/tasks/guest_resize_disk_task.go b/pkg/compute/tasks/guest_resize_disk_task.go index 96c4be0620..a88f261517 100644 --- a/pkg/compute/tasks/guest_resize_disk_task.go +++ b/pkg/compute/tasks/guest_resize_disk_task.go @@ -51,7 +51,7 @@ func (task *GuestResizeDiskTask) OnInit(ctx context.Context, obj db.IStandaloneM } pendingUsage := models.SQuota{} - err = task.GetPendingUsage(&pendingUsage) + err = task.GetPendingUsage(&pendingUsage, 0) if err != nil { task.OnTaskFailed(ctx, guest, err.Error()) return diff --git a/pkg/compute/tasks/instance_snapshot_and_clone_task.go b/pkg/compute/tasks/instance_snapshot_and_clone_task.go index 61ae09c6a6..2eab69683a 100644 --- a/pkg/compute/tasks/instance_snapshot_and_clone_task.go +++ b/pkg/compute/tasks/instance_snapshot_and_clone_task.go @@ -26,7 +26,6 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" - "yunion.io/x/onecloud/pkg/util/rbacutils" ) type InstanceSnapshotAndCloneTask struct { @@ -66,15 +65,15 @@ func (self *InstanceSnapshotAndCloneTask) SetStageFailed(ctx context.Context, re func (self *InstanceSnapshotAndCloneTask) finalReleasePendingUsage(ctx context.Context) { pendingUsage := models.SQuota{} - err := self.GetPendingUsage(&pendingUsage) + err := self.GetPendingUsage(&pendingUsage, 0) if err == nil && !pendingUsage.IsEmpty() { - isp := self.GetObject().(*models.SInstanceSnapshot) - guest := models.GuestManager.FetchGuestById(isp.GuestId) - quotaPlatform := guest.GetQuotaPlatformID() - models.QuotaManager.CancelPendingUsage( - ctx, self.UserCred, rbacutils.ScopeProject, - guest.GetOwnerId(), quotaPlatform, &pendingUsage, &pendingUsage, - ) + models.QuotaManager.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &pendingUsage) + } + + pendingRegionUsage := models.SRegionQuota{} + err = self.GetPendingUsage(&pendingRegionUsage, 1) + if err == nil && !pendingRegionUsage.IsEmpty() { + models.QuotaManager.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &pendingUsage) } } @@ -128,7 +127,7 @@ func (self *InstanceSnapshotAndCloneTask) doGuestCreate( continue } isp.AddRefCount(ctx) - models.GuestManager.OnCreateComplete(ctx, []db.IModel{newGuest}, self.UserCred, nil, input) + models.GuestManager.OnCreateComplete(ctx, []db.IModel{newGuest}, self.UserCred, self.UserCred, nil, input) } if len(errStr) > 0 { return fmt.Errorf(errStr) diff --git a/pkg/compute/tasks/instance_snapshot_create_task.go b/pkg/compute/tasks/instance_snapshot_create_task.go index 8dd96872d3..6037ad4131 100644 --- a/pkg/compute/tasks/instance_snapshot_create_task.go +++ b/pkg/compute/tasks/instance_snapshot_create_task.go @@ -29,7 +29,6 @@ import ( "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" "yunion.io/x/onecloud/pkg/util/rand" - "yunion.io/x/onecloud/pkg/util/rbacutils" ) type InstanceSnapshotCreateTask struct { @@ -46,16 +45,10 @@ func (self *InstanceSnapshotCreateTask) SetStageFailed(ctx context.Context, reas } func (self *InstanceSnapshotCreateTask) finalReleasePendingUsage(ctx context.Context) { - pendingUsage := models.SQuota{} - err := self.GetPendingUsage(&pendingUsage) + pendingUsage := models.SRegionQuota{} + err := self.GetPendingUsage(&pendingUsage, 0) if err == nil && !pendingUsage.IsEmpty() { - isp := self.GetObject().(*models.SInstanceSnapshot) - guest := models.GuestManager.FetchGuestById(isp.GuestId) - quotaPlatform := guest.GetQuotaPlatformID() - models.QuotaManager.CancelPendingUsage( - ctx, self.UserCred, rbacutils.ScopeProject, - guest.GetOwnerId(), quotaPlatform, &pendingUsage, &pendingUsage, - ) + models.RegionQuotaManager.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &pendingUsage) } } diff --git a/pkg/compute/tasks/pending_usage.go b/pkg/compute/tasks/pending_usage.go index f4feff7313..facd39522a 100644 --- a/pkg/compute/tasks/pending_usage.go +++ b/pkg/compute/tasks/pending_usage.go @@ -22,24 +22,52 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/onecloud/pkg/util/rbacutils" ) -func ClearTaskPendingUsage(ctx context.Context, task taskman.ITask, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string) error { +func ClearTaskPendingUsage(ctx context.Context, task taskman.ITask) error { + index := 0 pendingUsage := models.SQuota{} - err := task.GetPendingUsage(&pendingUsage) + err := task.GetPendingUsage(&pendingUsage, index) if err != nil { log.Errorf("GetPendingUsage fail %s", err) return errors.Wrap(err, "task.GetPendingUsage") } - err = models.QuotaManager.CancelPendingUsage(ctx, task.GetUserCred(), scope, ownerId, platform, &pendingUsage, &pendingUsage) + + err = models.QuotaManager.CancelPendingUsage(ctx, task.GetUserCred(), &pendingUsage, &pendingUsage) if err != nil { return errors.Wrap(err, "models.QuotaManager.CancelPendingUsage") } - err = task.ClearPendingUsage() - if err != nil { - return errors.Wrap(err, "task.ClearPendingUsage") + if pendingUsage.IsEmpty() { + err = task.ClearPendingUsage(index) + if err != nil { + return errors.Wrap(err, "task.ClearPendingUsage") + } + } else { + log.Warningf("pendingUsage is not empty after cancel????") + } + return nil +} + +func ClearTaskPendingRegionUsage(ctx context.Context, task taskman.ITask) error { + index := 1 + pendingUsage := models.SRegionQuota{} + err := task.GetPendingUsage(&pendingUsage, index) + if err != nil { + log.Errorf("GetPendingUsage fail %s", err) + return errors.Wrap(err, "task.GetPendingUsage") + } + + err = models.RegionQuotaManager.CancelPendingUsage(ctx, task.GetUserCred(), &pendingUsage, &pendingUsage) + if err != nil { + return errors.Wrap(err, "models.QuotaManager.CancelPendingUsage") + } + if pendingUsage.IsEmpty() { + err = task.ClearPendingUsage(index) + if err != nil { + return errors.Wrap(err, "task.ClearPendingUsage") + } + } else { + log.Warningf("pendingUsage is not empty after cancel????") } return nil } diff --git a/pkg/compute/tasks/schedule.go b/pkg/compute/tasks/schedule.go index e1ba394f3f..be8aba1ea5 100644 --- a/pkg/compute/tasks/schedule.go +++ b/pkg/compute/tasks/schedule.go @@ -35,7 +35,6 @@ import ( "yunion.io/x/onecloud/pkg/mcclient/auth" "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/util/logclient" - "yunion.io/x/onecloud/pkg/util/rbacutils" ) type IScheduleModel interface { @@ -47,7 +46,7 @@ type IScheduleModel interface { type IScheduleTask interface { GetUserCred() mcclient.TokenCredential GetSchedParams() (*schedapi.ScheduleInput, error) - GetPendingUsage(quota quotas.IQuota) error + GetPendingUsage(quota quotas.IQuota, index int) error SetStage(stageName string, data *jsonutils.JSONDict) error SetStageFailed(ctx context.Context, reason string) @@ -160,24 +159,29 @@ func doScheduleObjects( func cancelPendingUsage(ctx context.Context, task IScheduleTask) { pendingUsage := models.SQuota{} - err := task.GetPendingUsage(&pendingUsage) + err := task.GetPendingUsage(&pendingUsage, 0) if err != nil { log.Errorf("Taks GetPendingUsage fail %s", err) return } - schedInput, err := task.GetSchedParams() + if !pendingUsage.IsEmpty() { + err = models.QuotaManager.CancelPendingUsage(ctx, task.GetUserCred(), &pendingUsage, &pendingUsage) + if err != nil { + log.Errorf("cancelpendingusage error %s", err) + } + } + + pendingRegionUsage := models.SRegionQuota{} + err = task.GetPendingUsage(&pendingRegionUsage, 0) if err != nil { - log.Errorf("cancelPendingUsage GetSchedParams fail: %s", err) + log.Errorf("Taks GetRegionPendingUsage fail %s", err) return } - ownerId := db.SOwnerId{ - ProjectId: schedInput.ServerConfig.Project, - DomainId: schedInput.ServerConfig.Domain, - } - quotaPlatform := models.GetQuotaPlatformID(schedInput.Hypervisor) - err = models.QuotaManager.CancelPendingUsage(ctx, task.GetUserCred(), rbacutils.ScopeProject, &ownerId, quotaPlatform, &pendingUsage, &pendingUsage) - if err != nil { - log.Errorf("cancelpendingusage error %s", err) + if !pendingRegionUsage.IsEmpty() { + err = models.RegionQuotaManager.CancelPendingUsage(ctx, task.GetUserCred(), &pendingRegionUsage, &pendingRegionUsage) + if err != nil { + log.Errorf("cancelpendingusage error %s", err) + } } } diff --git a/pkg/compute/usages/handler.go b/pkg/compute/usages/handler.go index 0e4d6c907a..35d646d43d 100644 --- a/pkg/compute/usages/handler.go +++ b/pkg/compute/usages/handler.go @@ -63,7 +63,7 @@ func (u Usage) Include(nus ...Usage) Usage { return u } -type objUsageFunc func(rbacutils.TRbacScope, mcclient.IIdentityProvider, bool, db.IStandaloneModel, []string, []string, []string, string) (Usage, error) +type objUsageFunc func(rbacutils.TRbacScope, mcclient.IIdentityProvider, bool, []db.IStandaloneModel, []string, []string, []string, string) (Usage, error) func getRangeObjId(ctx context.Context) (string, error) { params := appctx.AppContextParams(ctx) @@ -116,7 +116,7 @@ func rangeObjHandler( providers := json.GetQueryStringArray(query, "provider") brands := json.GetQueryStringArray(query, "brand") cloudEnv, _ := query.GetString("cloud_env") - usage, err := reporter(scope, ownerId, isOwner, obj, hostTypes, providers, brands, cloudEnv) + usage, err := reporter(scope, ownerId, isOwner, []db.IStandaloneModel{obj}, hostTypes, providers, brands, cloudEnv) if err != nil { httperrors.GeneralServerError(w, err) return @@ -166,35 +166,35 @@ func getQuery(r *http.Request) json.JSONObject { return query } -func ReportHostUsage(scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, isOwner bool, host db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) (Usage, error) { - return ReportGeneralUsage(scope, userCred, isOwner, host, hostTypes, providers, brands, cloudEnv) +func ReportHostUsage(scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, isOwner bool, hosts []db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) (Usage, error) { + return ReportGeneralUsage(scope, userCred, isOwner, hosts, hostTypes, providers, brands, cloudEnv) } -func ReportWireUsage(scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, isOwner bool, wire db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) (Usage, error) { - return ReportGeneralUsage(scope, userCred, isOwner, wire, hostTypes, providers, brands, cloudEnv) +func ReportWireUsage(scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, isOwner bool, wires []db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) (Usage, error) { + return ReportGeneralUsage(scope, userCred, isOwner, wires, hostTypes, providers, brands, cloudEnv) } -func ReportCloudAccountUsage(scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, isOwner bool, account db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) (Usage, error) { - return ReportGeneralUsage(scope, userCred, isOwner, account, hostTypes, providers, brands, cloudEnv) +func ReportCloudAccountUsage(scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, isOwner bool, accounts []db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) (Usage, error) { + return ReportGeneralUsage(scope, userCred, isOwner, accounts, hostTypes, providers, brands, cloudEnv) } -func ReportCloudProviderUsage(scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, isOwner bool, provider db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) (Usage, error) { - return ReportGeneralUsage(scope, userCred, isOwner, provider, hostTypes, providers, brands, cloudEnv) +func ReportCloudProviderUsage(scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, isOwner bool, managers []db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) (Usage, error) { + return ReportGeneralUsage(scope, userCred, isOwner, managers, hostTypes, providers, brands, cloudEnv) } -func ReportSchedtagUsage(scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, isOwner bool, schedtag db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) (Usage, error) { - return ReportGeneralUsage(scope, userCred, isOwner, schedtag, hostTypes, providers, brands, cloudEnv) +func ReportSchedtagUsage(scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, isOwner bool, schedtags []db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) (Usage, error) { + return ReportGeneralUsage(scope, userCred, isOwner, schedtags, hostTypes, providers, brands, cloudEnv) } -func ReportZoneUsage(scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, isOwner bool, zone db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) (Usage, error) { - return ReportGeneralUsage(scope, userCred, isOwner, zone, hostTypes, providers, brands, cloudEnv) +func ReportZoneUsage(scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, isOwner bool, zones []db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) (Usage, error) { + return ReportGeneralUsage(scope, userCred, isOwner, zones, hostTypes, providers, brands, cloudEnv) } -func ReportCloudRegionUsage(scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, isOwner bool, cloudRegion db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) (Usage, error) { - return ReportGeneralUsage(scope, userCred, isOwner, cloudRegion, hostTypes, providers, brands, cloudEnv) +func ReportCloudRegionUsage(scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, isOwner bool, cloudRegions []db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) (Usage, error) { + return ReportGeneralUsage(scope, userCred, isOwner, cloudRegions, hostTypes, providers, brands, cloudEnv) } -func getAdminGeneralUsage(userCred mcclient.IIdentityProvider, rangeObj db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) (count Usage, err error) { +func getAdminGeneralUsage(userCred mcclient.IIdentityProvider, rangeObjs []db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) (count Usage, err error) { count = RegionUsage(providers, brands, cloudEnv) zone := ZoneUsage(providers, brands, cloudEnv) count.Include(zone) @@ -204,11 +204,11 @@ func getAdminGeneralUsage(userCred mcclient.IIdentityProvider, rangeObj db.IStan var pmemTotal float64 var pcpuTotal float64 - hostEnabledUsage := HostEnabledUsage("", userCred, rangeObj, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv) + hostEnabledUsage := HostEnabledUsage("", userCred, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv) pmemTotal = float64(hostEnabledUsage.Get("enabled_hosts.memory").(int64)) pcpuTotal = float64(hostEnabledUsage.Get("enabled_hosts.cpu").(int64)) - if rangeObj != nil && rangeObj.Keyword() == "host" { - host := rangeObj.(*models.SHost) + if len(rangeObjs) > 0 && rangeObjs[0].Keyword() == "host" { + host := rangeObjs[0].(*models.SHost) pmemTotal = float64(host.MemSize) pcpuTotal = float64(host.CpuCount) count.Add("memory", host.MemSize) @@ -217,11 +217,11 @@ func getAdminGeneralUsage(userCred mcclient.IIdentityProvider, rangeObj db.IStan count.Add("cpu.virtual", host.GetVirtualCPUCount()) } - guestRunningUsage := GuestRunningUsage("all.running_servers", rbacutils.ScopeSystem, nil, rangeObj, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv) + guestRunningUsage := GuestRunningUsage("all.running_servers", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv) runningMem := guestRunningUsage.Get("all.running_servers.memory").(int) runningCpu := guestRunningUsage.Get("all.running_servers.cpu").(int) - containerRunningUsage := containerUsage("all.containers", rbacutils.ScopeSystem, nil, rangeObj, hostTypes, nil, providers, brands, cloudEnv) + containerRunningUsage := containerUsage("all.containers", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, nil, providers, brands, cloudEnv) containerRunningMem := containerRunningUsage.Get("all.containers.memory").(int) containerRunningCpu := containerRunningUsage.Get("all.containers.cpu").(int) runningMem += containerRunningMem @@ -237,93 +237,93 @@ func getAdminGeneralUsage(userCred mcclient.IIdentityProvider, rangeObj db.IStan count.Add("all.memory_commit_rate.running", runningMemCmtRate) count.Add("all.cpu_commit_rate.running", runningCpuCmtRate) - storageUsage := StorageUsage("", rangeObj, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv) + storageUsage := StorageUsage("", rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv) count.Include( - HostAllUsage("", userCred, rangeObj, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv), - HostAllUsage("prepaid_pool", userCred, rangeObj, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), - HostAllUsage("any_pool", userCred, rangeObj, hostTypes, nil, providers, brands, cloudEnv), + HostAllUsage("", userCred, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv), + HostAllUsage("prepaid_pool", userCred, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), + HostAllUsage("any_pool", userCred, rangeObjs, hostTypes, nil, providers, brands, cloudEnv), hostEnabledUsage, - HostEnabledUsage("prepaid_pool", userCred, rangeObj, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), - HostEnabledUsage("any_pool", userCred, rangeObj, hostTypes, nil, providers, brands, cloudEnv), + HostEnabledUsage("prepaid_pool", userCred, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), + HostEnabledUsage("any_pool", userCred, rangeObjs, hostTypes, nil, providers, brands, cloudEnv), - BaremetalUsage(userCred, rangeObj, hostTypes, providers, brands, cloudEnv), + BaremetalUsage(userCred, rangeObjs, hostTypes, providers, brands, cloudEnv), storageUsage, - StorageUsage("prepaid_pool", rangeObj, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), - StorageUsage("any_pool", rangeObj, hostTypes, nil, providers, brands, cloudEnv), + StorageUsage("prepaid_pool", rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), + StorageUsage("any_pool", rangeObjs, hostTypes, nil, providers, brands, cloudEnv), - GuestNormalUsage("all.servers", rbacutils.ScopeSystem, nil, rangeObj, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv), - GuestNormalUsage("all.servers.prepaid_pool", rbacutils.ScopeSystem, nil, rangeObj, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), - GuestNormalUsage("all.servers.any_pool", rbacutils.ScopeSystem, nil, rangeObj, hostTypes, nil, providers, brands, cloudEnv), + GuestNormalUsage("all.servers", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv), + GuestNormalUsage("all.servers.prepaid_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), + GuestNormalUsage("all.servers.any_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, nil, providers, brands, cloudEnv), - GuestPendingDeleteUsage("all.pending_delete_servers", rbacutils.ScopeSystem, nil, rangeObj, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv), - GuestPendingDeleteUsage("all.pending_delete_servers.prepaid_pool", rbacutils.ScopeSystem, nil, rangeObj, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), - GuestPendingDeleteUsage("all.pending_delete_servers.any_pool", rbacutils.ScopeSystem, nil, rangeObj, hostTypes, nil, providers, brands, cloudEnv), + GuestPendingDeleteUsage("all.pending_delete_servers", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv), + GuestPendingDeleteUsage("all.pending_delete_servers.prepaid_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), + GuestPendingDeleteUsage("all.pending_delete_servers.any_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, nil, providers, brands, cloudEnv), - GuestReadyUsage("all.ready_servers", rbacutils.ScopeSystem, nil, rangeObj, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv), - GuestReadyUsage("all.ready_servers.prepaid_pool", rbacutils.ScopeSystem, nil, rangeObj, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), - GuestReadyUsage("all.ready_servers.any_pool", rbacutils.ScopeSystem, nil, rangeObj, hostTypes, nil, providers, brands, cloudEnv), + GuestReadyUsage("all.ready_servers", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv), + GuestReadyUsage("all.ready_servers.prepaid_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), + GuestReadyUsage("all.ready_servers.any_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, nil, providers, brands, cloudEnv), guestRunningUsage, - GuestRunningUsage("all.running_servers.prepaid_pool", rbacutils.ScopeSystem, nil, rangeObj, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), - GuestRunningUsage("all.running_servers.any_pool", rbacutils.ScopeSystem, nil, rangeObj, hostTypes, nil, providers, brands, cloudEnv), + GuestRunningUsage("all.running_servers.prepaid_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), + GuestRunningUsage("all.running_servers.any_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, nil, providers, brands, cloudEnv), containerRunningUsage, - IsolatedDeviceUsage("", rangeObj, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv), - IsolatedDeviceUsage("prepaid_pool", rangeObj, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), - IsolatedDeviceUsage("any_pool", rangeObj, hostTypes, nil, providers, brands, cloudEnv), + IsolatedDeviceUsage("", rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv), + IsolatedDeviceUsage("prepaid_pool", rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), + IsolatedDeviceUsage("any_pool", rangeObjs, hostTypes, nil, providers, brands, cloudEnv), - WireUsage(rangeObj, hostTypes, providers, brands, cloudEnv), + WireUsage(rangeObjs, hostTypes, providers, brands, cloudEnv), - NetworkUsage("all", rbacutils.ScopeSystem, nil, providers, brands, cloudEnv, rangeObj), + NetworkUsage("all", rbacutils.ScopeSystem, nil, providers, brands, cloudEnv, rangeObjs), - EipUsage(rbacutils.ScopeSystem, nil, rangeObj, providers, brands, cloudEnv), + EipUsage(rbacutils.ScopeSystem, nil, rangeObjs, providers, brands, cloudEnv), - BucketUsage(rbacutils.ScopeSystem, nil, rangeObj, providers, brands, cloudEnv), + BucketUsage(rbacutils.ScopeSystem, nil, rangeObjs, providers, brands, cloudEnv), - SnapshotUsage(rbacutils.ScopeSystem, nil, rangeObj, providers, brands, cloudEnv), + SnapshotUsage(rbacutils.ScopeSystem, nil, rangeObjs, providers, brands, cloudEnv), ) return } -func getCommonGeneralUsage(scope rbacutils.TRbacScope, cred mcclient.IIdentityProvider, rangeObj db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) (count Usage, err error) { - guestNormalUsage := GuestNormalUsage("servers", scope, cred, rangeObj, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv) +func getCommonGeneralUsage(scope rbacutils.TRbacScope, cred mcclient.IIdentityProvider, rangeObjs []db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) (count Usage, err error) { + guestNormalUsage := GuestNormalUsage("servers", scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv) - containerUsage := containerUsage("containers", scope, cred, rangeObj, hostTypes, nil, providers, brands, cloudEnv) + containerUsage := containerUsage("containers", scope, cred, rangeObjs, hostTypes, nil, providers, brands, cloudEnv) - eipUsage := EipUsage(scope, cred, rangeObj, providers, brands, cloudEnv) + eipUsage := EipUsage(scope, cred, rangeObjs, providers, brands, cloudEnv) - bucketUsage := BucketUsage(scope, cred, rangeObj, providers, brands, cloudEnv) + bucketUsage := BucketUsage(scope, cred, rangeObjs, providers, brands, cloudEnv) - snapshotUsage := SnapshotUsage(scope, cred, rangeObj, providers, brands, cloudEnv) + snapshotUsage := SnapshotUsage(scope, cred, rangeObjs, providers, brands, cloudEnv) - disksUsage := disksUsage("", rangeObj, nil, nil, providers, brands, cloudEnv, scope, cred) + disksUsage := disksUsage("", rangeObjs, nil, nil, providers, brands, cloudEnv, scope, cred) - nicsUsage := nicsUsage(rangeObj, nil, providers, brands, cloudEnv, scope, cred) + nicsUsage := nicsUsage(rangeObjs, nil, providers, brands, cloudEnv, scope, cred) count = guestNormalUsage.Include( - GuestNormalUsage("servers.prepaid_pool", scope, cred, rangeObj, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), - GuestNormalUsage("servers.any_pool", scope, cred, rangeObj, hostTypes, nil, providers, brands, cloudEnv), + GuestNormalUsage("servers.prepaid_pool", scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), + GuestNormalUsage("servers.any_pool", scope, cred, rangeObjs, hostTypes, nil, providers, brands, cloudEnv), - GuestRunningUsage("running_servers", scope, cred, rangeObj, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv), - GuestRunningUsage("running_servers.prepaid_pool", scope, cred, rangeObj, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), - GuestRunningUsage("running_servers.any_pool", scope, cred, rangeObj, hostTypes, nil, providers, brands, cloudEnv), + GuestRunningUsage("running_servers", scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv), + GuestRunningUsage("running_servers.prepaid_pool", scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), + GuestRunningUsage("running_servers.any_pool", scope, cred, rangeObjs, hostTypes, nil, providers, brands, cloudEnv), - GuestPendingDeleteUsage("pending_delete_servers", scope, cred, rangeObj, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv), - GuestPendingDeleteUsage("pending_delete_servers.prepaid_pool", scope, cred, rangeObj, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), - GuestPendingDeleteUsage("pending_delete_servers.any_pool", scope, cred, rangeObj, hostTypes, nil, providers, brands, cloudEnv), + GuestPendingDeleteUsage("pending_delete_servers", scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv), + GuestPendingDeleteUsage("pending_delete_servers.prepaid_pool", scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), + GuestPendingDeleteUsage("pending_delete_servers.any_pool", scope, cred, rangeObjs, hostTypes, nil, providers, brands, cloudEnv), - GuestReadyUsage("ready_servers", scope, cred, rangeObj, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv), - GuestReadyUsage("ready_servers.prepaid_pool", scope, cred, rangeObj, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), - GuestReadyUsage("ready_servers.any_pool", scope, cred, rangeObj, hostTypes, nil, providers, brands, cloudEnv), + GuestReadyUsage("ready_servers", scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv), + GuestReadyUsage("ready_servers.prepaid_pool", scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv), + GuestReadyUsage("ready_servers.any_pool", scope, cred, rangeObjs, hostTypes, nil, providers, brands, cloudEnv), containerUsage, - NetworkUsage("", scope, cred, providers, brands, cloudEnv, rangeObj), + NetworkUsage("", scope, cred, providers, brands, cloudEnv, rangeObjs), eipUsage, @@ -338,23 +338,32 @@ func getCommonGeneralUsage(scope rbacutils.TRbacScope, cred mcclient.IIdentityPr return } -func ReportGeneralUsage(scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, isOwner bool, rangeObj db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) (count Usage, err error) { +func ReportGeneralUsage( + scope rbacutils.TRbacScope, + userCred mcclient.IIdentityProvider, + isOwner bool, + rangeObjs []db.IStandaloneModel, + hostTypes []string, + providers []string, + brands []string, + cloudEnv string, +) (count Usage, err error) { count = make(map[string]interface{}) if scope == rbacutils.ScopeSystem || isOwner { - count, err = getAdminGeneralUsage(userCred, rangeObj, hostTypes, providers, brands, cloudEnv) + count, err = getAdminGeneralUsage(userCred, rangeObjs, hostTypes, providers, brands, cloudEnv) if err != nil { return } } if scope.HigherEqual(rbacutils.ScopeDomain) { - commonUsage, err := getCommonGeneralUsage(rbacutils.ScopeDomain, userCred, rangeObj, hostTypes, providers, brands, cloudEnv) + commonUsage, err := getCommonGeneralUsage(rbacutils.ScopeDomain, userCred, rangeObjs, hostTypes, providers, brands, cloudEnv) if err == nil { count.Include(commonUsage) } } else if scope.HigherEqual(rbacutils.ScopeProject) { - commonUsage, err := getCommonGeneralUsage(rbacutils.ScopeProject, userCred, rangeObj, hostTypes, providers, brands, cloudEnv) + commonUsage, err := getCommonGeneralUsage(rbacutils.ScopeProject, userCred, rangeObjs, hostTypes, providers, brands, cloudEnv) if err == nil { count.Include(commonUsage) } @@ -401,7 +410,12 @@ func VpcUsage(providers []string, brands []string, cloudEnv string) Usage { return count } -func StorageUsage(prefix string, rangeObj db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string) Usage { +func StorageUsage( + prefix string, + rangeObjs []db.IStandaloneModel, + hostTypes []string, resourceTypes []string, + providers []string, brands []string, cloudEnv string, +) Usage { sPrefix := "storages" dPrefix := "all.disks" if len(prefix) > 0 { @@ -409,7 +423,12 @@ func StorageUsage(prefix string, rangeObj db.IStandaloneModel, hostTypes []strin dPrefix = fmt.Sprintf("%s.%s", dPrefix, prefix) } count := make(map[string]interface{}) - result := models.StorageManager.TotalCapacity(rangeObj, hostTypes, resourceTypes, providers, brands, cloudEnv, rbacutils.ScopeSystem, nil) + result := models.StorageManager.TotalCapacity( + rangeObjs, + hostTypes, resourceTypes, + providers, brands, cloudEnv, + rbacutils.ScopeSystem, nil, + ) count[sPrefix] = result.Capacity count[fmt.Sprintf("%s.virtual", sPrefix)] = result.CapacityVirtual count[dPrefix] = result.CapacityUsed @@ -426,13 +445,13 @@ func StorageUsage(prefix string, rangeObj db.IStandaloneModel, hostTypes []strin return count } -func disksUsage(prefix string, rangeObj db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) Usage { +func disksUsage(prefix string, rangeObjs []db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) Usage { dPrefix := "disks" if len(prefix) > 0 { dPrefix = fmt.Sprintf("%s.%s", dPrefix, prefix) } count := make(map[string]interface{}) - result := models.StorageManager.TotalCapacity(rangeObj, hostTypes, resourceTypes, providers, brands, cloudEnv, scope, ownerId) + result := models.StorageManager.TotalCapacity(rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, scope, ownerId) count[dPrefix] = result.CapacityUsed count[fmt.Sprintf("%s.unready", dPrefix)] = result.CapacityUnready count[fmt.Sprintf("%s.attached", dPrefix)] = result.AttachedCapacity @@ -441,9 +460,9 @@ func disksUsage(prefix string, rangeObj db.IStandaloneModel, hostTypes []string, return count } -func WireUsage(rangeObj db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) Usage { +func WireUsage(rangeObjs []db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) Usage { count := make(map[string]interface{}) - result := models.WireManager.TotalCount(rangeObj, hostTypes, providers, brands, cloudEnv, rbacutils.ScopeSystem, nil) + result := models.WireManager.TotalCount(rangeObjs, hostTypes, providers, brands, cloudEnv, rbacutils.ScopeSystem, nil) count["wires"] = result.WiresCount count["networks"] = result.NetCount count["all.nics.guest"] = result.GuestNicCount @@ -455,9 +474,9 @@ func WireUsage(rangeObj db.IStandaloneModel, hostTypes []string, providers []str return count } -func nicsUsage(rangeObj db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) Usage { +func nicsUsage(rangeObjs []db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) Usage { count := make(map[string]interface{}) - result := models.WireManager.TotalCount(rangeObj, hostTypes, providers, brands, cloudEnv, scope, ownerId) + result := models.WireManager.TotalCount(rangeObjs, hostTypes, providers, brands, cloudEnv, scope, ownerId) count["nics.guest"] = result.GuestNicCount count["nics.group"] = result.GroupNicCount count["nics.lb"] = result.LbNicCount @@ -473,36 +492,36 @@ func prefixKey(prefix string, key string) string { } } -func NetworkUsage(prefix string, scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, providers []string, brands []string, cloudEnv string, rangeObj db.IStandaloneModel) Usage { +func NetworkUsage(prefix string, scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, providers []string, brands []string, cloudEnv string, rangeObjs []db.IStandaloneModel) Usage { count := make(map[string]interface{}) - ret := models.NetworkManager.TotalPortCount(scope, userCred, providers, brands, cloudEnv, rangeObj) + ret := models.NetworkManager.TotalPortCount(scope, userCred, providers, brands, cloudEnv, rangeObjs) count[prefixKey(prefix, "ports")] = ret.Count count[prefixKey(prefix, "ports_exit")] = ret.CountExt return count } -func HostAllUsage(pref string, userCred mcclient.IIdentityProvider, rangeObj db.IStandaloneModel, +func HostAllUsage(pref string, userCred mcclient.IIdentityProvider, rangeObjs []db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string) Usage { prefix := "hosts" if len(pref) > 0 { prefix = fmt.Sprintf("%s.%s", prefix, pref) } - return hostUsage(userCred, prefix, rangeObj, hostTypes, resourceTypes, providers, brands, cloudEnv, tristate.None, tristate.None) + return hostUsage(userCred, prefix, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, tristate.None, tristate.None) } -func HostEnabledUsage(pref string, userCred mcclient.IIdentityProvider, rangeObj db.IStandaloneModel, +func HostEnabledUsage(pref string, userCred mcclient.IIdentityProvider, rangeObjs []db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string) Usage { prefix := "enabled_hosts" if len(pref) > 0 { prefix = fmt.Sprintf("%s.%s", prefix, pref) } - return hostUsage(userCred, prefix, rangeObj, hostTypes, resourceTypes, providers, brands, cloudEnv, tristate.True, tristate.None) + return hostUsage(userCred, prefix, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, tristate.True, tristate.None) } -func BaremetalUsage(userCred mcclient.IIdentityProvider, rangeObj db.IStandaloneModel, +func BaremetalUsage(userCred mcclient.IIdentityProvider, rangeObjs []db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) Usage { prefix := "baremetals" - count := hostUsage(userCred, prefix, rangeObj, hostTypes, nil, providers, brands, cloudEnv, tristate.None, tristate.True) + count := hostUsage(userCred, prefix, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, tristate.None, tristate.True) delete(count, fmt.Sprintf("%s.memory.virtual", prefix)) delete(count, fmt.Sprintf("%s.cpu.virtual", prefix)) return count @@ -510,13 +529,13 @@ func BaremetalUsage(userCred mcclient.IIdentityProvider, rangeObj db.IStandalone func hostUsage( userCred mcclient.IIdentityProvider, prefix string, - rangeObj db.IStandaloneModel, hostTypes []string, + rangeObjs []db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string, enabled, isBaremetal tristate.TriState, ) Usage { count := make(map[string]interface{}) - result := models.HostManager.TotalCount(userCred, rangeObj, "", "", hostTypes, resourceTypes, providers, brands, cloudEnv, enabled, isBaremetal) + result := models.HostManager.TotalCount(userCred, rangeObjs, "", "", hostTypes, resourceTypes, providers, brands, cloudEnv, enabled, isBaremetal) count[prefix] = result.Count count[fmt.Sprintf("%s.memory", prefix)] = result.Memory count[fmt.Sprintf("%s.cpu", prefix)] = result.CPU @@ -526,34 +545,34 @@ func hostUsage( return count } -func GuestNormalUsage(prefix string, scope rbacutils.TRbacScope, cred mcclient.IIdentityProvider, rangeObj db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string) Usage { - return guestUsage(prefix, scope, cred, rangeObj, hostTypes, resourceTypes, providers, brands, cloudEnv, nil, false) +func GuestNormalUsage(prefix string, scope rbacutils.TRbacScope, cred mcclient.IIdentityProvider, rangeObjs []db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string) Usage { + return guestUsage(prefix, scope, cred, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, nil, false) } -func GuestPendingDeleteUsage(prefix string, scope rbacutils.TRbacScope, cred mcclient.IIdentityProvider, rangeObj db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string) Usage { - return guestUsage(prefix, scope, cred, rangeObj, hostTypes, resourceTypes, providers, brands, cloudEnv, nil, true) +func GuestPendingDeleteUsage(prefix string, scope rbacutils.TRbacScope, cred mcclient.IIdentityProvider, rangeObjs []db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string) Usage { + return guestUsage(prefix, scope, cred, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, nil, true) } -func GuestRunningUsage(prefix string, scope rbacutils.TRbacScope, cred mcclient.IIdentityProvider, rangeObj db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string) Usage { - return guestUsage(prefix, scope, cred, rangeObj, hostTypes, resourceTypes, providers, brands, cloudEnv, []string{api.VM_RUNNING}, false) +func GuestRunningUsage(prefix string, scope rbacutils.TRbacScope, cred mcclient.IIdentityProvider, rangeObjs []db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string) Usage { + return guestUsage(prefix, scope, cred, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, []string{api.VM_RUNNING}, false) } -func GuestReadyUsage(prefix string, scope rbacutils.TRbacScope, cred mcclient.IIdentityProvider, rangeObj db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string) Usage { - return guestUsage(prefix, scope, cred, rangeObj, hostTypes, resourceTypes, providers, brands, cloudEnv, []string{api.VM_READY}, false) +func GuestReadyUsage(prefix string, scope rbacutils.TRbacScope, cred mcclient.IIdentityProvider, rangeObjs []db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string) Usage { + return guestUsage(prefix, scope, cred, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, []string{api.VM_READY}, false) } func guestHypervisorsUsage( prefix string, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, - rangeObj db.IStandaloneModel, + rangeObjs []db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string, status, hypervisors []string, pendingDelete bool, ) Usage { // temporarily hide system resources // XXX needs more work later - guest := models.GuestManager.TotalCount(scope, ownerId, rangeObj, status, hypervisors, false, pendingDelete, hostTypes, resourceTypes, providers, brands, cloudEnv) + guest := models.GuestManager.TotalCount(scope, ownerId, rangeObjs, status, hypervisors, false, pendingDelete, hostTypes, resourceTypes, providers, brands, cloudEnv) count := make(map[string]interface{}) count[prefix] = guest.TotalGuestCount count[fmt.Sprintf("%s.cpu", prefix)] = guest.TotalCpuCount @@ -574,26 +593,26 @@ func guestHypervisorsUsage( return count } -func guestUsage(prefix string, scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, rangeObj db.IStandaloneModel, +func guestUsage(prefix string, scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, rangeObjs []db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string, status []string, pendingDelete bool) Usage { hypervisors := sets.NewString(api.HYPERVISORS...) hypervisors.Delete(api.HYPERVISOR_CONTAINER) - return guestHypervisorsUsage(prefix, scope, userCred, rangeObj, hostTypes, resourceTypes, providers, brands, cloudEnv, status, hypervisors.List(), pendingDelete) + return guestHypervisorsUsage(prefix, scope, userCred, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, status, hypervisors.List(), pendingDelete) } -func containerUsage(prefix string, scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, rangeObj db.IStandaloneModel, +func containerUsage(prefix string, scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, rangeObjs []db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string) Usage { hypervisors := []string{api.HYPERVISOR_CONTAINER} - return guestHypervisorsUsage(prefix, scope, userCred, rangeObj, hostTypes, resourceTypes, providers, brands, cloudEnv, nil, hypervisors, false) + return guestHypervisorsUsage(prefix, scope, userCred, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, nil, hypervisors, false) } -func IsolatedDeviceUsage(pref string, rangeObj db.IStandaloneModel, hostType []string, resourceTypes []string, providers []string, brands []string, cloudEnv string) Usage { +func IsolatedDeviceUsage(pref string, rangeObjs []db.IStandaloneModel, hostType []string, resourceTypes []string, providers []string, brands []string, cloudEnv string) Usage { prefix := "isolated_devices" if len(pref) > 0 { prefix = fmt.Sprintf("%s.%s", prefix, pref) } - ret, _ := models.IsolatedDeviceManager.TotalCount(hostType, resourceTypes, providers, brands, cloudEnv, rangeObj) + ret, _ := models.IsolatedDeviceManager.TotalCount(hostType, resourceTypes, providers, brands, cloudEnv, rangeObjs) count := make(map[string]interface{}) count[prefix] = ret.Devices return count @@ -607,9 +626,9 @@ func getKey(projectId, key string) string { } } -func EipUsage(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, rangeObj db.IStandaloneModel, providers []string, brands []string, cloudEnv string) Usage { +func EipUsage(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, rangeObjs []db.IStandaloneModel, providers []string, brands []string, cloudEnv string) Usage { projectId := mcclient.OwnerIdString(ownerId, scope) - eipUsage := models.ElasticipManager.TotalCount(scope, ownerId, rangeObj, providers, brands, cloudEnv) + eipUsage := models.ElasticipManager.TotalCount(scope, ownerId, rangeObjs, providers, brands, cloudEnv) count := make(map[string]interface{}) count[getKey(projectId, "eip")] = eipUsage.Total() count[getKey(projectId, "eip.public_ip")] = eipUsage.PublicIPCount @@ -619,9 +638,9 @@ func EipUsage(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, ra return count } -func BucketUsage(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, rangeObj db.IStandaloneModel, providers []string, brands []string, cloudEnv string) Usage { +func BucketUsage(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, rangeObjs []db.IStandaloneModel, providers []string, brands []string, cloudEnv string) Usage { projectId := mcclient.OwnerIdString(ownerId, scope) - bucketUsage := models.BucketManager.TotalCount(scope, ownerId, rangeObj, providers, brands, cloudEnv) + bucketUsage := models.BucketManager.TotalCount(scope, ownerId, rangeObjs, providers, brands, cloudEnv) count := make(map[string]interface{}) count[getKey(projectId, "buckets")] = bucketUsage.Buckets count[getKey(projectId, "bucket_objects")] = bucketUsage.Objects @@ -629,9 +648,9 @@ func BucketUsage(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, return count } -func SnapshotUsage(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, rangeObj db.IStandaloneModel, providers []string, brands []string, cloudEnv string) Usage { +func SnapshotUsage(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, rangeObjs []db.IStandaloneModel, providers []string, brands []string, cloudEnv string) Usage { projectId := mcclient.OwnerIdString(ownerId, scope) - cnt, _ := models.TotalSnapshotCount(scope, ownerId, rangeObj, providers, brands, cloudEnv) + cnt, _ := models.TotalSnapshotCount(scope, ownerId, rangeObjs, providers, brands, cloudEnv) count := make(map[string]interface{}) count[getKey(projectId, "snapshot")] = cnt return count diff --git a/pkg/image/models/image_guest.go b/pkg/image/models/image_guest.go index ecf9274d89..c53c90e47c 100644 --- a/pkg/image/models/image_guest.go +++ b/pkg/image/models/image_guest.go @@ -39,6 +39,7 @@ import ( "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/logclient" "yunion.io/x/onecloud/pkg/util/rbacutils" + "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" ) type SGuestImageManager struct { @@ -74,8 +75,9 @@ func (manager *SGuestImageManager) ValidateCreateData(ctx context.Context, userC imageNum, _ := data.Int("image_number") pendingUsage := SQuota{Image: int(imageNum)} - if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, rbacutils.ScopeProject, userCred, nil, - &pendingUsage); err != nil { + keys := quotas.OwnerIdQuotaKeys(rbacutils.ScopeProject, ownerId) + pendingUsage.SetKeys(keys) + if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, &pendingUsage); err != nil { return nil, httperrors.NewOutOfQuotaError("%s", err) } @@ -146,7 +148,9 @@ func (gi *SGuestImage) PostCreate(ctx context.Context, userCred mcclient.TokenCr imageNumber, _ := data.Int("image_number") pendingUsage := SQuota{Image: int(imageNumber)} - QuotaManager.CancelPendingUsage(ctx, userCred, rbacutils.ScopeProject, userCred, nil, &pendingUsage, &pendingUsage) + keys := quotas.OwnerIdQuotaKeys(rbacutils.ScopeProject, ownerId) + pendingUsage.SetKeys(keys) + QuotaManager.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage) if !suc { gi.SetStatus(userCred, api.IMAGE_STATUS_KILLED, "create subimage failed") diff --git a/pkg/image/models/images.go b/pkg/image/models/images.go index 033cd51881..4b4793d9d6 100644 --- a/pkg/image/models/images.go +++ b/pkg/image/models/images.go @@ -50,6 +50,8 @@ import ( "yunion.io/x/onecloud/pkg/util/qemuimg" "yunion.io/x/onecloud/pkg/util/rbacutils" "yunion.io/x/onecloud/pkg/util/streamutils" + "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" ) const ( @@ -355,17 +357,25 @@ func (self *SImage) GetExtraDetailsHeaders(ctx context.Context, userCred mcclien } func (manager *SImageManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - _, err := manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) + input := apis.VirtualResourceCreateInput{} + err := data.Unmarshal(&input) + if err != nil { + return nil, httperrors.NewInternalServerError("unmarshal StandaloneResourceCreateInput fail %s", err) + } + input, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input) if err != nil { return nil, err } + data.Update(jsonutils.Marshal(input)) // If this image is the part of guest image (contains "guest_image_id"), // we do not need to check and set pending quota // because that pending quota has been checked and set in SGuestImage.ValidateCreateData if !data.Contains("guest_image_id") { pendingUsage := SQuota{Image: 1} - if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, rbacutils.ScopeProject, userCred, nil, &pendingUsage); err != nil { + keys := quotas.OwnerIdQuotaKeys(rbacutils.ScopeProject, ownerId) + pendingUsage.SetKeys(keys) + if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, &pendingUsage); err != nil { return nil, httperrors.NewOutOfQuotaError("%s", err) } } @@ -493,7 +503,9 @@ func (self *SImage) PostCreate(ctx context.Context, userCred mcclient.TokenCrede // if SImage belong to a guest image, pending quota will not be set. if self.IsGuestImage.IsFalse() { pendingUsage := SQuota{Image: 1} - QuotaManager.CancelPendingUsage(ctx, userCred, rbacutils.ScopeProject, userCred, nil, &pendingUsage, &pendingUsage) + keys := quotas.OwnerIdQuotaKeys(rbacutils.ScopeProject, ownerId) + pendingUsage.SetKeys(keys) + QuotaManager.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage) } if data.Contains("properties") { diff --git a/pkg/image/models/quotas.go b/pkg/image/models/quotas.go index be29103ebc..c86d4f7baa 100644 --- a/pkg/image/models/quotas.go +++ b/pkg/image/models/quotas.go @@ -22,47 +22,94 @@ import ( identityapi "yunion.io/x/onecloud/pkg/apis/identity" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" + commonOptions "yunion.io/x/onecloud/pkg/cloudcommon/options" "yunion.io/x/onecloud/pkg/image/options" - "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/rbacutils" + "yunion.io/x/onecloud/pkg/mcclient/auth" ) type SQuotaManager struct { quotas.SQuotaBaseManager } -var QuotaManager *SQuotaManager -var QuotaUsageManager *SQuotaManager +var ( + ImageQuota SQuota + QuotaManager *SQuotaManager + QuotaUsageManager *SQuotaManager + QuotaPendingUsageManager *SQuotaManager +) func init() { - pendingStore := quotas.NewMemoryQuotaStore() + ImageQuota = SQuota{} + + QuotaPendingUsageManager = &SQuotaManager{ + SQuotaBaseManager: quotas.NewQuotaUsageManager(SQuota{}, "quota_pending_usage_tbl"), + } + QuotaPendingUsageManager.SetVirtualObject(QuotaPendingUsageManager) QuotaUsageManager = &SQuotaManager{ SQuotaBaseManager: quotas.NewQuotaUsageManager(SQuota{}, "quota_usage_tbl"), } + QuotaUsageManager.SetVirtualObject(QuotaUsageManager) QuotaManager = &SQuotaManager{ - SQuotaBaseManager: quotas.NewQuotaBaseManager(SQuota{}, "quota_tbl", pendingStore, QuotaUsageManager), + SQuotaBaseManager: quotas.NewQuotaBaseManager(SQuota{}, "quota_tbl", QuotaPendingUsageManager, QuotaUsageManager, + "image_quota", "image_quotas"), } + QuotaManager.SetVirtualObject(QuotaManager) } type SQuota struct { quotas.SQuotaBase + quotas.SBaseQuotaKeys + Image int } -func (self *SQuota) FetchSystemQuota(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) { - base := 0 - if scope == rbacutils.ScopeDomain { - base = 10 - } else if ownerId.GetProjectDomainId() == identityapi.DEFAULT_DOMAIN_ID && ownerId.GetProjectName() == identityapi.SystemAdminProject { - base = 1 - } - self.Image = options.Options.DefaultImageQuota * base +func (self *SQuota) GetKeys() quotas.IQuotaKeys { + return self.SBaseQuotaKeys } -func (self *SQuota) FetchUsage(ctx context.Context, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, platform []string) error { +func (self *SQuota) SetKeys(keys quotas.IQuotaKeys) { + self.SBaseQuotaKeys = keys.(quotas.SBaseQuotaKeys) +} + +func (self *SQuota) FetchSystemQuota() { + keys := self.SBaseQuotaKeys + base := 0 + switch options.Options.DefaultQuotaValue { + case commonOptions.DefaultQuotaUnlimit: + base = -1 + case commonOptions.DefaultQuotaZero: + base = 0 + if keys.Scope() == rbacutils.ScopeDomain { // domain level quota + base = 10 + } else if keys.DomainId == identityapi.DEFAULT_DOMAIN_ID && keys.ProjectId == auth.AdminCredential().GetProjectId() { + base = 1 + } + case commonOptions.DefaultQuotaDefault: + base = 1 + if keys.Scope() == rbacutils.ScopeDomain { + base = 10 + } + } + defaultValue := func(def int) int { + if base < 0 { + return -1 + } else { + return def * base + } + } + self.Image = defaultValue(options.Options.DefaultImageQuota) +} + +func (self *SQuota) FetchUsage(ctx context.Context) error { + keys := self.SBaseQuotaKeys + + scope := keys.Scope() + ownerId := keys.OwnerId() + count := ImageManager.count(scope, ownerId, "", tristate.None, false) self.Image = int(count["total"].Count) return nil @@ -77,7 +124,7 @@ func (self *SQuota) IsEmpty() bool { func (self *SQuota) Add(quota quotas.IQuota) { squota := quota.(*SQuota) - self.Image = self.Image + squota.Image + self.Image = self.Image + quotas.NonNegative(squota.Image) } func (self *SQuota) Sub(quota quotas.IQuota) { diff --git a/pkg/mcclient/modules/mod_quotas.go b/pkg/mcclient/modules/mod_quotas.go index 0a5cef0394..ee871cf0a1 100644 --- a/pkg/mcclient/modules/mod_quotas.go +++ b/pkg/mcclient/modules/mod_quotas.go @@ -143,15 +143,12 @@ func (this *QuotaManager) DoQuotaSet(s *mcclient.ClientSession, params jsonutils return this.doPost(s, params, url) } -func (this *QuotaManager) DoQuotaCheck(s *mcclient.ClientSession, params jsonutils.JSONObject) (jsonutils.JSONObject, error) { - url := this.getURL(params) - url = fmt.Sprintf("%s/check_quota", url) - return this.doPost(s, params, url) -} - var ( - Quotas QuotaManager - ImageQuotas QuotaManager + Quotas QuotaManager + ProjectQuotas QuotaManager + RegionQuotas QuotaManager + ZoneQuotas QuotaManager + ImageQuotas QuotaManager ) func init() { @@ -160,6 +157,21 @@ func init() { []string{})} registerCompute(&Quotas) + ProjectQuotas = QuotaManager{NewComputeManager("project_quota", "project_quotas", + []string{}, + []string{})} + registerCompute(&ProjectQuotas) + + RegionQuotas = QuotaManager{NewComputeManager("region_quota", "region_quotas", + []string{}, + []string{})} + registerCompute(&RegionQuotas) + + ZoneQuotas = QuotaManager{NewComputeManager("zone_quota", "zone_quotas", + []string{}, + []string{})} + registerCompute(&ZoneQuotas) + ImageQuotas = QuotaManager{NewImageManager("quota", "quotas", []string{}, []string{})} diff --git a/pkg/mcclient/options/disks.go b/pkg/mcclient/options/disks.go index 0ea00a939c..71491ee539 100644 --- a/pkg/mcclient/options/disks.go +++ b/pkg/mcclient/options/disks.go @@ -51,11 +51,11 @@ func (o DiskCreateOptions) Params() (*api.DiskCreateInput, error) { PreferZone: o.Zone, PreferWire: o.Wire, PreferHost: o.Host, - Description: o.Desc, - Name: o.NAME, DiskConfig: config, Hypervisor: o.Hypervisor, } + params.Description = o.Desc + params.Name = o.NAME if o.Storage != "" { params.Storage = o.Storage } diff --git a/pkg/mcclient/options/schedulers.go b/pkg/mcclient/options/schedulers.go index ac9f4e515e..09d840b17f 100644 --- a/pkg/mcclient/options/schedulers.go +++ b/pkg/mcclient/options/schedulers.go @@ -38,6 +38,8 @@ func (o SchedulerTestBaseOptions) data(s *mcclient.ClientSession) (*scheduler.Se data := new(scheduler.ServerConfig) data.ServerConfigs = config + data.Project = o.Project + if o.Mem > 0 { data.Memory = o.Mem } diff --git a/pkg/mcclient/options/servers.go b/pkg/mcclient/options/servers.go index 1b76c89752..c6a578d3c9 100644 --- a/pkg/mcclient/options/servers.go +++ b/pkg/mcclient/options/servers.go @@ -154,7 +154,6 @@ func (o ServerConfigs) Data() (*computeapi.ServerConfigs, error) { PreferBackupHost: o.BackupHost, Hypervisor: o.Hypervisor, ResourceType: o.ResourceType, - Project: o.Project, Backup: o.Backup, Count: o.Count, } @@ -360,10 +359,8 @@ func (opts *ServerCreateOptionalOptions) OptionalParams() (*computeapi.ServerCre Vga: opts.Vga, Vdi: opts.Vdi, Bios: opts.Bios, - Description: opts.Desc, ShutdownBehavior: opts.ShutdownBehavior, AutoStart: opts.AutoStart, - IsSystem: opts.System, Duration: opts.Duration, AutoPrepaidRecycle: opts.AutoPrepaidRecycle, EipBw: opts.EipBw, @@ -375,6 +372,17 @@ func (opts *ServerCreateOptionalOptions) OptionalParams() (*computeapi.ServerCre Secgroups: opts.Secgroups, } + params.Description = opts.Desc + params.IsSystem = &opts.System + + params.Project = opts.Project + params.ProjectId = opts.Project + + if opts.GenerateName { + params.GenerateName = opts.NAME + } else { + params.Name = opts.NAME + } if regutils.MatchSize(opts.MemSpec) { memSize, err := fileutils.GetSizeMb(opts.MemSpec, 'M', 1024) if err != nil { diff --git a/pkg/mcclient/utils/fetchnames.go b/pkg/mcclient/utils/fetchnames.go new file mode 100644 index 0000000000..f331f42f32 --- /dev/null +++ b/pkg/mcclient/utils/fetchnames.go @@ -0,0 +1,91 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "context" + "strings" + "fmt" + + "yunion.io/x/jsonutils" + + identityapi "yunion.io/x/onecloud/pkg/apis/identity" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/cloudcommon/consts" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/cloudcommon/db" +) + +func MapKeys(idMap map[string]string) []string { + keys := make([]string, len(idMap)) + idx := 0 + for k := range idMap { + keys[idx] = k + idx += 1 + } + return keys +} + +func FetchDomainNames(ctx context.Context, domainMap map[string]string) (map[string]string, error) { + s := auth.GetAdminSession(ctx, consts.GetRegion(), "v1") + query := jsonutils.NewDict() + query.Add(jsonutils.NewString(fmt.Sprintf("id.equals(%s)", strings.Join(MapKeys(domainMap), ","))), "filter.0") + query.Add(jsonutils.NewInt(int64(len(domainMap))), "limit") + results, err := modules.Domains.List(s, query) + if err == nil { + for i := range results.Data { + // update cache + domainId, _ := results.Data[i].GetString("id") + domainName, _ := results.Data[i].GetString("name") + db.TenantCacheManager.Save(ctx, domainId, domainName, identityapi.KeystoneDomainRoot, identityapi.KeystoneDomainRoot) + domainMap[domainId] = domainName + } + for k, v := range domainMap { + if len(v) == 0 { + db.TenantCacheManager.Delete(ctx, k) + } + } + return domainMap, nil + } else { + return domainMap, err + } +} + +func FetchTenantNames(ctx context.Context, tenantMap map[string]string) (map[string]string, error) { + s := auth.GetAdminSession(ctx, consts.GetRegion(), "v1") + query := jsonutils.NewDict() + query.Add(jsonutils.NewString(fmt.Sprintf("id.equals(%s)", strings.Join(MapKeys(tenantMap), ","))), "filter.0") + query.Add(jsonutils.NewInt(int64(len(tenantMap))), "limit") + results, err := modules.Projects.List(s, query) + if err == nil { + for i := range results.Data { + // update cache + projId, _ := results.Data[i].GetString("id") + projName, _ := results.Data[i].GetString("name") + projDomainId, _ := results.Data[i].GetString("domain_id") + projDomain, _ := results.Data[i].GetString("project_domain") + db.TenantCacheManager.Save(ctx, projId, projName, projDomainId, projDomain) + tenantMap[projId] = projName + } + for k, v := range tenantMap { + if len(v) == 0 { + db.TenantCacheManager.Delete(ctx, k) + } + } + return tenantMap, nil + } else { + return tenantMap, err + } +} diff --git a/pkg/multicloud/aliyun/region.go b/pkg/multicloud/aliyun/region.go index bb608e8abc..afe99aa3c2 100644 --- a/pkg/multicloud/aliyun/region.go +++ b/pkg/multicloud/aliyun/region.go @@ -206,6 +206,10 @@ func (self *SRegion) GetProvider() string { return CLOUD_PROVIDER_ALIYUN } +func (self *SRegion) GetCloudEnv() string { + return "" +} + func (self *SRegion) GetGeographicInfo() cloudprovider.SGeographicInfo { if info, ok := LatitudeAndLongitude[self.RegionId]; ok { return info diff --git a/pkg/multicloud/aws/region.go b/pkg/multicloud/aws/region.go index c00537cd22..7e5d9cc57b 100644 --- a/pkg/multicloud/aws/region.go +++ b/pkg/multicloud/aws/region.go @@ -51,9 +51,10 @@ var RegionLocations = map[string]string{ "us-east-1": "美国东部(弗吉尼亚北部)", "us-west-1": "美国西部(加利福尼亚北部)", "us-west-2": "美国西部(俄勒冈)", - "ap-south-1": "亚太地区(孟买)", + "ap-east-1": "亚太区域(香港)", + "ap-south-1": "亚太区域(孟买)", + "ap-northeast-3": "亚太区域(大阪-本地)", "ap-northeast-2": "亚太区域(首尔)", - "ap-northeast-3": "亚太区域(大阪)", "ap-southeast-1": "亚太区域(新加坡)", "ap-southeast-2": "亚太区域(悉尼)", "ap-northeast-1": "亚太区域(东京)", @@ -64,8 +65,11 @@ var RegionLocations = map[string]string{ "eu-west-1": "欧洲(爱尔兰)", "eu-west-2": "欧洲(伦敦)", "eu-west-3": "欧洲(巴黎)", + "eu-north-1": "欧洲(斯德哥尔摩)", + "me-south-1": "中东(巴林)", "sa-east-1": "南美洲(圣保罗)", - "us-gov-west-1": "AWS GovCloud(美国)", + "us-gov-west-1": "AWS GovCloud(美国西部)", + "us-gov-east-1": "AWS GovCloud(美国东部)", } const ( @@ -648,6 +652,10 @@ func (self *SRegion) GetProvider() string { return CLOUD_PROVIDER_AWS } +func (self *SRegion) GetCloudEnv() string { + return self.client.accessUrl +} + func (self *SRegion) CreateInstanceSimple(name string, imgId string, cpu int, memGB int, storageType string, dataDiskSizesGB []int, networkId string, publicKey string) (*SInstance, error) { izones, err := self.GetIZones() if err != nil { diff --git a/pkg/multicloud/azure/region.go b/pkg/multicloud/azure/region.go index d73d7c6cc4..f732a6d359 100644 --- a/pkg/multicloud/azure/region.go +++ b/pkg/multicloud/azure/region.go @@ -141,6 +141,10 @@ func (self *SRegion) GetProvider() string { return CLOUD_PROVIDER_AZURE } +func (self *SRegion) GetCloudEnv() string { + return self.client.envName +} + func (self *SRegion) trimGeographicString(geographic string) string { return strings.TrimFunc(geographic, func(r rune) bool { return !((r >= '0' && r <= '9') || r == '.' || r == '-') diff --git a/pkg/multicloud/huawei/huawei.go b/pkg/multicloud/huawei/huawei.go index 7964402b7f..cd2b2aec3f 100644 --- a/pkg/multicloud/huawei/huawei.go +++ b/pkg/multicloud/huawei/huawei.go @@ -56,7 +56,7 @@ type SHuaweiClient struct { providerId string providerName string projectId string // 华为云项目ID. - accessUrl string // 服务区域 ChinaCloud | InternationalCloud + cloudEnv string // 服务区域 ChinaCloud | InternationalCloud accessKey string secret string @@ -73,12 +73,12 @@ type SHuaweiClient struct { // 初次导入Subaccount时，参数account对应cloudaccounts表中的account字段，即accesskey。此时projectID为空， // 只能进行同步子账号、查询region列表等projectId无关的操作。 // todo: 通过accessurl支持国际站。目前暂时未支持国际站 -func NewHuaweiClient(providerId, providerName, accessurl, accessKey, secret, projectId string, debug bool) (*SHuaweiClient, error) { +func NewHuaweiClient(providerId, providerName, cloudEnv, accessKey, secret, projectId string, debug bool) (*SHuaweiClient, error) { client := SHuaweiClient{ providerId: providerId, providerName: providerName, projectId: projectId, - accessUrl: accessurl, + cloudEnv: cloudEnv, accessKey: accessKey, secret: secret, debug: debug, @@ -405,7 +405,7 @@ func (self *SHuaweiClient) GetVersion() string { } func (self *SHuaweiClient) GetAccessEnv() string { - switch self.accessUrl { + switch self.cloudEnv { case HUAWEI_INTERNATIONAL_CLOUDENV: return api.CLOUD_ACCESS_ENV_HUAWEI_GLOBAL case HUAWEI_CHINA_CLOUDENV: diff --git a/pkg/multicloud/huawei/region.go b/pkg/multicloud/huawei/region.go index 434b67d503..9d84e473e7 100644 --- a/pkg/multicloud/huawei/region.go +++ b/pkg/multicloud/huawei/region.go @@ -705,6 +705,10 @@ func (self *SRegion) GetProvider() string { return CLOUD_PROVIDER_HUAWEI } +func (self *SRegion) GetCloudEnv() string { + return self.client.cloudEnv +} + // https://support.huaweicloud.com/api-vpc/zh-cn_topic_0020090615.html // 目前desc字段并没有用到 func (self *SRegion) CreateSecurityGroup(vpcId string, name string, desc string) (*SSecurityGroup, error) { diff --git a/pkg/multicloud/objectstore/objectstore.go b/pkg/multicloud/objectstore/objectstore.go index 869b8444e5..23b5f6dd06 100644 --- a/pkg/multicloud/objectstore/objectstore.go +++ b/pkg/multicloud/objectstore/objectstore.go @@ -134,6 +134,10 @@ func (cli *SObjectStoreClient) GetProvider() string { return api.CLOUD_PROVIDER_GENERICS3 } +func (cli *SObjectStoreClient) GetCloudEnv() string { + return "" +} + ////////////////////////////// IBucketProvider ////////////////////////////// func (cli *SObjectStoreClient) NewBucket(bucket s3cli.BucketInfo) cloudprovider.ICloudBucket { diff --git a/pkg/multicloud/openstack/region.go b/pkg/multicloud/openstack/region.go index 668f01f370..363c454c1f 100644 --- a/pkg/multicloud/openstack/region.go +++ b/pkg/multicloud/openstack/region.go @@ -73,6 +73,10 @@ func (region *SRegion) GetProvider() string { return CLOUD_PROVIDER_OPENSTACK } +func (region *SRegion) GetCloudEnv() string { + return "" +} + func (region *SRegion) GetGeographicInfo() cloudprovider.SGeographicInfo { return cloudprovider.SGeographicInfo{} } diff --git a/pkg/multicloud/qcloud/region.go b/pkg/multicloud/qcloud/region.go index 3d4bee98aa..0944387285 100644 --- a/pkg/multicloud/qcloud/region.go +++ b/pkg/multicloud/qcloud/region.go @@ -256,6 +256,10 @@ func (self *SRegion) GetProvider() string { return CLOUD_PROVIDER_QCLOUD } +func (self *SRegion) GetCloudEnv() string { + return "" +} + func (self *SRegion) CreateIVpc(name string, desc string, cidr string) (cloudprovider.ICloudVpc, error) { params := make(map[string]string) if len(cidr) > 0 { diff --git a/pkg/multicloud/ucloud/latitud_and_longitude.go b/pkg/multicloud/ucloud/latitud_and_longitude.go index 0f394e014a..c06fbf5541 100644 --- a/pkg/multicloud/ucloud/latitud_and_longitude.go +++ b/pkg/multicloud/ucloud/latitud_and_longitude.go @@ -44,4 +44,5 @@ var LatitudeAndLongitude = map[string]cloudprovider.SGeographicInfo{ "uk-london": {Latitude: 51.5073509, Longitude: -0.1277583, City: api.CITY_LONDON, CountryCode: api.COUNTRY_CODE_GB}, "afr-nigeria": {Latitude: 6.5243793, Longitude: 3.3792057, City: api.CITY_LAGOS, CountryCode: api.COUNTRY_CODE_NG}, "vn-sng": {Latitude: 10.8230989, Longitude: 106.6296638, City: api.CITY_HO_CHI_MINH, CountryCode: api.COUNTRY_CODE_VN}, + "cn-qz": {Latitude: 24.9037185, Longitude: 118.5134676, City: api.CITY_QUAN_ZHOU, CountryCode: api.COUNTRY_CODE_CN}, } diff --git a/pkg/multicloud/ucloud/region.go b/pkg/multicloud/ucloud/region.go index e29067026b..bb54536b07 100644 --- a/pkg/multicloud/ucloud/region.go +++ b/pkg/multicloud/ucloud/region.go @@ -451,6 +451,10 @@ func (self *SRegion) GetProvider() string { return CLOUD_PROVIDER_UCLOUD } +func (self *SRegion) GetCloudEnv() string { + return "" +} + func (self *SRegion) DoListAll(action string, params SParams, result interface{}) error { params.Set("Region", self.GetId()) return self.client.DoListAll(action, params, result) diff --git a/pkg/multicloud/zstack/region.go b/pkg/multicloud/zstack/region.go index b97541de05..14547e3fec 100644 --- a/pkg/multicloud/zstack/region.go +++ b/pkg/multicloud/zstack/region.go @@ -19,10 +19,9 @@ import ( "net/url" "strings" - "github.com/pkg/errors" - "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/secrules" api "yunion.io/x/onecloud/pkg/apis/compute" @@ -74,6 +73,10 @@ func (region *SRegion) GetProvider() string { return CLOUD_PROVIDER_ZSTACK } +func (region *SRegion) GetCloudEnv() string { + return "" +} + func (region *SRegion) GetGeographicInfo() cloudprovider.SGeographicInfo { return cloudprovider.SGeographicInfo{} } diff --git a/scripts/advchecks.py b/scripts/advchecks.py index 438d4e5935..07cf9dec72 100755 --- a/scripts/advchecks.py +++ b/scripts/advchecks.py @@ -31,7 +31,7 @@ if __name__ == '__main__': cms = find_lgtms(comments) if len(rvs) == 0: print("No reviwer is assigned, give up check...") - os.exit(-1) + sys.exit(-1) print("Assigned reviwers: %s" % ", ".join(rvs)) print("Lgtm reviwers: %s" % ", ".join(cms)) req = [] diff --git a/vendor/github.com/stretchr/testify/assert/assertion_format.go b/vendor/github.com/stretchr/testify/assert/assertion_format.go index aa1c2b95cd..e0364e9e7f 100644 --- a/vendor/github.com/stretchr/testify/assert/assertion_format.go +++ b/vendor/github.com/stretchr/testify/assert/assertion_format.go @@ -113,6 +113,17 @@ func Errorf(t TestingT, err error, msg string, args ...interface{}) bool { return Error(t, err, append([]interface{}{msg}, args...)...) } +// Eventuallyf asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. +// +// assert.Eventuallyf(t, func() bool { return true; }, time.Second, 10*time.Millisecond, "error message %s", "formatted") +func Eventuallyf(t TestingT, condition func() bool, waitFor time.Duration, tick time.Duration, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Eventually(t, condition, waitFor, tick, append([]interface{}{msg}, args...)...) +} + // Exactlyf asserts that two objects are equal in value and type. // // assert.Exactlyf(t, int32(123, "error message %s", "formatted"), int64(123)) @@ -157,6 +168,31 @@ func FileExistsf(t TestingT, path string, msg string, args ...interface{}) bool return FileExists(t, path, append([]interface{}{msg}, args...)...) } +// Greaterf asserts that the first element is greater than the second +// +// assert.Greaterf(t, 2, 1, "error message %s", "formatted") +// assert.Greaterf(t, float64(2, "error message %s", "formatted"), float64(1)) +// assert.Greaterf(t, "b", "a", "error message %s", "formatted") +func Greaterf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Greater(t, e1, e2, append([]interface{}{msg}, args...)...) +} + +// GreaterOrEqualf asserts that the first element is greater than or equal to the second +// +// assert.GreaterOrEqualf(t, 2, 1, "error message %s", "formatted") +// assert.GreaterOrEqualf(t, 2, 2, "error message %s", "formatted") +// assert.GreaterOrEqualf(t, "b", "a", "error message %s", "formatted") +// assert.GreaterOrEqualf(t, "b", "b", "error message %s", "formatted") +func GreaterOrEqualf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return GreaterOrEqual(t, e1, e2, append([]interface{}{msg}, args...)...) +} + // HTTPBodyContainsf asserts that a specified handler returns a // body that contains a string. // @@ -289,6 +325,14 @@ func JSONEqf(t TestingT, expected string, actual string, msg string, args ...int return JSONEq(t, expected, actual, append([]interface{}{msg}, args...)...) } +// YAMLEqf asserts that two YAML strings are equivalent. +func YAMLEqf(t TestingT, expected string, actual string, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return YAMLEq(t, expected, actual, append([]interface{}{msg}, args...)...) +} + // Lenf asserts that the specified object has specific length. // Lenf also fails if the object has a type that len() not accept. // @@ -300,6 +344,31 @@ func Lenf(t TestingT, object interface{}, length int, msg string, args ...interf return Len(t, object, length, append([]interface{}{msg}, args...)...) } +// Lessf asserts that the first element is less than the second +// +// assert.Lessf(t, 1, 2, "error message %s", "formatted") +// assert.Lessf(t, float64(1, "error message %s", "formatted"), float64(2)) +// assert.Lessf(t, "a", "b", "error message %s", "formatted") +func Lessf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Less(t, e1, e2, append([]interface{}{msg}, args...)...) +} + +// LessOrEqualf asserts that the first element is less than or equal to the second +// +// assert.LessOrEqualf(t, 1, 2, "error message %s", "formatted") +// assert.LessOrEqualf(t, 2, 2, "error message %s", "formatted") +// assert.LessOrEqualf(t, "a", "b", "error message %s", "formatted") +// assert.LessOrEqualf(t, "b", "b", "error message %s", "formatted") +func LessOrEqualf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return LessOrEqual(t, e1, e2, append([]interface{}{msg}, args...)...) +} + // Nilf asserts that the specified object is nil. // // assert.Nilf(t, err, "error message %s", "formatted") @@ -444,6 +513,19 @@ func Regexpf(t TestingT, rx interface{}, str interface{}, msg string, args ...in return Regexp(t, rx, str, append([]interface{}{msg}, args...)...) } +// Samef asserts that two pointers reference the same object. +// +// assert.Samef(t, ptr1, ptr2, "error message %s", "formatted") +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func Samef(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Same(t, expected, actual, append([]interface{}{msg}, args...)...) +} + // Subsetf asserts that the specified list(array, slice...) contains all // elements given in the specified subset(array, slice...). // diff --git a/vendor/github.com/stretchr/testify/assert/assertion_forward.go b/vendor/github.com/stretchr/testify/assert/assertion_forward.go index de39f794e7..26830403a9 100644 --- a/vendor/github.com/stretchr/testify/assert/assertion_forward.go +++ b/vendor/github.com/stretchr/testify/assert/assertion_forward.go @@ -215,6 +215,28 @@ func (a *Assertions) Errorf(err error, msg string, args ...interface{}) bool { return Errorf(a.t, err, msg, args...) } +// Eventually asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. +// +// a.Eventually(func() bool { return true; }, time.Second, 10*time.Millisecond) +func (a *Assertions) Eventually(condition func() bool, waitFor time.Duration, tick time.Duration, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Eventually(a.t, condition, waitFor, tick, msgAndArgs...) +} + +// Eventuallyf asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. +// +// a.Eventuallyf(func() bool { return true; }, time.Second, 10*time.Millisecond, "error message %s", "formatted") +func (a *Assertions) Eventuallyf(condition func() bool, waitFor time.Duration, tick time.Duration, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Eventuallyf(a.t, condition, waitFor, tick, msg, args...) +} + // Exactly asserts that two objects are equal in value and type. // // a.Exactly(int32(123), int64(123)) @@ -303,6 +325,56 @@ func (a *Assertions) FileExistsf(path string, msg string, args ...interface{}) b return FileExistsf(a.t, path, msg, args...) } +// Greater asserts that the first element is greater than the second +// +// a.Greater(2, 1) +// a.Greater(float64(2), float64(1)) +// a.Greater("b", "a") +func (a *Assertions) Greater(e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Greater(a.t, e1, e2, msgAndArgs...) +} + +// GreaterOrEqual asserts that the first element is greater than or equal to the second +// +// a.GreaterOrEqual(2, 1) +// a.GreaterOrEqual(2, 2) +// a.GreaterOrEqual("b", "a") +// a.GreaterOrEqual("b", "b") +func (a *Assertions) GreaterOrEqual(e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return GreaterOrEqual(a.t, e1, e2, msgAndArgs...) +} + +// GreaterOrEqualf asserts that the first element is greater than or equal to the second +// +// a.GreaterOrEqualf(2, 1, "error message %s", "formatted") +// a.GreaterOrEqualf(2, 2, "error message %s", "formatted") +// a.GreaterOrEqualf("b", "a", "error message %s", "formatted") +// a.GreaterOrEqualf("b", "b", "error message %s", "formatted") +func (a *Assertions) GreaterOrEqualf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return GreaterOrEqualf(a.t, e1, e2, msg, args...) +} + +// Greaterf asserts that the first element is greater than the second +// +// a.Greaterf(2, 1, "error message %s", "formatted") +// a.Greaterf(float64(2, "error message %s", "formatted"), float64(1)) +// a.Greaterf("b", "a", "error message %s", "formatted") +func (a *Assertions) Greaterf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Greaterf(a.t, e1, e2, msg, args...) +} + // HTTPBodyContains asserts that a specified handler returns a // body that contains a string. // @@ -567,6 +639,22 @@ func (a *Assertions) JSONEqf(expected string, actual string, msg string, args .. return JSONEqf(a.t, expected, actual, msg, args...) } +// YAMLEq asserts that two YAML strings are equivalent. +func (a *Assertions) YAMLEq(expected string, actual string, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return YAMLEq(a.t, expected, actual, msgAndArgs...) +} + +// YAMLEqf asserts that two YAML strings are equivalent. +func (a *Assertions) YAMLEqf(expected string, actual string, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return YAMLEqf(a.t, expected, actual, msg, args...) +} + // Len asserts that the specified object has specific length. // Len also fails if the object has a type that len() not accept. // @@ -589,6 +677,56 @@ func (a *Assertions) Lenf(object interface{}, length int, msg string, args ...in return Lenf(a.t, object, length, msg, args...) } +// Less asserts that the first element is less than the second +// +// a.Less(1, 2) +// a.Less(float64(1), float64(2)) +// a.Less("a", "b") +func (a *Assertions) Less(e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Less(a.t, e1, e2, msgAndArgs...) +} + +// LessOrEqual asserts that the first element is less than or equal to the second +// +// a.LessOrEqual(1, 2) +// a.LessOrEqual(2, 2) +// a.LessOrEqual("a", "b") +// a.LessOrEqual("b", "b") +func (a *Assertions) LessOrEqual(e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return LessOrEqual(a.t, e1, e2, msgAndArgs...) +} + +// LessOrEqualf asserts that the first element is less than or equal to the second +// +// a.LessOrEqualf(1, 2, "error message %s", "formatted") +// a.LessOrEqualf(2, 2, "error message %s", "formatted") +// a.LessOrEqualf("a", "b", "error message %s", "formatted") +// a.LessOrEqualf("b", "b", "error message %s", "formatted") +func (a *Assertions) LessOrEqualf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return LessOrEqualf(a.t, e1, e2, msg, args...) +} + +// Lessf asserts that the first element is less than the second +// +// a.Lessf(1, 2, "error message %s", "formatted") +// a.Lessf(float64(1, "error message %s", "formatted"), float64(2)) +// a.Lessf("a", "b", "error message %s", "formatted") +func (a *Assertions) Lessf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Lessf(a.t, e1, e2, msg, args...) +} + // Nil asserts that the specified object is nil. // // a.Nil(err) @@ -877,6 +1015,32 @@ func (a *Assertions) Regexpf(rx interface{}, str interface{}, msg string, args . return Regexpf(a.t, rx, str, msg, args...) } +// Same asserts that two pointers reference the same object. +// +// a.Same(ptr1, ptr2) +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func (a *Assertions) Same(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Same(a.t, expected, actual, msgAndArgs...) +} + +// Samef asserts that two pointers reference the same object. +// +// a.Samef(ptr1, ptr2, "error message %s", "formatted") +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func (a *Assertions) Samef(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Samef(a.t, expected, actual, msg, args...) +} + // Subset asserts that the specified list(array, slice...) contains all // elements given in the specified subset(array, slice...). // diff --git a/vendor/github.com/stretchr/testify/assert/assertion_order.go b/vendor/github.com/stretchr/testify/assert/assertion_order.go new file mode 100644 index 0000000000..15a486ca6e --- /dev/null +++ b/vendor/github.com/stretchr/testify/assert/assertion_order.go @@ -0,0 +1,309 @@ +package assert + +import ( + "fmt" + "reflect" +) + +func compare(obj1, obj2 interface{}, kind reflect.Kind) (int, bool) { + switch kind { + case reflect.Int: + { + intobj1 := obj1.(int) + intobj2 := obj2.(int) + if intobj1 > intobj2 { + return -1, true + } + if intobj1 == intobj2 { + return 0, true + } + if intobj1 < intobj2 { + return 1, true + } + } + case reflect.Int8: + { + int8obj1 := obj1.(int8) + int8obj2 := obj2.(int8) + if int8obj1 > int8obj2 { + return -1, true + } + if int8obj1 == int8obj2 { + return 0, true + } + if int8obj1 < int8obj2 { + return 1, true + } + } + case reflect.Int16: + { + int16obj1 := obj1.(int16) + int16obj2 := obj2.(int16) + if int16obj1 > int16obj2 { + return -1, true + } + if int16obj1 == int16obj2 { + return 0, true + } + if int16obj1 < int16obj2 { + return 1, true + } + } + case reflect.Int32: + { + int32obj1 := obj1.(int32) + int32obj2 := obj2.(int32) + if int32obj1 > int32obj2 { + return -1, true + } + if int32obj1 == int32obj2 { + return 0, true + } + if int32obj1 < int32obj2 { + return 1, true + } + } + case reflect.Int64: + { + int64obj1 := obj1.(int64) + int64obj2 := obj2.(int64) + if int64obj1 > int64obj2 { + return -1, true + } + if int64obj1 == int64obj2 { + return 0, true + } + if int64obj1 < int64obj2 { + return 1, true + } + } + case reflect.Uint: + { + uintobj1 := obj1.(uint) + uintobj2 := obj2.(uint) + if uintobj1 > uintobj2 { + return -1, true + } + if uintobj1 == uintobj2 { + return 0, true + } + if uintobj1 < uintobj2 { + return 1, true + } + } + case reflect.Uint8: + { + uint8obj1 := obj1.(uint8) + uint8obj2 := obj2.(uint8) + if uint8obj1 > uint8obj2 { + return -1, true + } + if uint8obj1 == uint8obj2 { + return 0, true + } + if uint8obj1 < uint8obj2 { + return 1, true + } + } + case reflect.Uint16: + { + uint16obj1 := obj1.(uint16) + uint16obj2 := obj2.(uint16) + if uint16obj1 > uint16obj2 { + return -1, true + } + if uint16obj1 == uint16obj2 { + return 0, true + } + if uint16obj1 < uint16obj2 { + return 1, true + } + } + case reflect.Uint32: + { + uint32obj1 := obj1.(uint32) + uint32obj2 := obj2.(uint32) + if uint32obj1 > uint32obj2 { + return -1, true + } + if uint32obj1 == uint32obj2 { + return 0, true + } + if uint32obj1 < uint32obj2 { + return 1, true + } + } + case reflect.Uint64: + { + uint64obj1 := obj1.(uint64) + uint64obj2 := obj2.(uint64) + if uint64obj1 > uint64obj2 { + return -1, true + } + if uint64obj1 == uint64obj2 { + return 0, true + } + if uint64obj1 < uint64obj2 { + return 1, true + } + } + case reflect.Float32: + { + float32obj1 := obj1.(float32) + float32obj2 := obj2.(float32) + if float32obj1 > float32obj2 { + return -1, true + } + if float32obj1 == float32obj2 { + return 0, true + } + if float32obj1 < float32obj2 { + return 1, true + } + } + case reflect.Float64: + { + float64obj1 := obj1.(float64) + float64obj2 := obj2.(float64) + if float64obj1 > float64obj2 { + return -1, true + } + if float64obj1 == float64obj2 { + return 0, true + } + if float64obj1 < float64obj2 { + return 1, true + } + } + case reflect.String: + { + stringobj1 := obj1.(string) + stringobj2 := obj2.(string) + if stringobj1 > stringobj2 { + return -1, true + } + if stringobj1 == stringobj2 { + return 0, true + } + if stringobj1 < stringobj2 { + return 1, true + } + } + } + + return 0, false +} + +// Greater asserts that the first element is greater than the second +// +// assert.Greater(t, 2, 1) +// assert.Greater(t, float64(2), float64(1)) +// assert.Greater(t, "b", "a") +func Greater(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + e1Kind := reflect.ValueOf(e1).Kind() + e2Kind := reflect.ValueOf(e2).Kind() + if e1Kind != e2Kind { + return Fail(t, "Elements should be the same type", msgAndArgs...) + } + + res, isComparable := compare(e1, e2, e1Kind) + if !isComparable { + return Fail(t, fmt.Sprintf("Can not compare type \"%s\"", reflect.TypeOf(e1)), msgAndArgs...) + } + + if res != -1 { + return Fail(t, fmt.Sprintf("\"%v\" is not greater than \"%v\"", e1, e2), msgAndArgs...) + } + + return true +} + +// GreaterOrEqual asserts that the first element is greater than or equal to the second +// +// assert.GreaterOrEqual(t, 2, 1) +// assert.GreaterOrEqual(t, 2, 2) +// assert.GreaterOrEqual(t, "b", "a") +// assert.GreaterOrEqual(t, "b", "b") +func GreaterOrEqual(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + e1Kind := reflect.ValueOf(e1).Kind() + e2Kind := reflect.ValueOf(e2).Kind() + if e1Kind != e2Kind { + return Fail(t, "Elements should be the same type", msgAndArgs...) + } + + res, isComparable := compare(e1, e2, e1Kind) + if !isComparable { + return Fail(t, fmt.Sprintf("Can not compare type \"%s\"", reflect.TypeOf(e1)), msgAndArgs...) + } + + if res != -1 && res != 0 { + return Fail(t, fmt.Sprintf("\"%v\" is not greater than or equal to \"%v\"", e1, e2), msgAndArgs...) + } + + return true +} + +// Less asserts that the first element is less than the second +// +// assert.Less(t, 1, 2) +// assert.Less(t, float64(1), float64(2)) +// assert.Less(t, "a", "b") +func Less(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + e1Kind := reflect.ValueOf(e1).Kind() + e2Kind := reflect.ValueOf(e2).Kind() + if e1Kind != e2Kind { + return Fail(t, "Elements should be the same type", msgAndArgs...) + } + + res, isComparable := compare(e1, e2, e1Kind) + if !isComparable { + return Fail(t, fmt.Sprintf("Can not compare type \"%s\"", reflect.TypeOf(e1)), msgAndArgs...) + } + + if res != 1 { + return Fail(t, fmt.Sprintf("\"%v\" is not less than \"%v\"", e1, e2), msgAndArgs...) + } + + return true +} + +// LessOrEqual asserts that the first element is less than or equal to the second +// +// assert.LessOrEqual(t, 1, 2) +// assert.LessOrEqual(t, 2, 2) +// assert.LessOrEqual(t, "a", "b") +// assert.LessOrEqual(t, "b", "b") +func LessOrEqual(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + e1Kind := reflect.ValueOf(e1).Kind() + e2Kind := reflect.ValueOf(e2).Kind() + if e1Kind != e2Kind { + return Fail(t, "Elements should be the same type", msgAndArgs...) + } + + res, isComparable := compare(e1, e2, e1Kind) + if !isComparable { + return Fail(t, fmt.Sprintf("Can not compare type \"%s\"", reflect.TypeOf(e1)), msgAndArgs...) + } + + if res != 1 && res != 0 { + return Fail(t, fmt.Sprintf("\"%v\" is not less than or equal to \"%v\"", e1, e2), msgAndArgs...) + } + + return true +} diff --git a/vendor/github.com/stretchr/testify/assert/assertions.go b/vendor/github.com/stretchr/testify/assert/assertions.go index 9bd4a80e48..044da8b01f 100644 --- a/vendor/github.com/stretchr/testify/assert/assertions.go +++ b/vendor/github.com/stretchr/testify/assert/assertions.go @@ -18,6 +18,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pmezard/go-difflib/difflib" + yaml "gopkg.in/yaml.v2" ) //go:generate go run ../_codegen/main.go -output-package=assert -template=assertion_format.go.tmpl @@ -350,6 +351,37 @@ func Equal(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) } +// Same asserts that two pointers reference the same object. +// +// assert.Same(t, ptr1, ptr2) +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func Same(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + expectedPtr, actualPtr := reflect.ValueOf(expected), reflect.ValueOf(actual) + if expectedPtr.Kind() != reflect.Ptr || actualPtr.Kind() != reflect.Ptr { + return Fail(t, "Invalid operation: both arguments must be pointers", msgAndArgs...) + } + + expectedType, actualType := reflect.TypeOf(expected), reflect.TypeOf(actual) + if expectedType != actualType { + return Fail(t, fmt.Sprintf("Pointer expected to be of type %v, but was %v", + expectedType, actualType), msgAndArgs...) + } + + if expected != actual { + return Fail(t, fmt.Sprintf("Not same: \n"+ + "expected: %p %#v\n"+ + "actual : %p %#v", expected, expected, actual, actual), msgAndArgs...) + } + + return true +} + // formatUnequalValues takes two values of arbitrary types and returns string // representations appropriate to be presented to the user. // @@ -479,14 +511,14 @@ func isEmpty(object interface{}) bool { // collection types are empty when they have no element case reflect.Array, reflect.Chan, reflect.Map, reflect.Slice: return objValue.Len() == 0 - // pointers are empty if nil or if the value they point to is empty + // pointers are empty if nil or if the value they point to is empty case reflect.Ptr: if objValue.IsNil() { return true } deref := objValue.Elem().Interface() return isEmpty(deref) - // for all other types, compare against the zero value + // for all other types, compare against the zero value default: zero := reflect.Zero(objValue.Type()) return reflect.DeepEqual(object, zero.Interface()) @@ -629,7 +661,7 @@ func NotEqual(t TestingT, expected, actual interface{}, msgAndArgs ...interface{ func includeElement(list interface{}, element interface{}) (ok, found bool) { listValue := reflect.ValueOf(list) - elementValue := reflect.ValueOf(element) + listKind := reflect.TypeOf(list).Kind() defer func() { if e := recover(); e != nil { ok = false @@ -637,11 +669,12 @@ func includeElement(list interface{}, element interface{}) (ok, found bool) { } }() - if reflect.TypeOf(list).Kind() == reflect.String { + if listKind == reflect.String { + elementValue := reflect.ValueOf(element) return true, strings.Contains(listValue.String(), elementValue.String()) } - if reflect.TypeOf(list).Kind() == reflect.Map { + if listKind == reflect.Map { mapKeys := listValue.MapKeys() for i := 0; i < len(mapKeys); i++ { if ObjectsAreEqual(mapKeys[i].Interface(), element) { @@ -1337,6 +1370,24 @@ func JSONEq(t TestingT, expected string, actual string, msgAndArgs ...interface{ return Equal(t, expectedJSONAsInterface, actualJSONAsInterface, msgAndArgs...) } +// YAMLEq asserts that two YAML strings are equivalent. +func YAMLEq(t TestingT, expected string, actual string, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + var expectedYAMLAsInterface, actualYAMLAsInterface interface{} + + if err := yaml.Unmarshal([]byte(expected), &expectedYAMLAsInterface); err != nil { + return Fail(t, fmt.Sprintf("Expected value ('%s') is not valid yaml.\nYAML parsing error: '%s'", expected, err.Error()), msgAndArgs...) + } + + if err := yaml.Unmarshal([]byte(actual), &actualYAMLAsInterface); err != nil { + return Fail(t, fmt.Sprintf("Input ('%s') needs to be valid yaml.\nYAML error: '%s'", actual, err.Error()), msgAndArgs...) + } + + return Equal(t, expectedYAMLAsInterface, actualYAMLAsInterface, msgAndArgs...) +} + func typeAndKind(v interface{}) (reflect.Type, reflect.Kind) { t := reflect.TypeOf(v) k := t.Kind() @@ -1371,8 +1422,8 @@ func diff(expected interface{}, actual interface{}) string { e = spewConfig.Sdump(expected) a = spewConfig.Sdump(actual) } else { - e = expected.(string) - a = actual.(string) + e = reflect.ValueOf(expected).String() + a = reflect.ValueOf(actual).String() } diff, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{ @@ -1414,3 +1465,34 @@ var spewConfig = spew.ConfigState{ type tHelper interface { Helper() } + +// Eventually asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. +// +// assert.Eventually(t, func() bool { return true; }, time.Second, 10*time.Millisecond) +func Eventually(t TestingT, condition func() bool, waitFor time.Duration, tick time.Duration, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + timer := time.NewTimer(waitFor) + ticker := time.NewTicker(tick) + checkPassed := make(chan bool) + defer timer.Stop() + defer ticker.Stop() + defer close(checkPassed) + for { + select { + case <-timer.C: + return Fail(t, "Condition never satisfied", msgAndArgs...) + case result := <-checkPassed: + if result { + return true + } + case <-ticker.C: + go func() { + checkPassed <- condition() + }() + } + } +} diff --git a/vendor/github.com/stretchr/testify/mock/mock.go b/vendor/github.com/stretchr/testify/mock/mock.go index d6694ed78a..b5288af5b7 100644 --- a/vendor/github.com/stretchr/testify/mock/mock.go +++ b/vendor/github.com/stretchr/testify/mock/mock.go @@ -262,17 +262,21 @@ func (m *Mock) On(methodName string, arguments ...interface{}) *Call { // */ func (m *Mock) findExpectedCall(method string, arguments ...interface{}) (int, *Call) { - for i, call := range m.ExpectedCalls { - if call.Method == method && call.Repeatability > -1 { + var expectedCall *Call + for i, call := range m.ExpectedCalls { + if call.Method == method { _, diffCount := call.Arguments.Diff(arguments) if diffCount == 0 { - return i, call + expectedCall = call + if call.Repeatability > -1 { + return i, call + } } - } } - return -1, nil + + return -1, expectedCall } func (m *Mock) findClosestCall(method string, arguments ...interface{}) (*Call, string) { @@ -344,13 +348,17 @@ func (m *Mock) MethodCalled(methodName string, arguments ...interface{}) Argumen found, call := m.findExpectedCall(methodName, arguments...) if found < 0 { + // expected call found but it has already been called with repeatable times + if call != nil { + m.mutex.Unlock() + m.fail("\nassert: mock: The method has been called over %d times.\n\tEither do one more Mock.On(\"%s\").Return(...), or remove extra call.\n\tThis call was unexpected:\n\t\t%s\n\tat: %s", call.totalCalls, methodName, callString(methodName, arguments, true), assert.CallerInfo()) + } // we have to fail here - because we don't know what to do // as the return arguments. This is because: // // a) this is a totally unexpected call to this method, // b) the arguments are not what was expected, or // c) the developer has forgotten to add an accompanying On...Return pair. - closestCall, mismatch := m.findClosestCall(methodName, arguments...) m.mutex.Unlock() diff --git a/vendor/github.com/stretchr/testify/require/require.go b/vendor/github.com/stretchr/testify/require/require.go index 535f293490..c5903f5dbb 100644 --- a/vendor/github.com/stretchr/testify/require/require.go +++ b/vendor/github.com/stretchr/testify/require/require.go @@ -14,23 +14,23 @@ import ( // Condition uses a Comparison to assert a complex condition. func Condition(t TestingT, comp assert.Comparison, msgAndArgs ...interface{}) { - if assert.Condition(t, comp, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Condition(t, comp, msgAndArgs...) { + return + } t.FailNow() } // Conditionf uses a Comparison to assert a complex condition. func Conditionf(t TestingT, comp assert.Comparison, msg string, args ...interface{}) { - if assert.Conditionf(t, comp, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Conditionf(t, comp, msg, args...) { + return + } t.FailNow() } @@ -41,12 +41,12 @@ func Conditionf(t TestingT, comp assert.Comparison, msg string, args ...interfac // assert.Contains(t, ["Hello", "World"], "World") // assert.Contains(t, {"Hello": "World"}, "Hello") func Contains(t TestingT, s interface{}, contains interface{}, msgAndArgs ...interface{}) { - if assert.Contains(t, s, contains, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Contains(t, s, contains, msgAndArgs...) { + return + } t.FailNow() } @@ -57,34 +57,34 @@ func Contains(t TestingT, s interface{}, contains interface{}, msgAndArgs ...int // assert.Containsf(t, ["Hello", "World"], "World", "error message %s", "formatted") // assert.Containsf(t, {"Hello": "World"}, "Hello", "error message %s", "formatted") func Containsf(t TestingT, s interface{}, contains interface{}, msg string, args ...interface{}) { - if assert.Containsf(t, s, contains, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Containsf(t, s, contains, msg, args...) { + return + } t.FailNow() } // DirExists checks whether a directory exists in the given path. It also fails if the path is a file rather a directory or there is an error checking whether it exists. func DirExists(t TestingT, path string, msgAndArgs ...interface{}) { - if assert.DirExists(t, path, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.DirExists(t, path, msgAndArgs...) { + return + } t.FailNow() } // DirExistsf checks whether a directory exists in the given path. It also fails if the path is a file rather a directory or there is an error checking whether it exists. func DirExistsf(t TestingT, path string, msg string, args ...interface{}) { - if assert.DirExistsf(t, path, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.DirExistsf(t, path, msg, args...) { + return + } t.FailNow() } @@ -94,12 +94,12 @@ func DirExistsf(t TestingT, path string, msg string, args ...interface{}) { // // assert.ElementsMatch(t, [1, 3, 2, 3], [1, 3, 3, 2]) func ElementsMatch(t TestingT, listA interface{}, listB interface{}, msgAndArgs ...interface{}) { - if assert.ElementsMatch(t, listA, listB, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.ElementsMatch(t, listA, listB, msgAndArgs...) { + return + } t.FailNow() } @@ -109,12 +109,12 @@ func ElementsMatch(t TestingT, listA interface{}, listB interface{}, msgAndArgs // // assert.ElementsMatchf(t, [1, 3, 2, 3], [1, 3, 3, 2], "error message %s", "formatted") func ElementsMatchf(t TestingT, listA interface{}, listB interface{}, msg string, args ...interface{}) { - if assert.ElementsMatchf(t, listA, listB, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.ElementsMatchf(t, listA, listB, msg, args...) { + return + } t.FailNow() } @@ -123,12 +123,12 @@ func ElementsMatchf(t TestingT, listA interface{}, listB interface{}, msg string // // assert.Empty(t, obj) func Empty(t TestingT, object interface{}, msgAndArgs ...interface{}) { - if assert.Empty(t, object, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Empty(t, object, msgAndArgs...) { + return + } t.FailNow() } @@ -137,12 +137,12 @@ func Empty(t TestingT, object interface{}, msgAndArgs ...interface{}) { // // assert.Emptyf(t, obj, "error message %s", "formatted") func Emptyf(t TestingT, object interface{}, msg string, args ...interface{}) { - if assert.Emptyf(t, object, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Emptyf(t, object, msg, args...) { + return + } t.FailNow() } @@ -154,12 +154,12 @@ func Emptyf(t TestingT, object interface{}, msg string, args ...interface{}) { // referenced values (as opposed to the memory addresses). Function equality // cannot be determined and will always fail. func Equal(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...interface{}) { - if assert.Equal(t, expected, actual, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Equal(t, expected, actual, msgAndArgs...) { + return + } t.FailNow() } @@ -169,12 +169,12 @@ func Equal(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...i // actualObj, err := SomeFunction() // assert.EqualError(t, err, expectedErrorString) func EqualError(t TestingT, theError error, errString string, msgAndArgs ...interface{}) { - if assert.EqualError(t, theError, errString, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.EqualError(t, theError, errString, msgAndArgs...) { + return + } t.FailNow() } @@ -184,12 +184,12 @@ func EqualError(t TestingT, theError error, errString string, msgAndArgs ...inte // actualObj, err := SomeFunction() // assert.EqualErrorf(t, err, expectedErrorString, "error message %s", "formatted") func EqualErrorf(t TestingT, theError error, errString string, msg string, args ...interface{}) { - if assert.EqualErrorf(t, theError, errString, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.EqualErrorf(t, theError, errString, msg, args...) { + return + } t.FailNow() } @@ -198,12 +198,12 @@ func EqualErrorf(t TestingT, theError error, errString string, msg string, args // // assert.EqualValues(t, uint32(123), int32(123)) func EqualValues(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...interface{}) { - if assert.EqualValues(t, expected, actual, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.EqualValues(t, expected, actual, msgAndArgs...) { + return + } t.FailNow() } @@ -212,12 +212,12 @@ func EqualValues(t TestingT, expected interface{}, actual interface{}, msgAndArg // // assert.EqualValuesf(t, uint32(123, "error message %s", "formatted"), int32(123)) func EqualValuesf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) { - if assert.EqualValuesf(t, expected, actual, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.EqualValuesf(t, expected, actual, msg, args...) { + return + } t.FailNow() } @@ -229,12 +229,12 @@ func EqualValuesf(t TestingT, expected interface{}, actual interface{}, msg stri // referenced values (as opposed to the memory addresses). Function equality // cannot be determined and will always fail. func Equalf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) { - if assert.Equalf(t, expected, actual, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Equalf(t, expected, actual, msg, args...) { + return + } t.FailNow() } @@ -245,12 +245,12 @@ func Equalf(t TestingT, expected interface{}, actual interface{}, msg string, ar // assert.Equal(t, expectedError, err) // } func Error(t TestingT, err error, msgAndArgs ...interface{}) { - if assert.Error(t, err, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Error(t, err, msgAndArgs...) { + return + } t.FailNow() } @@ -261,9 +261,37 @@ func Error(t TestingT, err error, msgAndArgs ...interface{}) { // assert.Equal(t, expectedErrorf, err) // } func Errorf(t TestingT, err error, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if assert.Errorf(t, err, msg, args...) { return } + t.FailNow() +} + +// Eventually asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. +// +// assert.Eventually(t, func() bool { return true; }, time.Second, 10*time.Millisecond) +func Eventually(t TestingT, condition func() bool, waitFor time.Duration, tick time.Duration, msgAndArgs ...interface{}) { + if assert.Eventually(t, condition, waitFor, tick, msgAndArgs...) { + return + } + if h, ok := t.(tHelper); ok { + h.Helper() + } + t.FailNow() +} + +// Eventuallyf asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. +// +// assert.Eventuallyf(t, func() bool { return true; }, time.Second, 10*time.Millisecond, "error message %s", "formatted") +func Eventuallyf(t TestingT, condition func() bool, waitFor time.Duration, tick time.Duration, msg string, args ...interface{}) { + if assert.Eventuallyf(t, condition, waitFor, tick, msg, args...) { + return + } if h, ok := t.(tHelper); ok { h.Helper() } @@ -274,12 +302,12 @@ func Errorf(t TestingT, err error, msg string, args ...interface{}) { // // assert.Exactly(t, int32(123), int64(123)) func Exactly(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...interface{}) { - if assert.Exactly(t, expected, actual, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Exactly(t, expected, actual, msgAndArgs...) { + return + } t.FailNow() } @@ -287,56 +315,56 @@ func Exactly(t TestingT, expected interface{}, actual interface{}, msgAndArgs .. // // assert.Exactlyf(t, int32(123, "error message %s", "formatted"), int64(123)) func Exactlyf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) { - if assert.Exactlyf(t, expected, actual, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Exactlyf(t, expected, actual, msg, args...) { + return + } t.FailNow() } // Fail reports a failure through func Fail(t TestingT, failureMessage string, msgAndArgs ...interface{}) { - if assert.Fail(t, failureMessage, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Fail(t, failureMessage, msgAndArgs...) { + return + } t.FailNow() } // FailNow fails test func FailNow(t TestingT, failureMessage string, msgAndArgs ...interface{}) { - if assert.FailNow(t, failureMessage, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.FailNow(t, failureMessage, msgAndArgs...) { + return + } t.FailNow() } // FailNowf fails test func FailNowf(t TestingT, failureMessage string, msg string, args ...interface{}) { - if assert.FailNowf(t, failureMessage, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.FailNowf(t, failureMessage, msg, args...) { + return + } t.FailNow() } // Failf reports a failure through func Failf(t TestingT, failureMessage string, msg string, args ...interface{}) { - if assert.Failf(t, failureMessage, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Failf(t, failureMessage, msg, args...) { + return + } t.FailNow() } @@ -344,12 +372,12 @@ func Failf(t TestingT, failureMessage string, msg string, args ...interface{}) { // // assert.False(t, myBool) func False(t TestingT, value bool, msgAndArgs ...interface{}) { - if assert.False(t, value, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.False(t, value, msgAndArgs...) { + return + } t.FailNow() } @@ -357,34 +385,96 @@ func False(t TestingT, value bool, msgAndArgs ...interface{}) { // // assert.Falsef(t, myBool, "error message %s", "formatted") func Falsef(t TestingT, value bool, msg string, args ...interface{}) { - if assert.Falsef(t, value, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Falsef(t, value, msg, args...) { + return + } t.FailNow() } // FileExists checks whether a file exists in the given path. It also fails if the path points to a directory or there is an error when trying to check the file. func FileExists(t TestingT, path string, msgAndArgs ...interface{}) { - if assert.FileExists(t, path, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.FileExists(t, path, msgAndArgs...) { + return + } t.FailNow() } // FileExistsf checks whether a file exists in the given path. It also fails if the path points to a directory or there is an error when trying to check the file. func FileExistsf(t TestingT, path string, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if assert.FileExistsf(t, path, msg, args...) { return } + t.FailNow() +} + +// Greater asserts that the first element is greater than the second +// +// assert.Greater(t, 2, 1) +// assert.Greater(t, float64(2), float64(1)) +// assert.Greater(t, "b", "a") +func Greater(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) { if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Greater(t, e1, e2, msgAndArgs...) { + return + } + t.FailNow() +} + +// GreaterOrEqual asserts that the first element is greater than or equal to the second +// +// assert.GreaterOrEqual(t, 2, 1) +// assert.GreaterOrEqual(t, 2, 2) +// assert.GreaterOrEqual(t, "b", "a") +// assert.GreaterOrEqual(t, "b", "b") +func GreaterOrEqual(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.GreaterOrEqual(t, e1, e2, msgAndArgs...) { + return + } + t.FailNow() +} + +// GreaterOrEqualf asserts that the first element is greater than or equal to the second +// +// assert.GreaterOrEqualf(t, 2, 1, "error message %s", "formatted") +// assert.GreaterOrEqualf(t, 2, 2, "error message %s", "formatted") +// assert.GreaterOrEqualf(t, "b", "a", "error message %s", "formatted") +// assert.GreaterOrEqualf(t, "b", "b", "error message %s", "formatted") +func GreaterOrEqualf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.GreaterOrEqualf(t, e1, e2, msg, args...) { + return + } + t.FailNow() +} + +// Greaterf asserts that the first element is greater than the second +// +// assert.Greaterf(t, 2, 1, "error message %s", "formatted") +// assert.Greaterf(t, float64(2, "error message %s", "formatted"), float64(1)) +// assert.Greaterf(t, "b", "a", "error message %s", "formatted") +func Greaterf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Greaterf(t, e1, e2, msg, args...) { + return + } t.FailNow() } @@ -395,12 +485,12 @@ func FileExistsf(t TestingT, path string, msg string, args ...interface{}) { // // Returns whether the assertion was successful (true) or not (false). func HTTPBodyContains(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) { - if assert.HTTPBodyContains(t, handler, method, url, values, str, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.HTTPBodyContains(t, handler, method, url, values, str, msgAndArgs...) { + return + } t.FailNow() } @@ -411,12 +501,12 @@ func HTTPBodyContains(t TestingT, handler http.HandlerFunc, method string, url s // // Returns whether the assertion was successful (true) or not (false). func HTTPBodyContainsf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) { - if assert.HTTPBodyContainsf(t, handler, method, url, values, str, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.HTTPBodyContainsf(t, handler, method, url, values, str, msg, args...) { + return + } t.FailNow() } @@ -427,12 +517,12 @@ func HTTPBodyContainsf(t TestingT, handler http.HandlerFunc, method string, url // // Returns whether the assertion was successful (true) or not (false). func HTTPBodyNotContains(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) { - if assert.HTTPBodyNotContains(t, handler, method, url, values, str, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.HTTPBodyNotContains(t, handler, method, url, values, str, msgAndArgs...) { + return + } t.FailNow() } @@ -443,12 +533,12 @@ func HTTPBodyNotContains(t TestingT, handler http.HandlerFunc, method string, ur // // Returns whether the assertion was successful (true) or not (false). func HTTPBodyNotContainsf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) { - if assert.HTTPBodyNotContainsf(t, handler, method, url, values, str, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.HTTPBodyNotContainsf(t, handler, method, url, values, str, msg, args...) { + return + } t.FailNow() } @@ -458,12 +548,12 @@ func HTTPBodyNotContainsf(t TestingT, handler http.HandlerFunc, method string, u // // Returns whether the assertion was successful (true) or not (false). func HTTPError(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) { - if assert.HTTPError(t, handler, method, url, values, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.HTTPError(t, handler, method, url, values, msgAndArgs...) { + return + } t.FailNow() } @@ -473,12 +563,12 @@ func HTTPError(t TestingT, handler http.HandlerFunc, method string, url string, // // Returns whether the assertion was successful (true, "error message %s", "formatted") or not (false). func HTTPErrorf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) { - if assert.HTTPErrorf(t, handler, method, url, values, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.HTTPErrorf(t, handler, method, url, values, msg, args...) { + return + } t.FailNow() } @@ -488,12 +578,12 @@ func HTTPErrorf(t TestingT, handler http.HandlerFunc, method string, url string, // // Returns whether the assertion was successful (true) or not (false). func HTTPRedirect(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) { - if assert.HTTPRedirect(t, handler, method, url, values, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.HTTPRedirect(t, handler, method, url, values, msgAndArgs...) { + return + } t.FailNow() } @@ -503,12 +593,12 @@ func HTTPRedirect(t TestingT, handler http.HandlerFunc, method string, url strin // // Returns whether the assertion was successful (true, "error message %s", "formatted") or not (false). func HTTPRedirectf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) { - if assert.HTTPRedirectf(t, handler, method, url, values, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.HTTPRedirectf(t, handler, method, url, values, msg, args...) { + return + } t.FailNow() } @@ -518,12 +608,12 @@ func HTTPRedirectf(t TestingT, handler http.HandlerFunc, method string, url stri // // Returns whether the assertion was successful (true) or not (false). func HTTPSuccess(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) { - if assert.HTTPSuccess(t, handler, method, url, values, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.HTTPSuccess(t, handler, method, url, values, msgAndArgs...) { + return + } t.FailNow() } @@ -533,12 +623,12 @@ func HTTPSuccess(t TestingT, handler http.HandlerFunc, method string, url string // // Returns whether the assertion was successful (true) or not (false). func HTTPSuccessf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) { - if assert.HTTPSuccessf(t, handler, method, url, values, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.HTTPSuccessf(t, handler, method, url, values, msg, args...) { + return + } t.FailNow() } @@ -546,12 +636,12 @@ func HTTPSuccessf(t TestingT, handler http.HandlerFunc, method string, url strin // // assert.Implements(t, (*MyInterface)(nil), new(MyObject)) func Implements(t TestingT, interfaceObject interface{}, object interface{}, msgAndArgs ...interface{}) { - if assert.Implements(t, interfaceObject, object, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Implements(t, interfaceObject, object, msgAndArgs...) { + return + } t.FailNow() } @@ -559,12 +649,12 @@ func Implements(t TestingT, interfaceObject interface{}, object interface{}, msg // // assert.Implementsf(t, (*MyInterface, "error message %s", "formatted")(nil), new(MyObject)) func Implementsf(t TestingT, interfaceObject interface{}, object interface{}, msg string, args ...interface{}) { - if assert.Implementsf(t, interfaceObject, object, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Implementsf(t, interfaceObject, object, msg, args...) { + return + } t.FailNow() } @@ -572,56 +662,56 @@ func Implementsf(t TestingT, interfaceObject interface{}, object interface{}, ms // // assert.InDelta(t, math.Pi, (22 / 7.0), 0.01) func InDelta(t TestingT, expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) { - if assert.InDelta(t, expected, actual, delta, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.InDelta(t, expected, actual, delta, msgAndArgs...) { + return + } t.FailNow() } // InDeltaMapValues is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys. func InDeltaMapValues(t TestingT, expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) { - if assert.InDeltaMapValues(t, expected, actual, delta, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.InDeltaMapValues(t, expected, actual, delta, msgAndArgs...) { + return + } t.FailNow() } // InDeltaMapValuesf is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys. func InDeltaMapValuesf(t TestingT, expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) { - if assert.InDeltaMapValuesf(t, expected, actual, delta, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.InDeltaMapValuesf(t, expected, actual, delta, msg, args...) { + return + } t.FailNow() } // InDeltaSlice is the same as InDelta, except it compares two slices. func InDeltaSlice(t TestingT, expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) { - if assert.InDeltaSlice(t, expected, actual, delta, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.InDeltaSlice(t, expected, actual, delta, msgAndArgs...) { + return + } t.FailNow() } // InDeltaSlicef is the same as InDelta, except it compares two slices. func InDeltaSlicef(t TestingT, expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) { - if assert.InDeltaSlicef(t, expected, actual, delta, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.InDeltaSlicef(t, expected, actual, delta, msg, args...) { + return + } t.FailNow() } @@ -629,78 +719,78 @@ func InDeltaSlicef(t TestingT, expected interface{}, actual interface{}, delta f // // assert.InDeltaf(t, math.Pi, (22 / 7.0, "error message %s", "formatted"), 0.01) func InDeltaf(t TestingT, expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) { - if assert.InDeltaf(t, expected, actual, delta, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.InDeltaf(t, expected, actual, delta, msg, args...) { + return + } t.FailNow() } // InEpsilon asserts that expected and actual have a relative error less than epsilon func InEpsilon(t TestingT, expected interface{}, actual interface{}, epsilon float64, msgAndArgs ...interface{}) { - if assert.InEpsilon(t, expected, actual, epsilon, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.InEpsilon(t, expected, actual, epsilon, msgAndArgs...) { + return + } t.FailNow() } // InEpsilonSlice is the same as InEpsilon, except it compares each value from two slices. func InEpsilonSlice(t TestingT, expected interface{}, actual interface{}, epsilon float64, msgAndArgs ...interface{}) { - if assert.InEpsilonSlice(t, expected, actual, epsilon, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.InEpsilonSlice(t, expected, actual, epsilon, msgAndArgs...) { + return + } t.FailNow() } // InEpsilonSlicef is the same as InEpsilon, except it compares each value from two slices. func InEpsilonSlicef(t TestingT, expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) { - if assert.InEpsilonSlicef(t, expected, actual, epsilon, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.InEpsilonSlicef(t, expected, actual, epsilon, msg, args...) { + return + } t.FailNow() } // InEpsilonf asserts that expected and actual have a relative error less than epsilon func InEpsilonf(t TestingT, expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) { - if assert.InEpsilonf(t, expected, actual, epsilon, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.InEpsilonf(t, expected, actual, epsilon, msg, args...) { + return + } t.FailNow() } // IsType asserts that the specified objects are of the same type. func IsType(t TestingT, expectedType interface{}, object interface{}, msgAndArgs ...interface{}) { - if assert.IsType(t, expectedType, object, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.IsType(t, expectedType, object, msgAndArgs...) { + return + } t.FailNow() } // IsTypef asserts that the specified objects are of the same type. func IsTypef(t TestingT, expectedType interface{}, object interface{}, msg string, args ...interface{}) { - if assert.IsTypef(t, expectedType, object, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.IsTypef(t, expectedType, object, msg, args...) { + return + } t.FailNow() } @@ -708,12 +798,12 @@ func IsTypef(t TestingT, expectedType interface{}, object interface{}, msg strin // // assert.JSONEq(t, `{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`) func JSONEq(t TestingT, expected string, actual string, msgAndArgs ...interface{}) { - if assert.JSONEq(t, expected, actual, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.JSONEq(t, expected, actual, msgAndArgs...) { + return + } t.FailNow() } @@ -721,12 +811,34 @@ func JSONEq(t TestingT, expected string, actual string, msgAndArgs ...interface{ // // assert.JSONEqf(t, `{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`, "error message %s", "formatted") func JSONEqf(t TestingT, expected string, actual string, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if assert.JSONEqf(t, expected, actual, msg, args...) { return } + t.FailNow() +} + +// YAMLEq asserts that two YAML strings are equivalent. +func YAMLEq(t TestingT, expected string, actual string, msgAndArgs ...interface{}) { if h, ok := t.(tHelper); ok { h.Helper() } + if assert.YAMLEq(t, expected, actual, msgAndArgs...) { + return + } + t.FailNow() +} + +// YAMLEqf asserts that two YAML strings are equivalent. +func YAMLEqf(t TestingT, expected string, actual string, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.YAMLEqf(t, expected, actual, msg, args...) { + return + } t.FailNow() } @@ -735,12 +847,12 @@ func JSONEqf(t TestingT, expected string, actual string, msg string, args ...int // // assert.Len(t, mySlice, 3) func Len(t TestingT, object interface{}, length int, msgAndArgs ...interface{}) { - if assert.Len(t, object, length, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Len(t, object, length, msgAndArgs...) { + return + } t.FailNow() } @@ -749,12 +861,74 @@ func Len(t TestingT, object interface{}, length int, msgAndArgs ...interface{}) // // assert.Lenf(t, mySlice, 3, "error message %s", "formatted") func Lenf(t TestingT, object interface{}, length int, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if assert.Lenf(t, object, length, msg, args...) { return } + t.FailNow() +} + +// Less asserts that the first element is less than the second +// +// assert.Less(t, 1, 2) +// assert.Less(t, float64(1), float64(2)) +// assert.Less(t, "a", "b") +func Less(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) { if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Less(t, e1, e2, msgAndArgs...) { + return + } + t.FailNow() +} + +// LessOrEqual asserts that the first element is less than or equal to the second +// +// assert.LessOrEqual(t, 1, 2) +// assert.LessOrEqual(t, 2, 2) +// assert.LessOrEqual(t, "a", "b") +// assert.LessOrEqual(t, "b", "b") +func LessOrEqual(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.LessOrEqual(t, e1, e2, msgAndArgs...) { + return + } + t.FailNow() +} + +// LessOrEqualf asserts that the first element is less than or equal to the second +// +// assert.LessOrEqualf(t, 1, 2, "error message %s", "formatted") +// assert.LessOrEqualf(t, 2, 2, "error message %s", "formatted") +// assert.LessOrEqualf(t, "a", "b", "error message %s", "formatted") +// assert.LessOrEqualf(t, "b", "b", "error message %s", "formatted") +func LessOrEqualf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.LessOrEqualf(t, e1, e2, msg, args...) { + return + } + t.FailNow() +} + +// Lessf asserts that the first element is less than the second +// +// assert.Lessf(t, 1, 2, "error message %s", "formatted") +// assert.Lessf(t, float64(1, "error message %s", "formatted"), float64(2)) +// assert.Lessf(t, "a", "b", "error message %s", "formatted") +func Lessf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Lessf(t, e1, e2, msg, args...) { + return + } t.FailNow() } @@ -762,12 +936,12 @@ func Lenf(t TestingT, object interface{}, length int, msg string, args ...interf // // assert.Nil(t, err) func Nil(t TestingT, object interface{}, msgAndArgs ...interface{}) { - if assert.Nil(t, object, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Nil(t, object, msgAndArgs...) { + return + } t.FailNow() } @@ -775,12 +949,12 @@ func Nil(t TestingT, object interface{}, msgAndArgs ...interface{}) { // // assert.Nilf(t, err, "error message %s", "formatted") func Nilf(t TestingT, object interface{}, msg string, args ...interface{}) { - if assert.Nilf(t, object, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Nilf(t, object, msg, args...) { + return + } t.FailNow() } @@ -791,12 +965,12 @@ func Nilf(t TestingT, object interface{}, msg string, args ...interface{}) { // assert.Equal(t, expectedObj, actualObj) // } func NoError(t TestingT, err error, msgAndArgs ...interface{}) { - if assert.NoError(t, err, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.NoError(t, err, msgAndArgs...) { + return + } t.FailNow() } @@ -807,12 +981,12 @@ func NoError(t TestingT, err error, msgAndArgs ...interface{}) { // assert.Equal(t, expectedObj, actualObj) // } func NoErrorf(t TestingT, err error, msg string, args ...interface{}) { - if assert.NoErrorf(t, err, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.NoErrorf(t, err, msg, args...) { + return + } t.FailNow() } @@ -823,12 +997,12 @@ func NoErrorf(t TestingT, err error, msg string, args ...interface{}) { // assert.NotContains(t, ["Hello", "World"], "Earth") // assert.NotContains(t, {"Hello": "World"}, "Earth") func NotContains(t TestingT, s interface{}, contains interface{}, msgAndArgs ...interface{}) { - if assert.NotContains(t, s, contains, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.NotContains(t, s, contains, msgAndArgs...) { + return + } t.FailNow() } @@ -839,12 +1013,12 @@ func NotContains(t TestingT, s interface{}, contains interface{}, msgAndArgs ... // assert.NotContainsf(t, ["Hello", "World"], "Earth", "error message %s", "formatted") // assert.NotContainsf(t, {"Hello": "World"}, "Earth", "error message %s", "formatted") func NotContainsf(t TestingT, s interface{}, contains interface{}, msg string, args ...interface{}) { - if assert.NotContainsf(t, s, contains, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.NotContainsf(t, s, contains, msg, args...) { + return + } t.FailNow() } @@ -855,12 +1029,12 @@ func NotContainsf(t TestingT, s interface{}, contains interface{}, msg string, a // assert.Equal(t, "two", obj[1]) // } func NotEmpty(t TestingT, object interface{}, msgAndArgs ...interface{}) { - if assert.NotEmpty(t, object, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.NotEmpty(t, object, msgAndArgs...) { + return + } t.FailNow() } @@ -871,12 +1045,12 @@ func NotEmpty(t TestingT, object interface{}, msgAndArgs ...interface{}) { // assert.Equal(t, "two", obj[1]) // } func NotEmptyf(t TestingT, object interface{}, msg string, args ...interface{}) { - if assert.NotEmptyf(t, object, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.NotEmptyf(t, object, msg, args...) { + return + } t.FailNow() } @@ -887,12 +1061,12 @@ func NotEmptyf(t TestingT, object interface{}, msg string, args ...interface{}) // Pointer variable equality is determined based on the equality of the // referenced values (as opposed to the memory addresses). func NotEqual(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...interface{}) { - if assert.NotEqual(t, expected, actual, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.NotEqual(t, expected, actual, msgAndArgs...) { + return + } t.FailNow() } @@ -903,12 +1077,12 @@ func NotEqual(t TestingT, expected interface{}, actual interface{}, msgAndArgs . // Pointer variable equality is determined based on the equality of the // referenced values (as opposed to the memory addresses). func NotEqualf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) { - if assert.NotEqualf(t, expected, actual, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.NotEqualf(t, expected, actual, msg, args...) { + return + } t.FailNow() } @@ -916,12 +1090,12 @@ func NotEqualf(t TestingT, expected interface{}, actual interface{}, msg string, // // assert.NotNil(t, err) func NotNil(t TestingT, object interface{}, msgAndArgs ...interface{}) { - if assert.NotNil(t, object, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.NotNil(t, object, msgAndArgs...) { + return + } t.FailNow() } @@ -929,12 +1103,12 @@ func NotNil(t TestingT, object interface{}, msgAndArgs ...interface{}) { // // assert.NotNilf(t, err, "error message %s", "formatted") func NotNilf(t TestingT, object interface{}, msg string, args ...interface{}) { - if assert.NotNilf(t, object, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.NotNilf(t, object, msg, args...) { + return + } t.FailNow() } @@ -942,12 +1116,12 @@ func NotNilf(t TestingT, object interface{}, msg string, args ...interface{}) { // // assert.NotPanics(t, func(){ RemainCalm() }) func NotPanics(t TestingT, f assert.PanicTestFunc, msgAndArgs ...interface{}) { - if assert.NotPanics(t, f, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.NotPanics(t, f, msgAndArgs...) { + return + } t.FailNow() } @@ -955,12 +1129,12 @@ func NotPanics(t TestingT, f assert.PanicTestFunc, msgAndArgs ...interface{}) { // // assert.NotPanicsf(t, func(){ RemainCalm() }, "error message %s", "formatted") func NotPanicsf(t TestingT, f assert.PanicTestFunc, msg string, args ...interface{}) { - if assert.NotPanicsf(t, f, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.NotPanicsf(t, f, msg, args...) { + return + } t.FailNow() } @@ -969,12 +1143,12 @@ func NotPanicsf(t TestingT, f assert.PanicTestFunc, msg string, args ...interfac // assert.NotRegexp(t, regexp.MustCompile("starts"), "it's starting") // assert.NotRegexp(t, "^start", "it's not starting") func NotRegexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interface{}) { - if assert.NotRegexp(t, rx, str, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.NotRegexp(t, rx, str, msgAndArgs...) { + return + } t.FailNow() } @@ -983,12 +1157,12 @@ func NotRegexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interf // assert.NotRegexpf(t, regexp.MustCompile("starts", "error message %s", "formatted"), "it's starting") // assert.NotRegexpf(t, "^start", "it's not starting", "error message %s", "formatted") func NotRegexpf(t TestingT, rx interface{}, str interface{}, msg string, args ...interface{}) { - if assert.NotRegexpf(t, rx, str, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.NotRegexpf(t, rx, str, msg, args...) { + return + } t.FailNow() } @@ -997,12 +1171,12 @@ func NotRegexpf(t TestingT, rx interface{}, str interface{}, msg string, args .. // // assert.NotSubset(t, [1, 3, 4], [1, 2], "But [1, 3, 4] does not contain [1, 2]") func NotSubset(t TestingT, list interface{}, subset interface{}, msgAndArgs ...interface{}) { - if assert.NotSubset(t, list, subset, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.NotSubset(t, list, subset, msgAndArgs...) { + return + } t.FailNow() } @@ -1011,34 +1185,34 @@ func NotSubset(t TestingT, list interface{}, subset interface{}, msgAndArgs ...i // // assert.NotSubsetf(t, [1, 3, 4], [1, 2], "But [1, 3, 4] does not contain [1, 2]", "error message %s", "formatted") func NotSubsetf(t TestingT, list interface{}, subset interface{}, msg string, args ...interface{}) { - if assert.NotSubsetf(t, list, subset, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.NotSubsetf(t, list, subset, msg, args...) { + return + } t.FailNow() } // NotZero asserts that i is not the zero value for its type. func NotZero(t TestingT, i interface{}, msgAndArgs ...interface{}) { - if assert.NotZero(t, i, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.NotZero(t, i, msgAndArgs...) { + return + } t.FailNow() } // NotZerof asserts that i is not the zero value for its type. func NotZerof(t TestingT, i interface{}, msg string, args ...interface{}) { - if assert.NotZerof(t, i, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.NotZerof(t, i, msg, args...) { + return + } t.FailNow() } @@ -1046,12 +1220,12 @@ func NotZerof(t TestingT, i interface{}, msg string, args ...interface{}) { // // assert.Panics(t, func(){ GoCrazy() }) func Panics(t TestingT, f assert.PanicTestFunc, msgAndArgs ...interface{}) { - if assert.Panics(t, f, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Panics(t, f, msgAndArgs...) { + return + } t.FailNow() } @@ -1060,12 +1234,12 @@ func Panics(t TestingT, f assert.PanicTestFunc, msgAndArgs ...interface{}) { // // assert.PanicsWithValue(t, "crazy error", func(){ GoCrazy() }) func PanicsWithValue(t TestingT, expected interface{}, f assert.PanicTestFunc, msgAndArgs ...interface{}) { - if assert.PanicsWithValue(t, expected, f, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.PanicsWithValue(t, expected, f, msgAndArgs...) { + return + } t.FailNow() } @@ -1074,12 +1248,12 @@ func PanicsWithValue(t TestingT, expected interface{}, f assert.PanicTestFunc, m // // assert.PanicsWithValuef(t, "crazy error", func(){ GoCrazy() }, "error message %s", "formatted") func PanicsWithValuef(t TestingT, expected interface{}, f assert.PanicTestFunc, msg string, args ...interface{}) { - if assert.PanicsWithValuef(t, expected, f, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.PanicsWithValuef(t, expected, f, msg, args...) { + return + } t.FailNow() } @@ -1087,12 +1261,12 @@ func PanicsWithValuef(t TestingT, expected interface{}, f assert.PanicTestFunc, // // assert.Panicsf(t, func(){ GoCrazy() }, "error message %s", "formatted") func Panicsf(t TestingT, f assert.PanicTestFunc, msg string, args ...interface{}) { - if assert.Panicsf(t, f, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Panicsf(t, f, msg, args...) { + return + } t.FailNow() } @@ -1101,12 +1275,12 @@ func Panicsf(t TestingT, f assert.PanicTestFunc, msg string, args ...interface{} // assert.Regexp(t, regexp.MustCompile("start"), "it's starting") // assert.Regexp(t, "start...$", "it's not starting") func Regexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interface{}) { - if assert.Regexp(t, rx, str, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Regexp(t, rx, str, msgAndArgs...) { + return + } t.FailNow() } @@ -1115,12 +1289,44 @@ func Regexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interface // assert.Regexpf(t, regexp.MustCompile("start", "error message %s", "formatted"), "it's starting") // assert.Regexpf(t, "start...$", "it's not starting", "error message %s", "formatted") func Regexpf(t TestingT, rx interface{}, str interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if assert.Regexpf(t, rx, str, msg, args...) { return } + t.FailNow() +} + +// Same asserts that two pointers reference the same object. +// +// assert.Same(t, ptr1, ptr2) +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func Same(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...interface{}) { if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Same(t, expected, actual, msgAndArgs...) { + return + } + t.FailNow() +} + +// Samef asserts that two pointers reference the same object. +// +// assert.Samef(t, ptr1, ptr2, "error message %s", "formatted") +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func Samef(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Samef(t, expected, actual, msg, args...) { + return + } t.FailNow() } @@ -1129,12 +1335,12 @@ func Regexpf(t TestingT, rx interface{}, str interface{}, msg string, args ...in // // assert.Subset(t, [1, 2, 3], [1, 2], "But [1, 2, 3] does contain [1, 2]") func Subset(t TestingT, list interface{}, subset interface{}, msgAndArgs ...interface{}) { - if assert.Subset(t, list, subset, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Subset(t, list, subset, msgAndArgs...) { + return + } t.FailNow() } @@ -1143,12 +1349,12 @@ func Subset(t TestingT, list interface{}, subset interface{}, msgAndArgs ...inte // // assert.Subsetf(t, [1, 2, 3], [1, 2], "But [1, 2, 3] does contain [1, 2]", "error message %s", "formatted") func Subsetf(t TestingT, list interface{}, subset interface{}, msg string, args ...interface{}) { - if assert.Subsetf(t, list, subset, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Subsetf(t, list, subset, msg, args...) { + return + } t.FailNow() } @@ -1156,12 +1362,12 @@ func Subsetf(t TestingT, list interface{}, subset interface{}, msg string, args // // assert.True(t, myBool) func True(t TestingT, value bool, msgAndArgs ...interface{}) { - if assert.True(t, value, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.True(t, value, msgAndArgs...) { + return + } t.FailNow() } @@ -1169,12 +1375,12 @@ func True(t TestingT, value bool, msgAndArgs ...interface{}) { // // assert.Truef(t, myBool, "error message %s", "formatted") func Truef(t TestingT, value bool, msg string, args ...interface{}) { - if assert.Truef(t, value, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Truef(t, value, msg, args...) { + return + } t.FailNow() } @@ -1182,12 +1388,12 @@ func Truef(t TestingT, value bool, msg string, args ...interface{}) { // // assert.WithinDuration(t, time.Now(), time.Now(), 10*time.Second) func WithinDuration(t TestingT, expected time.Time, actual time.Time, delta time.Duration, msgAndArgs ...interface{}) { - if assert.WithinDuration(t, expected, actual, delta, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.WithinDuration(t, expected, actual, delta, msgAndArgs...) { + return + } t.FailNow() } @@ -1195,33 +1401,33 @@ func WithinDuration(t TestingT, expected time.Time, actual time.Time, delta time // // assert.WithinDurationf(t, time.Now(), time.Now(), 10*time.Second, "error message %s", "formatted") func WithinDurationf(t TestingT, expected time.Time, actual time.Time, delta time.Duration, msg string, args ...interface{}) { - if assert.WithinDurationf(t, expected, actual, delta, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.WithinDurationf(t, expected, actual, delta, msg, args...) { + return + } t.FailNow() } // Zero asserts that i is the zero value for its type. func Zero(t TestingT, i interface{}, msgAndArgs ...interface{}) { - if assert.Zero(t, i, msgAndArgs...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Zero(t, i, msgAndArgs...) { + return + } t.FailNow() } // Zerof asserts that i is the zero value for its type. func Zerof(t TestingT, i interface{}, msg string, args ...interface{}) { - if assert.Zerof(t, i, msg, args...) { - return - } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.Zerof(t, i, msg, args...) { + return + } t.FailNow() } diff --git a/vendor/github.com/stretchr/testify/require/require.go.tmpl b/vendor/github.com/stretchr/testify/require/require.go.tmpl index 6ffc751b5e..55e42ddebd 100644 --- a/vendor/github.com/stretchr/testify/require/require.go.tmpl +++ b/vendor/github.com/stretchr/testify/require/require.go.tmpl @@ -1,6 +1,6 @@ {{.Comment}} func {{.DocInfo.Name}}(t TestingT, {{.Params}}) { - if assert.{{.DocInfo.Name}}(t, {{.ForwardedParams}}) { return } if h, ok := t.(tHelper); ok { h.Helper() } + if assert.{{.DocInfo.Name}}(t, {{.ForwardedParams}}) { return } t.FailNow() } diff --git a/vendor/github.com/stretchr/testify/require/require_forward.go b/vendor/github.com/stretchr/testify/require/require_forward.go index 9fe41dbdc0..804fae035b 100644 --- a/vendor/github.com/stretchr/testify/require/require_forward.go +++ b/vendor/github.com/stretchr/testify/require/require_forward.go @@ -216,6 +216,28 @@ func (a *Assertions) Errorf(err error, msg string, args ...interface{}) { Errorf(a.t, err, msg, args...) } +// Eventually asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. +// +// a.Eventually(func() bool { return true; }, time.Second, 10*time.Millisecond) +func (a *Assertions) Eventually(condition func() bool, waitFor time.Duration, tick time.Duration, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Eventually(a.t, condition, waitFor, tick, msgAndArgs...) +} + +// Eventuallyf asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. +// +// a.Eventuallyf(func() bool { return true; }, time.Second, 10*time.Millisecond, "error message %s", "formatted") +func (a *Assertions) Eventuallyf(condition func() bool, waitFor time.Duration, tick time.Duration, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Eventuallyf(a.t, condition, waitFor, tick, msg, args...) +} + // Exactly asserts that two objects are equal in value and type. // // a.Exactly(int32(123), int64(123)) @@ -304,6 +326,56 @@ func (a *Assertions) FileExistsf(path string, msg string, args ...interface{}) { FileExistsf(a.t, path, msg, args...) } +// Greater asserts that the first element is greater than the second +// +// a.Greater(2, 1) +// a.Greater(float64(2), float64(1)) +// a.Greater("b", "a") +func (a *Assertions) Greater(e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Greater(a.t, e1, e2, msgAndArgs...) +} + +// GreaterOrEqual asserts that the first element is greater than or equal to the second +// +// a.GreaterOrEqual(2, 1) +// a.GreaterOrEqual(2, 2) +// a.GreaterOrEqual("b", "a") +// a.GreaterOrEqual("b", "b") +func (a *Assertions) GreaterOrEqual(e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + GreaterOrEqual(a.t, e1, e2, msgAndArgs...) +} + +// GreaterOrEqualf asserts that the first element is greater than or equal to the second +// +// a.GreaterOrEqualf(2, 1, "error message %s", "formatted") +// a.GreaterOrEqualf(2, 2, "error message %s", "formatted") +// a.GreaterOrEqualf("b", "a", "error message %s", "formatted") +// a.GreaterOrEqualf("b", "b", "error message %s", "formatted") +func (a *Assertions) GreaterOrEqualf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + GreaterOrEqualf(a.t, e1, e2, msg, args...) +} + +// Greaterf asserts that the first element is greater than the second +// +// a.Greaterf(2, 1, "error message %s", "formatted") +// a.Greaterf(float64(2, "error message %s", "formatted"), float64(1)) +// a.Greaterf("b", "a", "error message %s", "formatted") +func (a *Assertions) Greaterf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Greaterf(a.t, e1, e2, msg, args...) +} + // HTTPBodyContains asserts that a specified handler returns a // body that contains a string. // @@ -568,6 +640,22 @@ func (a *Assertions) JSONEqf(expected string, actual string, msg string, args .. JSONEqf(a.t, expected, actual, msg, args...) } +// YAMLEq asserts that two YAML strings are equivalent. +func (a *Assertions) YAMLEq(expected string, actual string, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + YAMLEq(a.t, expected, actual, msgAndArgs...) +} + +// YAMLEqf asserts that two YAML strings are equivalent. +func (a *Assertions) YAMLEqf(expected string, actual string, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + YAMLEqf(a.t, expected, actual, msg, args...) +} + // Len asserts that the specified object has specific length. // Len also fails if the object has a type that len() not accept. // @@ -590,6 +678,56 @@ func (a *Assertions) Lenf(object interface{}, length int, msg string, args ...in Lenf(a.t, object, length, msg, args...) } +// Less asserts that the first element is less than the second +// +// a.Less(1, 2) +// a.Less(float64(1), float64(2)) +// a.Less("a", "b") +func (a *Assertions) Less(e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Less(a.t, e1, e2, msgAndArgs...) +} + +// LessOrEqual asserts that the first element is less than or equal to the second +// +// a.LessOrEqual(1, 2) +// a.LessOrEqual(2, 2) +// a.LessOrEqual("a", "b") +// a.LessOrEqual("b", "b") +func (a *Assertions) LessOrEqual(e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + LessOrEqual(a.t, e1, e2, msgAndArgs...) +} + +// LessOrEqualf asserts that the first element is less than or equal to the second +// +// a.LessOrEqualf(1, 2, "error message %s", "formatted") +// a.LessOrEqualf(2, 2, "error message %s", "formatted") +// a.LessOrEqualf("a", "b", "error message %s", "formatted") +// a.LessOrEqualf("b", "b", "error message %s", "formatted") +func (a *Assertions) LessOrEqualf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + LessOrEqualf(a.t, e1, e2, msg, args...) +} + +// Lessf asserts that the first element is less than the second +// +// a.Lessf(1, 2, "error message %s", "formatted") +// a.Lessf(float64(1, "error message %s", "formatted"), float64(2)) +// a.Lessf("a", "b", "error message %s", "formatted") +func (a *Assertions) Lessf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Lessf(a.t, e1, e2, msg, args...) +} + // Nil asserts that the specified object is nil. // // a.Nil(err) @@ -878,6 +1016,32 @@ func (a *Assertions) Regexpf(rx interface{}, str interface{}, msg string, args . Regexpf(a.t, rx, str, msg, args...) } +// Same asserts that two pointers reference the same object. +// +// a.Same(ptr1, ptr2) +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func (a *Assertions) Same(expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Same(a.t, expected, actual, msgAndArgs...) +} + +// Samef asserts that two pointers reference the same object. +// +// a.Samef(ptr1, ptr2, "error message %s", "formatted") +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func (a *Assertions) Samef(expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Samef(a.t, expected, actual, msg, args...) +} + // Subset asserts that the specified list(array, slice...) contains all // elements given in the specified subset(array, slice...). // diff --git a/vendor/github.com/stretchr/testify/suite/suite.go b/vendor/github.com/stretchr/testify/suite/suite.go index 5cea8f8c75..d708d7d753 100644 --- a/vendor/github.com/stretchr/testify/suite/suite.go +++ b/vendor/github.com/stretchr/testify/suite/suite.go @@ -6,6 +6,7 @@ import ( "os" "reflect" "regexp" + "runtime/debug" "testing" "github.com/stretchr/testify/assert" @@ -58,7 +59,7 @@ func (suite *Suite) Assert() *assert.Assertions { func failOnPanic(t *testing.T) { r := recover() if r != nil { - t.Errorf("test panicked: %v", r) + t.Errorf("test panicked: %v\n%s", r, debug.Stack()) t.FailNow() } } @@ -82,15 +83,8 @@ func Run(t *testing.T, suite TestingSuite) { suite.SetT(t) defer failOnPanic(t) - if setupAllSuite, ok := suite.(SetupAllSuite); ok { - setupAllSuite.SetupSuite() - } - defer func() { - if tearDownAllSuite, ok := suite.(TearDownAllSuite); ok { - tearDownAllSuite.TearDownSuite() - } - }() - + suiteSetupDone := false + methodFinder := reflect.TypeOf(suite) tests := []testing.InternalTest{} for index := 0; index < methodFinder.NumMethod(); index++ { @@ -100,34 +94,46 @@ func Run(t *testing.T, suite TestingSuite) { fmt.Fprintf(os.Stderr, "testify: invalid regexp for -m: %s\n", err) os.Exit(1) } - if ok { - test := testing.InternalTest{ - Name: method.Name, - F: func(t *testing.T) { - parentT := suite.T() - suite.SetT(t) - defer failOnPanic(t) - - if setupTestSuite, ok := suite.(SetupTestSuite); ok { - setupTestSuite.SetupTest() - } - if beforeTestSuite, ok := suite.(BeforeTest); ok { - beforeTestSuite.BeforeTest(methodFinder.Elem().Name(), method.Name) - } - defer func() { - if afterTestSuite, ok := suite.(AfterTest); ok { - afterTestSuite.AfterTest(methodFinder.Elem().Name(), method.Name) - } - if tearDownTestSuite, ok := suite.(TearDownTestSuite); ok { - tearDownTestSuite.TearDownTest() - } - suite.SetT(parentT) - }() - method.Func.Call([]reflect.Value{reflect.ValueOf(suite)}) - }, - } - tests = append(tests, test) + if !ok { + continue } + if !suiteSetupDone { + if setupAllSuite, ok := suite.(SetupAllSuite); ok { + setupAllSuite.SetupSuite() + } + defer func() { + if tearDownAllSuite, ok := suite.(TearDownAllSuite); ok { + tearDownAllSuite.TearDownSuite() + } + }() + suiteSetupDone = true + } + test := testing.InternalTest{ + Name: method.Name, + F: func(t *testing.T) { + parentT := suite.T() + suite.SetT(t) + defer failOnPanic(t) + + if setupTestSuite, ok := suite.(SetupTestSuite); ok { + setupTestSuite.SetupTest() + } + if beforeTestSuite, ok := suite.(BeforeTest); ok { + beforeTestSuite.BeforeTest(methodFinder.Elem().Name(), method.Name) + } + defer func() { + if afterTestSuite, ok := suite.(AfterTest); ok { + afterTestSuite.AfterTest(methodFinder.Elem().Name(), method.Name) + } + if tearDownTestSuite, ok := suite.(TearDownTestSuite); ok { + tearDownTestSuite.TearDownTest() + } + suite.SetT(parentT) + }() + method.Func.Call([]reflect.Value{reflect.ValueOf(suite)}) + }, + } + tests = append(tests, test) } runTests(t, tests) } diff --git a/vendor/modules.txt b/vendor/modules.txt index 2f16c968d4..15dfdd0e11 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -477,7 +477,7 @@ github.com/spaolacci/murmur3 github.com/spf13/pflag # github.com/stretchr/objx v0.1.1 github.com/stretchr/objx -# github.com/stretchr/testify v1.3.0 +# github.com/stretchr/testify v1.4.0 github.com/stretchr/testify/assert github.com/stretchr/testify/mock github.com/stretchr/testify/require @@ -839,7 +839,7 @@ yunion.io/x/pkg/util/workqueue yunion.io/x/pkg/utils # yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e yunion.io/x/s3cli -# yunion.io/x/sqlchemy v0.0.0-20191122085525-2d3bfdb3f51c +# yunion.io/x/sqlchemy v0.0.0-20191206101534-32ae27ae30cb yunion.io/x/sqlchemy # yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3 yunion.io/x/structarg diff --git a/vendor/yunion.io/x/jsonutils/access.go b/vendor/yunion.io/x/jsonutils/access.go index 389a621dc9..f612945f68 100644 --- a/vendor/yunion.io/x/jsonutils/access.go +++ b/vendor/yunion.io/x/jsonutils/access.go @@ -380,6 +380,13 @@ func (this *JSONFloat) Float(keys ...string) (float64, error) { return this.data, nil } +func (this *JSONInt) Float(keys ...string) (float64, error) { + if len(keys) > 0 { + return 0.0, ErrOutOfKeyRange // fmt.Errorf("Out of key range: %s", keys) + } + return float64(this.data), nil +} + func (this *JSONString) Float(keys ...string) (float64, error) { if len(keys) > 0 { return 0.0, ErrOutOfKeyRange // fmt.Errorf("Out of key range: %s", keys) diff --git a/vendor/yunion.io/x/sqlchemy/inc.go b/vendor/yunion.io/x/sqlchemy/inc.go new file mode 100644 index 0000000000..04f701bae3 --- /dev/null +++ b/vendor/yunion.io/x/sqlchemy/inc.go @@ -0,0 +1,156 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlchemy + +import ( + "bytes" + "database/sql" + "fmt" + "reflect" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/gotypes" + "yunion.io/x/pkg/util/reflectutils" +) + +func (t *STableSpec) Increment(diff interface{}, target interface{}) error { + return t.incrementInternal(diff, "+", target) +} + +func (t *STableSpec) Decrement(diff interface{}, target interface{}) error { + return t.incrementInternal(diff, "-", target) +} + +func (t *STableSpec) incrementInternal(diff interface{}, opcode string, target interface{}) error { + if target == nil { + if reflect.ValueOf(diff).Kind() != reflect.Ptr { + return errors.Wrap(ErrNeedsPointer, "Incremental input must be a Pointer") + } + } else { + if reflect.ValueOf(target).Kind() != reflect.Ptr { + return errors.Wrap(ErrNeedsPointer, "Incremental update target must be a Pointer") + } + } + + dataValue := reflect.Indirect(reflect.ValueOf(diff)) + fields := reflectutils.FetchStructFieldValueSet(dataValue) + + primaries := make(map[string]interface{}) + vars := make([]interface{}, 0) + versionFields := make([]string, 0) + updatedFields := make([]string, 0) + incFields := make([]string, 0) + + for _, c := range t.columns { + k := c.Name() + v, _ := fields.GetInterface(k) + if c.IsPrimary() { + if !gotypes.IsNil(v) && !c.IsZero(v) { + primaries[k] = v + } else if c.IsText() { + primaries[k] = "" + } else { + return ErrEmptyPrimaryKey + } + continue + } + dtc, ok := c.(*SDateTimeColumn) + if ok && dtc.IsUpdatedAt { + updatedFields = append(updatedFields, k) + continue + } + nc, ok := c.(*SIntegerColumn) + if ok && nc.IsAutoVersion { + versionFields = append(versionFields, k) + continue + } + if c.IsNumeric() && !c.IsZero(v) { + incFields = append(incFields, k) + vars = append(vars, v) + continue + } + } + + if len(vars) == 0 { + return ErrNoDataToUpdate + } + if len(primaries) == 0 { + return ErrEmptyPrimaryKey + } + + var buf bytes.Buffer + buf.WriteString(fmt.Sprintf("UPDATE `%s` SET ", t.name)) + first := true + for _, k := range incFields { + if first { + first = false + } else { + buf.WriteString(", ") + } + buf.WriteString(fmt.Sprintf("`%s` = `%s` %s ?", k, k, opcode)) + } + for _, versionField := range versionFields { + buf.WriteString(fmt.Sprintf(", `%s` = `%s` + 1", versionField, versionField)) + } + for _, updatedField := range updatedFields { + buf.WriteString(fmt.Sprintf(", `%s` = UTC_TIMESTAMP()", updatedField)) + } + + buf.WriteString(" WHERE ") + first = true + for k, v := range primaries { + if first { + first = false + } else { + buf.WriteString(" AND ") + } + buf.WriteString(fmt.Sprintf("`%s` = ?", k)) + vars = append(vars, v) + } + + if DEBUG_SQLCHEMY { + log.Infof("Update: %s %s", buf.String(), vars) + } + + results, err := _db.Exec(buf.String(), vars...) + if err != nil { + return errors.Wrapf(err, "_db.Exec %s %#v", buf.String(), vars) + } + aCnt, err := results.RowsAffected() + if err != nil { + return errors.Wrap(err, "results.RowsAffected") + } + if aCnt != 1 { + if aCnt == 0 { + return sql.ErrNoRows + } else { + return errors.Wrapf(ErrUnexpectRowCount, "affected rows %d != 1", aCnt) + } + } + q := t.Query() + for k, v := range primaries { + q = q.Equals(k, v) + } + if target != nil { + err = q.First(target) + } else { + err = q.First(diff) + } + if err != nil { + return errors.Wrap(err, "query after update failed") + } + return nil +}