Files
cloudpods/pkg/multicloud/objectstore/objectstore.go
2021-12-07 19:22:11 +08:00

597 lines
17 KiB
Go

// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package objectstore
import (
"net/url"
"os"
"strings"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/util/secrules"
"yunion.io/x/s3cli"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudcommon/object"
"yunion.io/x/onecloud/pkg/cloudprovider"
"yunion.io/x/onecloud/pkg/multicloud"
"yunion.io/x/onecloud/pkg/util/httputils"
)
type ObjectStoreClientConfig struct {
cpcfg cloudprovider.ProviderConfig
endpoint string
accessKey string
accessSecret string
debug bool
}
func NewObjectStoreClientConfig(endpoint, accessKey, accessSecret string) *ObjectStoreClientConfig {
cfg := &ObjectStoreClientConfig{
endpoint: endpoint,
accessKey: accessKey,
accessSecret: accessSecret,
}
return cfg
}
func (cfg *ObjectStoreClientConfig) CloudproviderConfig(cpcfg cloudprovider.ProviderConfig) *ObjectStoreClientConfig {
cfg.cpcfg = cpcfg
return cfg
}
func (cfg *ObjectStoreClientConfig) Debug(debug bool) *ObjectStoreClientConfig {
cfg.debug = debug
return cfg
}
func (cfg *ObjectStoreClientConfig) GetCloudproviderConfig() cloudprovider.ProviderConfig {
return cfg.cpcfg
}
func (cfg *ObjectStoreClientConfig) GetEndpoint() string {
return cfg.endpoint
}
func (cfg *ObjectStoreClientConfig) GetAccessKey() string {
return cfg.accessKey
}
func (cfg *ObjectStoreClientConfig) GetAccessSecret() string {
return cfg.accessSecret
}
func (cfg *ObjectStoreClientConfig) GetDebug() bool {
return cfg.debug
}
type SObjectStoreClient struct {
object.SObject
*ObjectStoreClientConfig
cloudprovider.SFakeOnPremiseRegion
multicloud.SRegion
ownerId string
ownerName string
iBuckets []cloudprovider.ICloudBucket
client *s3cli.Client
}
func NewObjectStoreClient(cfg *ObjectStoreClientConfig) (*SObjectStoreClient, error) {
return NewObjectStoreClientAndFetch(cfg, true)
}
func NewObjectStoreClientAndFetch(cfg *ObjectStoreClientConfig, doFetch bool) (*SObjectStoreClient, error) {
client := SObjectStoreClient{
ObjectStoreClientConfig: cfg,
}
parts, err := url.Parse(cfg.endpoint)
if err != nil {
return nil, errors.Wrap(err, "url.Parse endpoint")
}
useSsl := false
if parts.Scheme == "https" {
useSsl = true
}
cli, err := s3cli.New(
parts.Host,
client.accessKey,
client.accessSecret,
useSsl,
client.debug,
)
if err != nil {
return nil, errors.Wrap(err, "minio.New")
}
tr := httputils.GetTransport(true)
tr.Proxy = cfg.cpcfg.ProxyFunc
cli.SetCustomTransport(tr)
client.client = cli
client.SetVirtualObject(&client)
if client.debug {
cli.TraceOn(os.Stderr)
}
if doFetch {
err = client.FetchBuckets()
if err != nil {
return nil, errors.Wrap(err, "fetchBuckets")
}
}
return &client, nil
}
func (cli *SObjectStoreClient) GetSubAccounts() ([]cloudprovider.SSubAccount, error) {
subAccount := cloudprovider.SSubAccount{
Account: cli.accessKey,
Name: cli.cpcfg.Name,
HealthStatus: api.CLOUD_PROVIDER_HEALTH_NORMAL,
}
return []cloudprovider.SSubAccount{subAccount}, nil
}
func (cli *SObjectStoreClient) GetAccountId() string {
return cli.ownerId
}
func (cli *SObjectStoreClient) GetIRegion() cloudprovider.ICloudRegion {
return cli.GetVirtualObject().(cloudprovider.ICloudRegion)
}
func (cli *SObjectStoreClient) GetIBucketProvider() IBucketProvider {
return cli.GetVirtualObject().(IBucketProvider)
}
func (cli *SObjectStoreClient) GetI18n() cloudprovider.SModelI18nTable {
table := cloudprovider.SModelI18nTable{}
table["name"] = cloudprovider.NewSModelI18nEntry(cli.GetName()).CN(cli.GetName())
return table
}
func (cli *SObjectStoreClient) GetVersion() string {
return ""
}
func (cli *SObjectStoreClient) About() jsonutils.JSONObject {
about := jsonutils.NewDict()
return about
}
func (cli *SObjectStoreClient) GetProvider() string {
return api.CLOUD_PROVIDER_GENERICS3
}
func (cli *SObjectStoreClient) GetCloudEnv() string {
return ""
}
////////////////////////////// IBucketProvider //////////////////////////////
func (cli *SObjectStoreClient) NewBucket(bucket s3cli.BucketInfo) cloudprovider.ICloudBucket {
return &SBucket{
client: cli.GetIBucketProvider(),
Name: bucket.Name,
CreatedAt: bucket.CreationDate,
}
}
func (cli *SObjectStoreClient) GetEndpoint() string {
return cli.endpoint
}
func (cli *SObjectStoreClient) S3Client() *s3cli.Client {
return cli.client
}
func (cli *SObjectStoreClient) GetClientRC() map[string]string {
return map[string]string{
"S3_ACCESS_KEY": cli.accessKey,
"S3_SECRET": cli.accessSecret,
"S3_ACCESS_URL": cli.endpoint,
"S3_BACKEND": api.CLOUD_PROVIDER_GENERICS3,
}
}
///////////////////////////////// fake impletementations //////////////////////
func (cli *SObjectStoreClient) GetIZones() ([]cloudprovider.ICloudZone, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetIVpcs() ([]cloudprovider.ICloudVpc, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetIEips() ([]cloudprovider.ICloudEIP, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetIVpcById(id string) (cloudprovider.ICloudVpc, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetIZoneById(id string) (cloudprovider.ICloudZone, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetIEipById(id string) (cloudprovider.ICloudEIP, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetIVMById(id string) (cloudprovider.ICloudVM, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetIDiskById(id string) (cloudprovider.ICloudDisk, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) SyncSecurityGroup(secgroupId string, vpcId string, name string, desc string, rules []secrules.SecurityRule) (string, error) {
return "", cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) CreateIVpc(opts *cloudprovider.VpcCreateOptions) (cloudprovider.ICloudVpc, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) CreateEIP(eip *cloudprovider.SEip) (cloudprovider.ICloudEIP, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetISnapshotById(snapshotId string) (cloudprovider.ICloudSnapshot, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) CreateSnapshotPolicy(*cloudprovider.SnapshotPolicyInput) (string, error) {
return "", cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) DeleteSnapshotPolicy(string) error {
return cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) ApplySnapshotPolicyToDisks(snapshotPolicyId string, diskId string) error {
return cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) CancelSnapshotPolicyToDisks(snapshotPolicyId string, diskId string) error {
return cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetISnapshotPolicies() ([]cloudprovider.ICloudSnapshotPolicy, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetISnapshotPolicyById(snapshotPolicyId string) (cloudprovider.ICloudSnapshotPolicy, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetIHosts() ([]cloudprovider.ICloudHost, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetIHostById(id string) (cloudprovider.ICloudHost, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetIStorages() ([]cloudprovider.ICloudStorage, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetIStoragecaches() ([]cloudprovider.ICloudStoragecache, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetIStoragecacheById(id string) (cloudprovider.ICloudStoragecache, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetILoadBalancers() ([]cloudprovider.ICloudLoadbalancer, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetILoadBalancerAcls() ([]cloudprovider.ICloudLoadbalancerAcl, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetILoadBalancerCertificates() ([]cloudprovider.ICloudLoadbalancerCertificate, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetILoadBalancerById(loadbalancerId string) (cloudprovider.ICloudLoadbalancer, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetILoadBalancerAclById(aclId string) (cloudprovider.ICloudLoadbalancerAcl, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetILoadBalancerCertificateById(certId string) (cloudprovider.ICloudLoadbalancerCertificate, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) CreateILoadBalancer(loadbalancer *cloudprovider.SLoadbalancer) (cloudprovider.ICloudLoadbalancer, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) CreateILoadBalancerAcl(acl *cloudprovider.SLoadbalancerAccessControlList) (cloudprovider.ICloudLoadbalancerAcl, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) CreateILoadBalancerCertificate(cert *cloudprovider.SLoadbalancerCertificate) (cloudprovider.ICloudLoadbalancerCertificate, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) GetISkus() ([]cloudprovider.ICloudSku, error) {
return nil, cloudprovider.ErrNotSupported
}
func (cli *SObjectStoreClient) CreateISku(opts *cloudprovider.SServerSkuCreateOption) (cloudprovider.ICloudSku, error) {
return nil, cloudprovider.ErrNotSupported
}
////////////////////////////////// S3 API ///////////////////////////////////
func (cli *SObjectStoreClient) GetIBuckets() ([]cloudprovider.ICloudBucket, error) {
return cli.getIBuckets()
}
func (self *SObjectStoreClient) invalidateIBuckets() {
self.iBuckets = nil
}
func (self *SObjectStoreClient) getIBuckets() ([]cloudprovider.ICloudBucket, error) {
if self.iBuckets == nil {
err := self.FetchBuckets()
if err != nil {
return nil, errors.Wrap(err, "fetchBuckets")
}
}
return self.iBuckets, nil
}
func (cli *SObjectStoreClient) FetchBuckets() error {
result, err := cli.client.ListBuckets()
if err != nil {
return errors.Wrap(err, "client.ListBuckets")
}
cli.ownerId = result.Owner.ID
cli.ownerName = result.Owner.DisplayName
buckets := result.Buckets.Bucket
cli.iBuckets = make([]cloudprovider.ICloudBucket, len(buckets))
for i := range buckets {
b := cli.GetIBucketProvider().NewBucket(buckets[i])
cli.iBuckets[i] = b
}
return nil
}
func (cli *SObjectStoreClient) CreateIBucket(name string, storageClass string, acl string) error {
err := cli.client.MakeBucket(name, "")
if err != nil {
return errors.Wrap(err, "MakeBucket")
}
cli.invalidateIBuckets()
return nil
}
func minioErrCode(err error) int {
if srvErr, ok := err.(s3cli.ErrorResponse); ok {
return srvErr.StatusCode
}
if srvErr, ok := err.(*s3cli.ErrorResponse); ok {
return srvErr.StatusCode
}
return -1
}
func (cli *SObjectStoreClient) DeleteIBucket(name string) error {
err := cli.client.RemoveBucket(name)
if err != nil {
if minioErrCode(err) == 404 {
return nil
}
return errors.Wrap(err, "RemoveBucket")
}
cli.invalidateIBuckets()
return nil
}
func (cli *SObjectStoreClient) GetIBucketLocation(name string) (string, error) {
info, err := cli.client.GetBucketLocation(name)
if err != nil {
return "", errors.Wrap(err, "GetBucketLocation")
}
return info, nil
}
func (cli *SObjectStoreClient) GetIBucketWebsite(name string) (string, error) {
info, err := cli.client.GetBucketWebsite(name)
if err != nil {
return "", errors.Wrap(err, "GetBucketWebsite")
}
return info, nil
}
func (cli *SObjectStoreClient) GetIBucketReferer(name string) (string, error) {
info, err := cli.client.GetBucketReferer(name)
if err != nil {
return "", errors.Wrap(err, "GetBucketReferer")
}
return info, nil
}
func (cli *SObjectStoreClient) GetIBucketCors(name string) (string, error) {
info, err := cli.client.GetBucketCors(name)
if err != nil {
return "", errors.Wrap(err, "GetBucketCors")
}
return info, nil
}
func (cli *SObjectStoreClient) GetIBucketLogging(name string) (*s3cli.BucketLoggingStatus, error) {
info, err := cli.client.GetBucketLogging(name)
if err != nil {
return nil, errors.Wrap(err, "GetBucketLogging")
}
return info, nil
}
func (cli *SObjectStoreClient) SetIBucketLogging(name string, target string, targetPrefix string, email string) error {
conf := s3cli.BucketLoggingStatus{}
if len(target) > 0 {
conf.LoggingEnabled.TargetBucket = target
conf.LoggingEnabled.TargetPrefix = targetPrefix
conf.LoggingEnabled.TargetGrants.Grant = []s3cli.Grant{
{
Grantee: s3cli.Grantee{
Type: s3cli.GRANTEE_TYPE_EMAIL,
EmailAddress: email,
},
Permission: s3cli.PERMISSION_FULL_CONTROL,
},
}
}
err := cli.client.SetBucketLogging(name, conf)
if err != nil {
return errors.Wrap(err, "SetBucketLogging")
}
return nil
}
func (cli *SObjectStoreClient) GetIBucketInfo(name string) (string, error) {
info, err := cli.client.GetBucketInfo(name)
if err != nil {
return "", errors.Wrap(err, "GetBucketInfo")
}
return info, nil
}
func (cli *SObjectStoreClient) GetIBucketAcl(name string) (cloudprovider.TBucketACLType, error) {
acl, err := cli.client.GetBucketAcl(name)
if err != nil {
return "", errors.Wrap(err, "GetBucketAcl")
}
return cloudprovider.TBucketACLType(acl.GetCannedACL()), nil
}
func (cli *SObjectStoreClient) SetIBucketAcl(name string, cannedAcl cloudprovider.TBucketACLType) error {
acl := s3cli.CannedAcl(cli.ownerId, cli.ownerName, string(cannedAcl))
err := cli.client.SetBucketAcl(name, acl)
if err != nil {
if strings.Contains(err.Error(), "not implemented") {
// ignore not implemented error
return nil // cloudprovider.ErrNotImplemented
} else {
return errors.Wrap(err, "SetBucketAcl")
}
}
return nil
}
func (cli *SObjectStoreClient) GetObjectAcl(bucket, key string) (cloudprovider.TBucketACLType, error) {
acl, err := cli.client.GetObjectACL(bucket, key)
if err != nil {
return "", errors.Wrap(err, "GetBucketAcl")
}
return cloudprovider.TBucketACLType(acl.GetCannedACL()), nil
}
func (cli *SObjectStoreClient) SetObjectAcl(bucket, key string, cannedAcl cloudprovider.TBucketACLType) error {
acl := s3cli.CannedAcl(cli.ownerId, cli.ownerName, string(cannedAcl))
err := cli.client.SetObjectAcl(bucket, key, acl)
if err != nil {
return errors.Wrap(err, "SetObjectAcl")
}
return nil
}
func (cli *SObjectStoreClient) GetIBucketPolicy(name string) (string, error) {
policy, err := cli.client.GetBucketPolicy(name)
if err != nil {
return "", errors.Wrap(err, "GetBucketPolicy")
}
return policy, nil
}
func (cli *SObjectStoreClient) SetIBucketPolicy(name string, policy string) error {
err := cli.client.SetBucketPolicy(name, policy)
if err != nil {
return errors.Wrap(err, "SetBucketPolicy")
}
return nil
}
func (cli *SObjectStoreClient) GetIBucketLiftcycle(name string) (string, error) {
liftcycle, err := cli.client.GetBucketLifecycle(name)
if err != nil {
return "", errors.Wrap(err, "GetBucketLifecycle")
}
return liftcycle, nil
}
func (cli *SObjectStoreClient) IBucketExist(name string) (bool, error) {
exist, header, err := cli.client.BucketExists(name)
if err != nil {
return false, errors.Wrap(err, "BucketExists")
}
if header != nil {
log.Debugf("header: %s", jsonutils.Marshal(header))
}
return exist, nil
}
func (cli *SObjectStoreClient) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) {
return cloudprovider.GetIBucketById(cli, name)
}
func (cli *SObjectStoreClient) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) {
return cli.GetIBucketById(name)
}
func (self *SObjectStoreClient) GetCapabilities() []string {
caps := []string{
// cloudprovider.CLOUD_CAPABILITY_PROJECT,
// cloudprovider.CLOUD_CAPABILITY_COMPUTE,
// cloudprovider.CLOUD_CAPABILITY_NETWORK,
// cloudprovider.CLOUD_CAPABILITY_LOADBALANCER,
cloudprovider.CLOUD_CAPABILITY_OBJECTSTORE,
// cloudprovider.CLOUD_CAPABILITY_RDS,
// cloudprovider.CLOUD_CAPABILITY_CACHE,
// cloudprovider.CLOUD_CAPABILITY_EVENT,
}
return caps
}