From 1479ecd6f618204604a01ac24c975fdf5774501e Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Sun, 21 Nov 2021 10:01:55 +0800 Subject: [PATCH] fix: disk driver persistence --- pkg/apis/compute/guest_disk.go | 4 ++ pkg/compute/guestdrivers/kvm.go | 2 +- pkg/compute/models/guestdisks.go | 1 + pkg/compute/models/guests.go | 12 +++- pkg/hostman/guestman/qemu-arm.go | 89 ++++---------------------- pkg/hostman/guestman/qemu-kvm.go | 72 ++++++++++++++++----- pkg/hostman/guestman/qemu-kvmhelper.go | 85 +++++++++++------------- 7 files changed, 122 insertions(+), 143 deletions(-) diff --git a/pkg/apis/compute/guest_disk.go b/pkg/apis/compute/guest_disk.go index 85e61900e3..e90924add7 100644 --- a/pkg/apis/compute/guest_disk.go +++ b/pkg/apis/compute/guest_disk.go @@ -78,6 +78,7 @@ type GuestdiskJsonDesc struct { TemplateId string `json:"template_id"` ImagePath string `json:"image_path"` StorageId string `json:"storage_id"` + StorageType string `json:"storage_type"` Migrating bool `json:"migrating"` Path string `json:"path"` Format string `json:"format"` @@ -88,6 +89,7 @@ type GuestdiskJsonDesc struct { Mountpoint string `json:"mountpoint"` Dev string `json:"dev"` IsSSD bool `json:"is_ssd"` + NumQueues uint8 `json:"num_queues"` // esxi ImageInfo struct { @@ -97,4 +99,6 @@ type GuestdiskJsonDesc struct { } `json:"image_info"` TargetStorageId string `json:"target_storage_id"` + + Url string `json:"url"` } diff --git a/pkg/compute/guestdrivers/kvm.go b/pkg/compute/guestdrivers/kvm.go index 2f183033ed..8a2f1bc767 100644 --- a/pkg/compute/guestdrivers/kvm.go +++ b/pkg/compute/guestdrivers/kvm.go @@ -311,7 +311,7 @@ func (self *SKVMGuestDriver) RequestSyncstatusOnHost(ctx context.Context, guest header.Set(mcclient.REGION_VERSION, "v2") url := fmt.Sprintf("%s/servers/%s/status", host.ManagerUri, guest.Id) - _, res, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "GET", url, header, nil, false) + _, res, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "GET", url, header, nil, true) if err != nil { return nil, err } diff --git a/pkg/compute/models/guestdisks.go b/pkg/compute/models/guestdisks.go index de0c47a83c..29afec7421 100644 --- a/pkg/compute/models/guestdisks.go +++ b/pkg/compute/models/guestdisks.go @@ -193,6 +193,7 @@ func (self *SGuestdisk) GetJsonDescAtHost(host *SHost) *api.GuestdiskJsonDesc { desc.TemplateId = disk.GetTemplateId() if len(desc.TemplateId) > 0 { storage, _ := disk.GetStorage() + desc.StorageType = storage.StorageType storagecacheimg := StoragecachedimageManager.GetStoragecachedimage(storage.StoragecacheId, desc.TemplateId) if storagecacheimg != nil { desc.ImagePath = storagecacheimg.Path diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index ae2875b3a4..06e146eef4 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -3079,15 +3079,21 @@ func (self *SGuest) AttachDisk(ctx context.Context, disk *SDisk, userCred mcclie func (self *SGuest) attach2Disk(ctx context.Context, disk *SDisk, userCred mcclient.TokenCredential, driver string, cache string, mountpoint string) error { attached, err := self.isAttach2Disk(disk) if err != nil { - return err + return errors.Wrap(err, "isAttach2Disk") } if attached { return fmt.Errorf("Guest has been attached to disk") } if len(driver) == 0 { - osProf := self.GetOSProfile() - driver = osProf.DiskDriver + // depends the last disk of this guest + existingDisks, _ := self.GetGuestDisks() + if len(existingDisks) > 0 { + driver = existingDisks[len(existingDisks)-1].Driver + } else { + osProf := self.GetOSProfile() + driver = osProf.DiskDriver + } } guestdisk := SGuestdisk{} guestdisk.SetModelManager(GuestdiskManager, &guestdisk) diff --git a/pkg/hostman/guestman/qemu-arm.go b/pkg/hostman/guestman/qemu-arm.go index 74b1692977..78c92d9f70 100644 --- a/pkg/hostman/guestman/qemu-arm.go +++ b/pkg/hostman/guestman/qemu-arm.go @@ -19,11 +19,9 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" - "yunion.io/x/pkg/utils" api "yunion.io/x/onecloud/pkg/apis/compute" options "yunion.io/x/onecloud/pkg/hostman/options" - "yunion.io/x/onecloud/pkg/hostman/storageman" fileutils2 "yunion.io/x/onecloud/pkg/util/fileutils2" "yunion.io/x/onecloud/pkg/util/qemutils" ) @@ -32,47 +30,19 @@ func (s *SKVMGuestInstance) getArmMachine() string { return "virt-2.12" } -func (s *SKVMGuestInstance) getArmVdiskDesc(disk jsonutils.JSONObject) string { - diskIndex, _ := disk.Int("index") - diskDriver, _ := disk.GetString("driver") - numQueues, _ := disk.Int("num_queues") - - if numQueues == 0 { - numQueues = 4 - } - - if diskDriver == DISK_DRIVER_IDE || diskDriver == DISK_DRIVER_SATA { - // unsupported configuration: IDE controllers are unsupported - // for this QEMU binary or machine type - // replace with scsi - diskDriver = DISK_DRIVER_SCSI - } - - var cmd = "" - cmd += fmt.Sprintf(" -device %s", s.GetDiskDeviceModel(diskDriver)) - cmd += fmt.Sprintf(",drive=drive_%d", diskIndex) - if diskDriver == DISK_DRIVER_VIRTIO { - cmd += fmt.Sprintf(",bus=%s,addr=0x%x", s.GetPciBus(), s.GetDiskAddr(int(diskIndex))) - cmd += fmt.Sprintf(",num-queues=%d,vectors=%d,iothread=iothread0", numQueues, numQueues+1) - } else if utils.IsInStringArray(diskDriver, []string{DISK_DRIVER_SCSI, DISK_DRIVER_PVSCSI}) { - cmd += ",bus=scsi.0" - } - cmd += fmt.Sprintf(",id=drive_%d", diskIndex) - return cmd -} - // arm cpu unsupport hackintosh func (s *SKVMGuestInstance) generateArmStartScript(data *jsonutils.JSONDict) (string, error) { var ( - uuid, _ = s.Desc.GetString("uuid") - mem, _ = s.Desc.Int("mem") - cpu, _ = s.Desc.Int("cpu") - name, _ = s.Desc.GetString("name") - nics, _ = s.Desc.GetArray("nics") - disks, _ = s.Desc.GetArray("disks") - osname = s.getOsname() - cmd = "" + uuid, _ = s.Desc.GetString("uuid") + mem, _ = s.Desc.Int("mem") + cpu, _ = s.Desc.Int("cpu") + name, _ = s.Desc.GetString("name") + nics, _ = s.Desc.GetArray("nics") + osname = s.getOsname() + cmd = "" ) + disks := make([]api.GuestdiskJsonDesc, 0) + s.Desc.Unmarshal(&disks, "disks") vncPort, _ := data.Int("vnc_port") @@ -107,18 +77,11 @@ func (s *SKVMGuestInstance) generateArmStartScript(data *jsonutils.JSONDict) (st cmd += "sleep 1\n" cmd += fmt.Sprintf("echo %d > %s\n", vncPort, s.GetVncFilePath()) - diskObjs := make([]storageman.IDisk, len(disks)) - for idx, disk := range disks { - diskPath, _ := disk.GetString("path") - d, err := storageman.GetManager().GetDiskByPath(diskPath) - if err != nil { - return "", errors.Wrapf(err, "GetDiskByPath(%s)", diskPath) - } - diskObjs[idx] = d - - diskIndex, _ := disk.Int("index") - cmd += d.GetDiskSetupScripts(int(diskIndex)) + diskScripts, err := s.generateDiskSetupScripts(disks) + if err != nil { + return "", errors.Wrap(err, "generateDiskSetupScripts") } + cmd += diskScripts cmd += fmt.Sprintf("STATE_FILE=`ls -d %s* | head -n 1`\n", s.getStateFilePathRootPrefix()) cmd += fmt.Sprintf("PID_FILE=%s\n", s.GetPidFilePath()) @@ -251,31 +214,7 @@ function nic_mtu() { cmd += " -object iothread,id=iothread0" - var diskDrivers = []string{} - for _, disk := range disks { - diskDriver, _ := disk.GetString("driver") - if diskDriver == DISK_DRIVER_IDE || diskDriver == DISK_DRIVER_SATA { - // unsupported configuration: IDE controllers are unsupported - // for this QEMU binary or machine type - // replace with scsi - diskDriver = DISK_DRIVER_SCSI - } - diskDrivers = append(diskDrivers, diskDriver) - } - - if utils.IsInStringArray(DISK_DRIVER_SCSI, diskDrivers) { - cmd += " -device virtio-scsi-pci,id=scsi,iothread=iothread0,num_queues=4,vectors=5" - } else if utils.IsInStringArray(DISK_DRIVER_PVSCSI, diskDrivers) { - cmd += " -device pvscsi,id=scsi" - } - - for idx, disk := range disks { - format, _ := disk.GetString("format") - obj := diskObjs[idx] - enableFileLocking := obj.GetType() == api.STORAGE_LOCAL - cmd += s.getDriveDesc(disk, format, enableFileLocking) - cmd += s.getArmVdiskDesc(disk) - } + cmd += s.generateDiskParams(disks, true) cdrom, _ := s.Desc.Get("cdrom") if cdrom != nil && cdrom.Contains("path") { diff --git a/pkg/hostman/guestman/qemu-kvm.go b/pkg/hostman/guestman/qemu-kvm.go index 67f4bba51b..993284a3d6 100644 --- a/pkg/hostman/guestman/qemu-kvm.go +++ b/pkg/hostman/guestman/qemu-kvm.go @@ -31,7 +31,7 @@ import ( "yunion.io/x/pkg/util/seclib" "yunion.io/x/pkg/utils" - "yunion.io/x/onecloud/pkg/apis/compute" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/appctx" deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis" "yunion.io/x/onecloud/pkg/hostman/hostdeployer/deployclient" @@ -469,7 +469,7 @@ func (s *SKVMGuestInstance) onReceiveQMPEvent(event *monitor.Event) { if stype == "mirror" { mirrorStatus := s.MirrorJobStatus() if mirrorStatus.IsSucc() { - _, err := hostutils.UpdateServerStatus(context.Background(), s.GetId(), compute.VM_RUNNING, "BLOCK_JOB_READY") + _, err := hostutils.UpdateServerStatus(context.Background(), s.GetId(), api.VM_RUNNING, "BLOCK_JOB_READY") if err != nil { log.Errorf("onReceiveQMPEvent update server status error: %s", err) } @@ -560,12 +560,12 @@ func (s *SKVMGuestInstance) onMonitorDisConnect(err error) { func (s *SKVMGuestInstance) startDiskBackupMirror(ctx context.Context) { if ctx == nil || len(appctx.AppContextTaskId(ctx)) == 0 { - status := compute.VM_RUNNING + status := api.VM_RUNNING mirrorStatus := s.MirrorJobStatus() if mirrorStatus.InProcess() { - status = compute.VM_BLOCK_STREAM + status = api.VM_BLOCK_STREAM } else if mirrorStatus.IsFailed() { - status = compute.VM_BLOCK_STREAM_FAIL + status = api.VM_BLOCK_STREAM_FAIL s.SyncMirrorJobFailed("mirror job missing") } hostutils.UpdateServerStatus(context.Background(), s.GetId(), status, "") @@ -761,18 +761,18 @@ func (s *SKVMGuestInstance) SyncStatus(reason string) { } func (s *SKVMGuestInstance) CheckBlockOrRunning(jobs int) { - var status = compute.VM_RUNNING + var status = api.VM_RUNNING if jobs > 0 { if s.IsMaster() { mirrorStatus := s.MirrorJobStatus() if mirrorStatus.InProcess() { - status = compute.VM_BLOCK_STREAM + status = api.VM_BLOCK_STREAM } else if mirrorStatus.IsFailed() { - status = compute.VM_BLOCK_STREAM_FAIL + status = api.VM_BLOCK_STREAM_FAIL s.SyncMirrorJobFailed("Block job missing") } } else { - status = compute.VM_BLOCK_STREAM + status = api.VM_BLOCK_STREAM } } _, err := hostutils.UpdateServerStatus(context.Background(), s.Id, status, "") @@ -807,7 +807,7 @@ func (s *SKVMGuestInstance) GetVpcNIC() jsonutils.JSONObject { nics, _ := s.Desc.GetArray("nics") for _, nic := range nics { vpcProvider, _ := nic.GetString("vpc", "provider") - if vpcProvider == compute.VPC_PROVIDER_OVN { + if vpcProvider == api.VPC_PROVIDER_OVN { if ip, _ := nic.GetString("ip"); ip != "" { return nic } @@ -924,7 +924,7 @@ func (s *SKVMGuestInstance) delTmpDisks(ctx context.Context, migrated bool) erro if disk.Contains("path") { diskPath, _ := disk.GetString("path") d, _ := storageman.GetManager().GetDiskByPath(diskPath) - if d != nil && d.GetType() == compute.STORAGE_LOCAL && migrated { + if d != nil && d.GetType() == api.STORAGE_LOCAL && migrated { if err := d.DeleteAllSnapshot(); err != nil { log.Errorln(err) return err @@ -1149,7 +1149,7 @@ func getNicBridge(nic jsonutils.JSONObject) string { bridge, _ := nic.GetString("bridge") if len(bridge) == 0 { vpcProvider, _ := nic.GetString("vpc", "provider") - if vpcProvider == compute.VPC_PROVIDER_OVN { + if vpcProvider == api.VPC_PROVIDER_OVN { bridge = options.HostOptions.OvnIntegrationBridge } } @@ -1421,9 +1421,9 @@ func (s *SKVMGuestInstance) OnResumeSyncMetadataInfo() { meta.Set("__hugepage", jsonutils.NewString("native")) } if !options.HostOptions.HostCpuPassthrough || s.getOsname() == OS_NAME_MACOS { - meta.Set("__cpu_mode", jsonutils.NewString(compute.CPU_MODE_QEMU)) + meta.Set("__cpu_mode", jsonutils.NewString(api.CPU_MODE_QEMU)) } else { - meta.Set("__cpu_mode", jsonutils.NewString(compute.CPU_MODE_HOST)) + meta.Set("__cpu_mode", jsonutils.NewString(api.CPU_MODE_HOST)) } if s.syncMeta != nil { meta.Update(s.syncMeta) @@ -1596,7 +1596,7 @@ func (s *SKVMGuestInstance) PrepareMigrate(liveMigrage bool) (*jsonutils.JSONDic if err != nil { return nil, errors.Wrapf(err, "GetDiskByPath(%s)", diskPath) } - if d.GetType() == compute.STORAGE_LOCAL { + if d.GetType() == api.STORAGE_LOCAL { back, err := d.PrepareMigrate(liveMigrage) if err != nil { return nil, err @@ -1630,9 +1630,49 @@ func (s *SKVMGuestInstance) IsSharedStorage() bool { log.Errorf("failed find disk by path %s", diskPath) return false } - if !utils.IsInStringArray(disk.GetType(), compute.SHARED_STORAGE) { + if !utils.IsInStringArray(disk.GetType(), api.SHARED_STORAGE) { return false } } return true } + +func (s *SKVMGuestInstance) generateDiskSetupScripts(disks []api.GuestdiskJsonDesc) (string, error) { + cmd := " " + for _, disk := range disks { + diskPath := disk.Path + d, err := storageman.GetManager().GetDiskByPath(diskPath) + if err != nil { + return "", errors.Wrapf(err, "GetDiskByPath(%s)", diskPath) + } + if len(disk.StorageType) == 0 { + disk.StorageType = d.GetType() + } + + diskIndex := disk.Index + cmd += d.GetDiskSetupScripts(int(diskIndex)) + } + return cmd, nil +} + +func (s *SKVMGuestInstance) generateDiskParams(disks []api.GuestdiskJsonDesc, isArm bool) string { + cmd := " " + firstDriver := make(map[string]bool) + for _, disk := range disks { + driver := disk.Driver + if driver == DISK_DRIVER_SCSI || driver == DISK_DRIVER_PVSCSI { + if _, ok := firstDriver[driver]; !ok { + switch driver { + case DISK_DRIVER_SCSI: + cmd += " -device virtio-scsi-pci,id=scsi,iothread=iothread0,num_queues=4,vectors=5" + case DISK_DRIVER_PVSCSI: + cmd += " -device pvscsi,id=scsi" + } + firstDriver[driver] = true + } + } + cmd += s.getDriveDesc(disk, isArm) + cmd += s.getVdiskDesc(disk, isArm) + } + return cmd +} diff --git a/pkg/hostman/guestman/qemu-kvmhelper.go b/pkg/hostman/guestman/qemu-kvmhelper.go index 61ae9066b5..48eff6ffa9 100644 --- a/pkg/hostman/guestman/qemu-kvmhelper.go +++ b/pkg/hostman/guestman/qemu-kvmhelper.go @@ -31,7 +31,6 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/hostman/options" - "yunion.io/x/onecloud/pkg/hostman/storageman" "yunion.io/x/onecloud/pkg/util/fileutils2" "yunion.io/x/onecloud/pkg/util/qemutils" "yunion.io/x/onecloud/pkg/util/sysutils" @@ -164,10 +163,11 @@ func (s *SKVMGuestInstance) disablePvpanicDev() bool { return val == "true" } -func (s *SKVMGuestInstance) getDriveDesc(disk jsonutils.JSONObject, format string, fileLockingOff bool) string { - diskIndex, _ := disk.Int("index") - cacheMode, _ := disk.GetString("cache_mode") - aioMode, _ := disk.GetString("aio_mode") +func (s *SKVMGuestInstance) getDriveDesc(disk api.GuestdiskJsonDesc, isArm bool) string { + format := disk.Format + diskIndex := disk.Index + cacheMode := disk.CacheMode + aioMode := disk.AioMode cmd := " -drive" cmd += fmt.Sprintf(" file=$DISK_%d", diskIndex) @@ -179,11 +179,13 @@ func (s *SKVMGuestInstance) getDriveDesc(disk jsonutils.JSONObject, format strin cmd += ",format=raw" } cmd += fmt.Sprintf(",cache=%s", cacheMode) - cmd += fmt.Sprintf(",aio=%s", aioMode) - if disk.Contains("url") { // # a remote file backed image + if disk.StorageType == api.STORAGE_LOCAL { + cmd += fmt.Sprintf(",aio=%s", aioMode) + } + if len(disk.Url) > 0 { // # a remote file backed image cmd += ",copy-on-read=on" } - if fileLockingOff { + if isArm && disk.StorageType == api.STORAGE_LOCAL { cmd += ",file.locking=off" } // #cmd += ",media=disk" @@ -212,16 +214,23 @@ func (s *SKVMGuestInstance) GetDiskDeviceModel(driver string) string { } } -func (s *SKVMGuestInstance) getVdiskDesc(disk jsonutils.JSONObject) string { - diskIndex, _ := disk.Int("index") - diskDriver, _ := disk.GetString("driver") - numQueues, _ := disk.Int("num_queues") - isSsd := jsonutils.QueryBoolean(disk, "is_ssd", false) +func (s *SKVMGuestInstance) getVdiskDesc(disk api.GuestdiskJsonDesc, isArm bool) string { + diskIndex := disk.Index + diskDriver := disk.Driver + numQueues := disk.NumQueues + isSsd := disk.IsSSD if numQueues == 0 { numQueues = 4 } + if isArm && (diskDriver == DISK_DRIVER_IDE || diskDriver == DISK_DRIVER_SATA) { + // unsupported configuration: IDE controllers are unsupported + // for this QEMU binary or machine type + // replace with scsi + diskDriver = DISK_DRIVER_SCSI + } + var cmd = "" cmd += fmt.Sprintf(" -device %s", s.GetDiskDeviceModel(diskDriver)) cmd += fmt.Sprintf(",drive=drive_%d", diskIndex) @@ -380,15 +389,16 @@ func (s *SKVMGuestInstance) extraOptions() string { func (s *SKVMGuestInstance) _generateStartScript(data *jsonutils.JSONDict) (string, error) { var ( - uuid, _ = s.Desc.GetString("uuid") - mem, _ = s.Desc.Int("mem") - cpu, _ = s.Desc.Int("cpu") - name, _ = s.Desc.GetString("name") - nics, _ = s.Desc.GetArray("nics") - disks, _ = s.Desc.GetArray("disks") - osname = s.getOsname() - cmd = "" + uuid, _ = s.Desc.GetString("uuid") + mem, _ = s.Desc.Int("mem") + cpu, _ = s.Desc.Int("cpu") + name, _ = s.Desc.GetString("name") + nics, _ = s.Desc.GetArray("nics") + osname = s.getOsname() + cmd = "" ) + disks := make([]api.GuestdiskJsonDesc, 0) + s.Desc.Unmarshal(&disks, "disks") if osname == OS_NAME_MACOS { s.Desc.Set("machine", jsonutils.NewString("q35")) @@ -428,16 +438,11 @@ func (s *SKVMGuestInstance) _generateStartScript(data *jsonutils.JSONDict) (stri cmd += "sleep 1\n" cmd += fmt.Sprintf("echo %d > %s\n", vncPort, s.GetVncFilePath()) - for _, disk := range disks { - diskPath, _ := disk.GetString("path") - d, err := storageman.GetManager().GetDiskByPath(diskPath) - if err != nil { - return "", errors.Wrapf(err, "GetDiskByPath(%s)", diskPath) - } - - diskIndex, _ := disk.Int("index") - cmd += d.GetDiskSetupScripts(int(diskIndex)) + diskScripts, err := s.generateDiskSetupScripts(disks) + if err != nil { + return "", errors.Wrap(err, "generateDiskSetupScripts") } + cmd += diskScripts // cmd += fmt.Sprintf("STATE_FILE=`ls -d %s* | head -n 1`\n", s.getStateFilePathRootPrefix()) cmd += fmt.Sprintf("PID_FILE=%s\n", s.GetPidFilePath()) @@ -564,8 +569,7 @@ function nic_mtu() { if osname == OS_NAME_MACOS { cmd += " -device isa-applesmc,osk=ourhardworkbythesewordsguardedpleasedontsteal(c)AppleComputerInc" for i := 0; i < len(disks); i++ { - disk := disks[i].(*jsonutils.JSONDict) - disk.Set("driver", jsonutils.NewString(DISK_DRIVER_SATA)) + disks[i].Driver = DISK_DRIVER_SATA } for i := 0; i < len(nics); i++ { nic := nics[i].(*jsonutils.JSONDict) @@ -628,23 +632,8 @@ function nic_mtu() { } cmd += " -object iothread,id=iothread0" - var diskDrivers = []string{} - for _, disk := range disks { - driver, _ := disk.GetString("driver") - diskDrivers = append(diskDrivers, driver) - } - if utils.IsInStringArray(DISK_DRIVER_SCSI, diskDrivers) { - cmd += " -device virtio-scsi-pci,id=scsi,iothread=iothread0,num_queues=4,vectors=5" - } else if utils.IsInStringArray(DISK_DRIVER_PVSCSI, diskDrivers) { - cmd += " -device pvscsi,id=scsi" - } - - for _, disk := range disks { - format, _ := disk.GetString("format") - cmd += s.getDriveDesc(disk, format, false) - cmd += s.getVdiskDesc(disk) - } + cmd += s.generateDiskParams(disks, false) if osname != OS_NAME_MACOS { cmd += " -device ide-cd,drive=ide0-cd0,bus=ide.1"