fix: disk driver persistence

This commit is contained in:
Qiu Jian
2021-11-21 10:01:55 +08:00
parent cc8124bce8
commit 1479ecd6f6
7 changed files with 122 additions and 143 deletions

View File

@@ -78,6 +78,7 @@ type GuestdiskJsonDesc struct {
TemplateId string `json:"template_id"`
ImagePath string `json:"image_path"`
StorageId string `json:"storage_id"`
StorageType string `json:"storage_type"`
Migrating bool `json:"migrating"`
Path string `json:"path"`
Format string `json:"format"`
@@ -88,6 +89,7 @@ type GuestdiskJsonDesc struct {
Mountpoint string `json:"mountpoint"`
Dev string `json:"dev"`
IsSSD bool `json:"is_ssd"`
NumQueues uint8 `json:"num_queues"`
// esxi
ImageInfo struct {
@@ -97,4 +99,6 @@ type GuestdiskJsonDesc struct {
} `json:"image_info"`
TargetStorageId string `json:"target_storage_id"`
Url string `json:"url"`
}

View File

@@ -311,7 +311,7 @@ func (self *SKVMGuestDriver) RequestSyncstatusOnHost(ctx context.Context, guest
header.Set(mcclient.REGION_VERSION, "v2")
url := fmt.Sprintf("%s/servers/%s/status", host.ManagerUri, guest.Id)
_, res, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "GET", url, header, nil, false)
_, res, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "GET", url, header, nil, true)
if err != nil {
return nil, err
}

View File

@@ -193,6 +193,7 @@ func (self *SGuestdisk) GetJsonDescAtHost(host *SHost) *api.GuestdiskJsonDesc {
desc.TemplateId = disk.GetTemplateId()
if len(desc.TemplateId) > 0 {
storage, _ := disk.GetStorage()
desc.StorageType = storage.StorageType
storagecacheimg := StoragecachedimageManager.GetStoragecachedimage(storage.StoragecacheId, desc.TemplateId)
if storagecacheimg != nil {
desc.ImagePath = storagecacheimg.Path

View File

@@ -3079,15 +3079,21 @@ func (self *SGuest) AttachDisk(ctx context.Context, disk *SDisk, userCred mcclie
func (self *SGuest) attach2Disk(ctx context.Context, disk *SDisk, userCred mcclient.TokenCredential, driver string, cache string, mountpoint string) error {
attached, err := self.isAttach2Disk(disk)
if err != nil {
return err
return errors.Wrap(err, "isAttach2Disk")
}
if attached {
return fmt.Errorf("Guest has been attached to disk")
}
if len(driver) == 0 {
osProf := self.GetOSProfile()
driver = osProf.DiskDriver
// depends the last disk of this guest
existingDisks, _ := self.GetGuestDisks()
if len(existingDisks) > 0 {
driver = existingDisks[len(existingDisks)-1].Driver
} else {
osProf := self.GetOSProfile()
driver = osProf.DiskDriver
}
}
guestdisk := SGuestdisk{}
guestdisk.SetModelManager(GuestdiskManager, &guestdisk)

View File

@@ -19,11 +19,9 @@ import (
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/utils"
api "yunion.io/x/onecloud/pkg/apis/compute"
options "yunion.io/x/onecloud/pkg/hostman/options"
"yunion.io/x/onecloud/pkg/hostman/storageman"
fileutils2 "yunion.io/x/onecloud/pkg/util/fileutils2"
"yunion.io/x/onecloud/pkg/util/qemutils"
)
@@ -32,47 +30,19 @@ func (s *SKVMGuestInstance) getArmMachine() string {
return "virt-2.12"
}
func (s *SKVMGuestInstance) getArmVdiskDesc(disk jsonutils.JSONObject) string {
diskIndex, _ := disk.Int("index")
diskDriver, _ := disk.GetString("driver")
numQueues, _ := disk.Int("num_queues")
if numQueues == 0 {
numQueues = 4
}
if diskDriver == DISK_DRIVER_IDE || diskDriver == DISK_DRIVER_SATA {
// unsupported configuration: IDE controllers are unsupported
// for this QEMU binary or machine type
// replace with scsi
diskDriver = DISK_DRIVER_SCSI
}
var cmd = ""
cmd += fmt.Sprintf(" -device %s", s.GetDiskDeviceModel(diskDriver))
cmd += fmt.Sprintf(",drive=drive_%d", diskIndex)
if diskDriver == DISK_DRIVER_VIRTIO {
cmd += fmt.Sprintf(",bus=%s,addr=0x%x", s.GetPciBus(), s.GetDiskAddr(int(diskIndex)))
cmd += fmt.Sprintf(",num-queues=%d,vectors=%d,iothread=iothread0", numQueues, numQueues+1)
} else if utils.IsInStringArray(diskDriver, []string{DISK_DRIVER_SCSI, DISK_DRIVER_PVSCSI}) {
cmd += ",bus=scsi.0"
}
cmd += fmt.Sprintf(",id=drive_%d", diskIndex)
return cmd
}
// arm cpu unsupport hackintosh
func (s *SKVMGuestInstance) generateArmStartScript(data *jsonutils.JSONDict) (string, error) {
var (
uuid, _ = s.Desc.GetString("uuid")
mem, _ = s.Desc.Int("mem")
cpu, _ = s.Desc.Int("cpu")
name, _ = s.Desc.GetString("name")
nics, _ = s.Desc.GetArray("nics")
disks, _ = s.Desc.GetArray("disks")
osname = s.getOsname()
cmd = ""
uuid, _ = s.Desc.GetString("uuid")
mem, _ = s.Desc.Int("mem")
cpu, _ = s.Desc.Int("cpu")
name, _ = s.Desc.GetString("name")
nics, _ = s.Desc.GetArray("nics")
osname = s.getOsname()
cmd = ""
)
disks := make([]api.GuestdiskJsonDesc, 0)
s.Desc.Unmarshal(&disks, "disks")
vncPort, _ := data.Int("vnc_port")
@@ -107,18 +77,11 @@ func (s *SKVMGuestInstance) generateArmStartScript(data *jsonutils.JSONDict) (st
cmd += "sleep 1\n"
cmd += fmt.Sprintf("echo %d > %s\n", vncPort, s.GetVncFilePath())
diskObjs := make([]storageman.IDisk, len(disks))
for idx, disk := range disks {
diskPath, _ := disk.GetString("path")
d, err := storageman.GetManager().GetDiskByPath(diskPath)
if err != nil {
return "", errors.Wrapf(err, "GetDiskByPath(%s)", diskPath)
}
diskObjs[idx] = d
diskIndex, _ := disk.Int("index")
cmd += d.GetDiskSetupScripts(int(diskIndex))
diskScripts, err := s.generateDiskSetupScripts(disks)
if err != nil {
return "", errors.Wrap(err, "generateDiskSetupScripts")
}
cmd += diskScripts
cmd += fmt.Sprintf("STATE_FILE=`ls -d %s* | head -n 1`\n", s.getStateFilePathRootPrefix())
cmd += fmt.Sprintf("PID_FILE=%s\n", s.GetPidFilePath())
@@ -251,31 +214,7 @@ function nic_mtu() {
cmd += " -object iothread,id=iothread0"
var diskDrivers = []string{}
for _, disk := range disks {
diskDriver, _ := disk.GetString("driver")
if diskDriver == DISK_DRIVER_IDE || diskDriver == DISK_DRIVER_SATA {
// unsupported configuration: IDE controllers are unsupported
// for this QEMU binary or machine type
// replace with scsi
diskDriver = DISK_DRIVER_SCSI
}
diskDrivers = append(diskDrivers, diskDriver)
}
if utils.IsInStringArray(DISK_DRIVER_SCSI, diskDrivers) {
cmd += " -device virtio-scsi-pci,id=scsi,iothread=iothread0,num_queues=4,vectors=5"
} else if utils.IsInStringArray(DISK_DRIVER_PVSCSI, diskDrivers) {
cmd += " -device pvscsi,id=scsi"
}
for idx, disk := range disks {
format, _ := disk.GetString("format")
obj := diskObjs[idx]
enableFileLocking := obj.GetType() == api.STORAGE_LOCAL
cmd += s.getDriveDesc(disk, format, enableFileLocking)
cmd += s.getArmVdiskDesc(disk)
}
cmd += s.generateDiskParams(disks, true)
cdrom, _ := s.Desc.Get("cdrom")
if cdrom != nil && cdrom.Contains("path") {

View File

@@ -31,7 +31,7 @@ import (
"yunion.io/x/pkg/util/seclib"
"yunion.io/x/pkg/utils"
"yunion.io/x/onecloud/pkg/apis/compute"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/appctx"
deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
"yunion.io/x/onecloud/pkg/hostman/hostdeployer/deployclient"
@@ -469,7 +469,7 @@ func (s *SKVMGuestInstance) onReceiveQMPEvent(event *monitor.Event) {
if stype == "mirror" {
mirrorStatus := s.MirrorJobStatus()
if mirrorStatus.IsSucc() {
_, err := hostutils.UpdateServerStatus(context.Background(), s.GetId(), compute.VM_RUNNING, "BLOCK_JOB_READY")
_, err := hostutils.UpdateServerStatus(context.Background(), s.GetId(), api.VM_RUNNING, "BLOCK_JOB_READY")
if err != nil {
log.Errorf("onReceiveQMPEvent update server status error: %s", err)
}
@@ -560,12 +560,12 @@ func (s *SKVMGuestInstance) onMonitorDisConnect(err error) {
func (s *SKVMGuestInstance) startDiskBackupMirror(ctx context.Context) {
if ctx == nil || len(appctx.AppContextTaskId(ctx)) == 0 {
status := compute.VM_RUNNING
status := api.VM_RUNNING
mirrorStatus := s.MirrorJobStatus()
if mirrorStatus.InProcess() {
status = compute.VM_BLOCK_STREAM
status = api.VM_BLOCK_STREAM
} else if mirrorStatus.IsFailed() {
status = compute.VM_BLOCK_STREAM_FAIL
status = api.VM_BLOCK_STREAM_FAIL
s.SyncMirrorJobFailed("mirror job missing")
}
hostutils.UpdateServerStatus(context.Background(), s.GetId(), status, "")
@@ -761,18 +761,18 @@ func (s *SKVMGuestInstance) SyncStatus(reason string) {
}
func (s *SKVMGuestInstance) CheckBlockOrRunning(jobs int) {
var status = compute.VM_RUNNING
var status = api.VM_RUNNING
if jobs > 0 {
if s.IsMaster() {
mirrorStatus := s.MirrorJobStatus()
if mirrorStatus.InProcess() {
status = compute.VM_BLOCK_STREAM
status = api.VM_BLOCK_STREAM
} else if mirrorStatus.IsFailed() {
status = compute.VM_BLOCK_STREAM_FAIL
status = api.VM_BLOCK_STREAM_FAIL
s.SyncMirrorJobFailed("Block job missing")
}
} else {
status = compute.VM_BLOCK_STREAM
status = api.VM_BLOCK_STREAM
}
}
_, err := hostutils.UpdateServerStatus(context.Background(), s.Id, status, "")
@@ -807,7 +807,7 @@ func (s *SKVMGuestInstance) GetVpcNIC() jsonutils.JSONObject {
nics, _ := s.Desc.GetArray("nics")
for _, nic := range nics {
vpcProvider, _ := nic.GetString("vpc", "provider")
if vpcProvider == compute.VPC_PROVIDER_OVN {
if vpcProvider == api.VPC_PROVIDER_OVN {
if ip, _ := nic.GetString("ip"); ip != "" {
return nic
}
@@ -924,7 +924,7 @@ func (s *SKVMGuestInstance) delTmpDisks(ctx context.Context, migrated bool) erro
if disk.Contains("path") {
diskPath, _ := disk.GetString("path")
d, _ := storageman.GetManager().GetDiskByPath(diskPath)
if d != nil && d.GetType() == compute.STORAGE_LOCAL && migrated {
if d != nil && d.GetType() == api.STORAGE_LOCAL && migrated {
if err := d.DeleteAllSnapshot(); err != nil {
log.Errorln(err)
return err
@@ -1149,7 +1149,7 @@ func getNicBridge(nic jsonutils.JSONObject) string {
bridge, _ := nic.GetString("bridge")
if len(bridge) == 0 {
vpcProvider, _ := nic.GetString("vpc", "provider")
if vpcProvider == compute.VPC_PROVIDER_OVN {
if vpcProvider == api.VPC_PROVIDER_OVN {
bridge = options.HostOptions.OvnIntegrationBridge
}
}
@@ -1421,9 +1421,9 @@ func (s *SKVMGuestInstance) OnResumeSyncMetadataInfo() {
meta.Set("__hugepage", jsonutils.NewString("native"))
}
if !options.HostOptions.HostCpuPassthrough || s.getOsname() == OS_NAME_MACOS {
meta.Set("__cpu_mode", jsonutils.NewString(compute.CPU_MODE_QEMU))
meta.Set("__cpu_mode", jsonutils.NewString(api.CPU_MODE_QEMU))
} else {
meta.Set("__cpu_mode", jsonutils.NewString(compute.CPU_MODE_HOST))
meta.Set("__cpu_mode", jsonutils.NewString(api.CPU_MODE_HOST))
}
if s.syncMeta != nil {
meta.Update(s.syncMeta)
@@ -1596,7 +1596,7 @@ func (s *SKVMGuestInstance) PrepareMigrate(liveMigrage bool) (*jsonutils.JSONDic
if err != nil {
return nil, errors.Wrapf(err, "GetDiskByPath(%s)", diskPath)
}
if d.GetType() == compute.STORAGE_LOCAL {
if d.GetType() == api.STORAGE_LOCAL {
back, err := d.PrepareMigrate(liveMigrage)
if err != nil {
return nil, err
@@ -1630,9 +1630,49 @@ func (s *SKVMGuestInstance) IsSharedStorage() bool {
log.Errorf("failed find disk by path %s", diskPath)
return false
}
if !utils.IsInStringArray(disk.GetType(), compute.SHARED_STORAGE) {
if !utils.IsInStringArray(disk.GetType(), api.SHARED_STORAGE) {
return false
}
}
return true
}
func (s *SKVMGuestInstance) generateDiskSetupScripts(disks []api.GuestdiskJsonDesc) (string, error) {
cmd := " "
for _, disk := range disks {
diskPath := disk.Path
d, err := storageman.GetManager().GetDiskByPath(diskPath)
if err != nil {
return "", errors.Wrapf(err, "GetDiskByPath(%s)", diskPath)
}
if len(disk.StorageType) == 0 {
disk.StorageType = d.GetType()
}
diskIndex := disk.Index
cmd += d.GetDiskSetupScripts(int(diskIndex))
}
return cmd, nil
}
func (s *SKVMGuestInstance) generateDiskParams(disks []api.GuestdiskJsonDesc, isArm bool) string {
cmd := " "
firstDriver := make(map[string]bool)
for _, disk := range disks {
driver := disk.Driver
if driver == DISK_DRIVER_SCSI || driver == DISK_DRIVER_PVSCSI {
if _, ok := firstDriver[driver]; !ok {
switch driver {
case DISK_DRIVER_SCSI:
cmd += " -device virtio-scsi-pci,id=scsi,iothread=iothread0,num_queues=4,vectors=5"
case DISK_DRIVER_PVSCSI:
cmd += " -device pvscsi,id=scsi"
}
firstDriver[driver] = true
}
}
cmd += s.getDriveDesc(disk, isArm)
cmd += s.getVdiskDesc(disk, isArm)
}
return cmd
}

View File

@@ -31,7 +31,6 @@ import (
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/hostman/options"
"yunion.io/x/onecloud/pkg/hostman/storageman"
"yunion.io/x/onecloud/pkg/util/fileutils2"
"yunion.io/x/onecloud/pkg/util/qemutils"
"yunion.io/x/onecloud/pkg/util/sysutils"
@@ -164,10 +163,11 @@ func (s *SKVMGuestInstance) disablePvpanicDev() bool {
return val == "true"
}
func (s *SKVMGuestInstance) getDriveDesc(disk jsonutils.JSONObject, format string, fileLockingOff bool) string {
diskIndex, _ := disk.Int("index")
cacheMode, _ := disk.GetString("cache_mode")
aioMode, _ := disk.GetString("aio_mode")
func (s *SKVMGuestInstance) getDriveDesc(disk api.GuestdiskJsonDesc, isArm bool) string {
format := disk.Format
diskIndex := disk.Index
cacheMode := disk.CacheMode
aioMode := disk.AioMode
cmd := " -drive"
cmd += fmt.Sprintf(" file=$DISK_%d", diskIndex)
@@ -179,11 +179,13 @@ func (s *SKVMGuestInstance) getDriveDesc(disk jsonutils.JSONObject, format strin
cmd += ",format=raw"
}
cmd += fmt.Sprintf(",cache=%s", cacheMode)
cmd += fmt.Sprintf(",aio=%s", aioMode)
if disk.Contains("url") { // # a remote file backed image
if disk.StorageType == api.STORAGE_LOCAL {
cmd += fmt.Sprintf(",aio=%s", aioMode)
}
if len(disk.Url) > 0 { // # a remote file backed image
cmd += ",copy-on-read=on"
}
if fileLockingOff {
if isArm && disk.StorageType == api.STORAGE_LOCAL {
cmd += ",file.locking=off"
}
// #cmd += ",media=disk"
@@ -212,16 +214,23 @@ func (s *SKVMGuestInstance) GetDiskDeviceModel(driver string) string {
}
}
func (s *SKVMGuestInstance) getVdiskDesc(disk jsonutils.JSONObject) string {
diskIndex, _ := disk.Int("index")
diskDriver, _ := disk.GetString("driver")
numQueues, _ := disk.Int("num_queues")
isSsd := jsonutils.QueryBoolean(disk, "is_ssd", false)
func (s *SKVMGuestInstance) getVdiskDesc(disk api.GuestdiskJsonDesc, isArm bool) string {
diskIndex := disk.Index
diskDriver := disk.Driver
numQueues := disk.NumQueues
isSsd := disk.IsSSD
if numQueues == 0 {
numQueues = 4
}
if isArm && (diskDriver == DISK_DRIVER_IDE || diskDriver == DISK_DRIVER_SATA) {
// unsupported configuration: IDE controllers are unsupported
// for this QEMU binary or machine type
// replace with scsi
diskDriver = DISK_DRIVER_SCSI
}
var cmd = ""
cmd += fmt.Sprintf(" -device %s", s.GetDiskDeviceModel(diskDriver))
cmd += fmt.Sprintf(",drive=drive_%d", diskIndex)
@@ -380,15 +389,16 @@ func (s *SKVMGuestInstance) extraOptions() string {
func (s *SKVMGuestInstance) _generateStartScript(data *jsonutils.JSONDict) (string, error) {
var (
uuid, _ = s.Desc.GetString("uuid")
mem, _ = s.Desc.Int("mem")
cpu, _ = s.Desc.Int("cpu")
name, _ = s.Desc.GetString("name")
nics, _ = s.Desc.GetArray("nics")
disks, _ = s.Desc.GetArray("disks")
osname = s.getOsname()
cmd = ""
uuid, _ = s.Desc.GetString("uuid")
mem, _ = s.Desc.Int("mem")
cpu, _ = s.Desc.Int("cpu")
name, _ = s.Desc.GetString("name")
nics, _ = s.Desc.GetArray("nics")
osname = s.getOsname()
cmd = ""
)
disks := make([]api.GuestdiskJsonDesc, 0)
s.Desc.Unmarshal(&disks, "disks")
if osname == OS_NAME_MACOS {
s.Desc.Set("machine", jsonutils.NewString("q35"))
@@ -428,16 +438,11 @@ func (s *SKVMGuestInstance) _generateStartScript(data *jsonutils.JSONDict) (stri
cmd += "sleep 1\n"
cmd += fmt.Sprintf("echo %d > %s\n", vncPort, s.GetVncFilePath())
for _, disk := range disks {
diskPath, _ := disk.GetString("path")
d, err := storageman.GetManager().GetDiskByPath(diskPath)
if err != nil {
return "", errors.Wrapf(err, "GetDiskByPath(%s)", diskPath)
}
diskIndex, _ := disk.Int("index")
cmd += d.GetDiskSetupScripts(int(diskIndex))
diskScripts, err := s.generateDiskSetupScripts(disks)
if err != nil {
return "", errors.Wrap(err, "generateDiskSetupScripts")
}
cmd += diskScripts
// cmd += fmt.Sprintf("STATE_FILE=`ls -d %s* | head -n 1`\n", s.getStateFilePathRootPrefix())
cmd += fmt.Sprintf("PID_FILE=%s\n", s.GetPidFilePath())
@@ -564,8 +569,7 @@ function nic_mtu() {
if osname == OS_NAME_MACOS {
cmd += " -device isa-applesmc,osk=ourhardworkbythesewordsguardedpleasedontsteal(c)AppleComputerInc"
for i := 0; i < len(disks); i++ {
disk := disks[i].(*jsonutils.JSONDict)
disk.Set("driver", jsonutils.NewString(DISK_DRIVER_SATA))
disks[i].Driver = DISK_DRIVER_SATA
}
for i := 0; i < len(nics); i++ {
nic := nics[i].(*jsonutils.JSONDict)
@@ -628,23 +632,8 @@ function nic_mtu() {
}
cmd += " -object iothread,id=iothread0"
var diskDrivers = []string{}
for _, disk := range disks {
driver, _ := disk.GetString("driver")
diskDrivers = append(diskDrivers, driver)
}
if utils.IsInStringArray(DISK_DRIVER_SCSI, diskDrivers) {
cmd += " -device virtio-scsi-pci,id=scsi,iothread=iothread0,num_queues=4,vectors=5"
} else if utils.IsInStringArray(DISK_DRIVER_PVSCSI, diskDrivers) {
cmd += " -device pvscsi,id=scsi"
}
for _, disk := range disks {
format, _ := disk.GetString("format")
cmd += s.getDriveDesc(disk, format, false)
cmd += s.getVdiskDesc(disk)
}
cmd += s.generateDiskParams(disks, false)
if osname != OS_NAME_MACOS {
cmd += " -device ide-cd,drive=ide0-cd0,bus=ide.1"