mirror of
https://github.com/yunionio/cloudpods.git
synced 2026-07-01 12:14:18 +08:00
465 lines
14 KiB
Go
465 lines
14 KiB
Go
// Copyright 2019 Yunion
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package deployserver
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net"
|
|
"os"
|
|
"runtime/debug"
|
|
"strings"
|
|
"time"
|
|
|
|
"google.golang.org/grpc"
|
|
|
|
execlient "yunion.io/x/executor/client"
|
|
"yunion.io/x/log"
|
|
"yunion.io/x/pkg/errors"
|
|
|
|
commonconsts "yunion.io/x/onecloud/pkg/cloudcommon/consts"
|
|
"yunion.io/x/onecloud/pkg/cloudcommon/service"
|
|
"yunion.io/x/onecloud/pkg/hostman/diskutils"
|
|
"yunion.io/x/onecloud/pkg/hostman/diskutils/libguestfs"
|
|
"yunion.io/x/onecloud/pkg/hostman/diskutils/nbd"
|
|
"yunion.io/x/onecloud/pkg/hostman/diskutils/qemu_kvm"
|
|
"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
|
|
deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
|
|
"yunion.io/x/onecloud/pkg/hostman/hostdeployer/consts"
|
|
"yunion.io/x/onecloud/pkg/util/fileutils2"
|
|
"yunion.io/x/onecloud/pkg/util/procutils"
|
|
"yunion.io/x/onecloud/pkg/util/qemuimg"
|
|
"yunion.io/x/onecloud/pkg/util/seclib2"
|
|
"yunion.io/x/onecloud/pkg/util/winutils"
|
|
)
|
|
|
|
type DeployerServer struct {
|
|
deployapi.UnimplementedDeployAgentServer
|
|
}
|
|
|
|
var _ deployapi.DeployAgentServer = &DeployerServer{}
|
|
|
|
func apiDiskInfo(req *deployapi.DiskInfo) qemuimg.SImageInfo {
|
|
if req == nil {
|
|
return qemuimg.SImageInfo{}
|
|
}
|
|
info := qemuimg.SImageInfo{
|
|
Path: req.GetPath(),
|
|
}
|
|
passwd := req.GetEncryptPassword()
|
|
if len(passwd) > 0 {
|
|
info.Password = passwd
|
|
info.EncryptFormat = qemuimg.TEncryptFormat(req.GetEncryptFormat())
|
|
info.EncryptAlg = seclib2.TSymEncAlg(req.GetEncryptAlg())
|
|
}
|
|
return info
|
|
}
|
|
|
|
func (*DeployerServer) DeployGuestFs(ctx context.Context, req *deployapi.DeployParams,
|
|
) (res *deployapi.DeployGuestFsResponse, err error) {
|
|
// There will be some occasional unknown panic, so temporarily capture panic here.
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
log.Errorf("DeployGuestFs: %s", r)
|
|
debug.PrintStack()
|
|
msg := "panic: "
|
|
if str, ok := r.(fmt.Stringer); ok {
|
|
msg += str.String()
|
|
}
|
|
res, err = nil, errors.Error(msg)
|
|
}
|
|
}()
|
|
log.Infof("********* Deploy guest fs on %#v", apiDiskInfo(req.DiskInfo))
|
|
disk, err := diskutils.GetIDisk(diskutils.DiskParams{
|
|
Hypervisor: req.GuestDesc.Hypervisor,
|
|
DiskInfo: apiDiskInfo(req.GetDiskInfo()),
|
|
VddkInfo: req.VddkInfo,
|
|
}, DeployOption.ImageDeployDriver, false)
|
|
if err != nil {
|
|
log.Errorf("diskutils.GetIDisk fail %s", err)
|
|
return new(deployapi.DeployGuestFsResponse), errors.Wrap(err, "GetIDisk")
|
|
}
|
|
defer disk.Cleanup()
|
|
|
|
if err := disk.Connect(req.GuestDesc); err != nil {
|
|
log.Errorf("Failed to connect %s disk: %s", req.GuestDesc.Hypervisor, err)
|
|
return new(deployapi.DeployGuestFsResponse), errors.Wrap(err, "Connect")
|
|
}
|
|
defer disk.Disconnect()
|
|
|
|
ret, err := disk.DeployGuestfs(req)
|
|
if ret == nil {
|
|
ret = new(deployapi.DeployGuestFsResponse)
|
|
}
|
|
if err != nil {
|
|
log.Errorf("failed deploy guest fs: %s", err)
|
|
}
|
|
return ret, err
|
|
}
|
|
|
|
func (*DeployerServer) ResizeFs(ctx context.Context, req *deployapi.ResizeFsParams) (res *deployapi.Empty, err error) {
|
|
// There will be some occasional unknown panic, so temporarily capture panic here.
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
log.Errorf("DeployGuestFs: %s, stack:\n %s", req.String(), debug.Stack())
|
|
msg := "panic: "
|
|
if str, ok := r.(fmt.Stringer); ok {
|
|
msg += str.String()
|
|
}
|
|
res, err = nil, errors.Error(msg)
|
|
}
|
|
}()
|
|
log.Infof("********* Resize fs on %#v", apiDiskInfo(req.DiskInfo))
|
|
disk, err := diskutils.GetIDisk(diskutils.DiskParams{
|
|
Hypervisor: req.Hypervisor,
|
|
DiskInfo: apiDiskInfo(req.GetDiskInfo()),
|
|
VddkInfo: req.VddkInfo,
|
|
}, DeployOption.ImageDeployDriver, false)
|
|
if err != nil {
|
|
return new(deployapi.Empty), errors.Wrap(err, "GetIDisk fail")
|
|
}
|
|
defer disk.Cleanup()
|
|
|
|
if err := disk.Connect(nil); err != nil {
|
|
return new(deployapi.Empty), errors.Wrap(err, "disk connect failed")
|
|
}
|
|
defer disk.Disconnect()
|
|
|
|
return disk.ResizeFs()
|
|
}
|
|
|
|
func (*DeployerServer) FormatFs(ctx context.Context, req *deployapi.FormatFsParams) (*deployapi.Empty, error) {
|
|
log.Infof("********* Format fs on %#v", apiDiskInfo(req.DiskInfo))
|
|
gd, err := diskutils.NewKVMGuestDisk(apiDiskInfo(req.GetDiskInfo()), DeployOption.ImageDeployDriver, false)
|
|
if err != nil {
|
|
return new(deployapi.Empty), errors.Wrap(err, "NewKVMGuestDisk")
|
|
}
|
|
defer gd.Cleanup()
|
|
|
|
if err := gd.Connect(nil); err == nil {
|
|
defer gd.Disconnect()
|
|
return gd.FormatFs(req)
|
|
} else {
|
|
log.Errorf("failed connect kvm disk %#v: %s", apiDiskInfo(req.DiskInfo), err)
|
|
}
|
|
return new(deployapi.Empty), nil
|
|
}
|
|
|
|
func (*DeployerServer) SaveToGlance(ctx context.Context, req *deployapi.SaveToGlanceParams) (*deployapi.SaveToGlanceResponse, error) {
|
|
log.Infof("********* %#v save to glance", apiDiskInfo(req.DiskInfo))
|
|
|
|
kvmDisk, err := diskutils.NewKVMGuestDisk(apiDiskInfo(req.GetDiskInfo()), DeployOption.ImageDeployDriver, false)
|
|
if err != nil {
|
|
return new(deployapi.SaveToGlanceResponse), errors.Wrap(err, "NewKVMGuestDisk")
|
|
}
|
|
defer kvmDisk.Cleanup()
|
|
|
|
err = kvmDisk.Connect(nil)
|
|
if err != nil {
|
|
return new(deployapi.SaveToGlanceResponse), errors.Wrapf(err, "kvmDisk.Connect %#v", apiDiskInfo(req.DiskInfo))
|
|
}
|
|
defer kvmDisk.Disconnect()
|
|
|
|
return kvmDisk.SaveToGlance(req)
|
|
}
|
|
|
|
func (s *DeployerServer) ProbeImageInfo(ctx context.Context, req *deployapi.ProbeImageInfoPramas) (*deployapi.ImageInfo, error) {
|
|
log.Infof("********* %#v probe image info", apiDiskInfo(req.DiskInfo))
|
|
kvmDisk, err := diskutils.NewKVMGuestDisk(apiDiskInfo(req.GetDiskInfo()), DeployOption.ImageDeployDriver, true)
|
|
if err != nil {
|
|
return new(deployapi.ImageInfo), errors.Wrap(err, "NewKVMGuestDisk")
|
|
}
|
|
defer kvmDisk.Cleanup()
|
|
|
|
if err := kvmDisk.Connect(nil); err != nil {
|
|
log.Errorf("Failed to connect kvm disk %#v: %s", apiDiskInfo(req.DiskInfo), err)
|
|
return new(deployapi.ImageInfo), errors.Wrap(err, "Disk connector failed to connect image")
|
|
}
|
|
defer kvmDisk.Disconnect()
|
|
|
|
return kvmDisk.ProbeImageInfo(req)
|
|
}
|
|
|
|
var connectedEsxiDisks = map[string]*diskutils.VDDKDisk{}
|
|
|
|
func (*DeployerServer) ConnectEsxiDisks(
|
|
ctx context.Context, req *deployapi.ConnectEsxiDisksParams,
|
|
) (*deployapi.EsxiDisksConnectionInfo, error) {
|
|
log.Infof("********* Connect esxi disks ...")
|
|
var (
|
|
err error
|
|
flatFilePath string
|
|
ret = new(deployapi.EsxiDisksConnectionInfo)
|
|
)
|
|
ret.Disks = make([]*deployapi.EsxiDiskInfo, len(req.AccessInfo))
|
|
for i := 0; i < len(req.AccessInfo); i++ {
|
|
disk, _ := diskutils.NewVDDKDisk(req.VddkInfo, req.AccessInfo[i].DiskPath, DeployOption.ImageDeployDriver, false)
|
|
flatFilePath, err = disk.ConnectBlockDevice()
|
|
if err != nil {
|
|
err = errors.Wrapf(err, "disk %s connect block device", req.AccessInfo[i].DiskPath)
|
|
break
|
|
}
|
|
connectedEsxiDisks[flatFilePath] = disk
|
|
ret.Disks[i] = &deployapi.EsxiDiskInfo{DiskPath: flatFilePath}
|
|
}
|
|
if err != nil {
|
|
for i := 0; i < len(req.AccessInfo); i++ {
|
|
if disk, ok := connectedEsxiDisks[req.AccessInfo[i].DiskPath]; ok {
|
|
if e := disk.DisconnectBlockDevice(); e != nil {
|
|
log.Errorf("disconnect disk %s: %s", req.AccessInfo[i].DiskPath, e)
|
|
} else {
|
|
delete(connectedEsxiDisks, req.AccessInfo[i].DiskPath)
|
|
}
|
|
}
|
|
}
|
|
return ret, err
|
|
}
|
|
return ret, nil
|
|
}
|
|
|
|
func (*DeployerServer) DisconnectEsxiDisks(
|
|
ctx context.Context, req *deployapi.EsxiDisksConnectionInfo,
|
|
) (*deployapi.Empty, error) {
|
|
log.Infof("********* Disconnect esxi disks ...")
|
|
for i := 0; i < len(req.Disks); i++ {
|
|
if disk, ok := connectedEsxiDisks[req.Disks[i].DiskPath]; ok {
|
|
if e := disk.DisconnectBlockDevice(); e != nil {
|
|
return new(deployapi.Empty), errors.Wrapf(e, "disconnect disk %s", req.Disks[i].DiskPath)
|
|
} else {
|
|
delete(connectedEsxiDisks, req.Disks[i].DiskPath)
|
|
}
|
|
} else {
|
|
log.Warningf("esxi disk %s not connected", req.Disks[i].DiskPath)
|
|
continue
|
|
}
|
|
}
|
|
return new(deployapi.Empty), nil
|
|
}
|
|
|
|
type SDeployService struct {
|
|
*service.SServiceBase
|
|
|
|
grpcServer *grpc.Server
|
|
}
|
|
|
|
func NewDeployService() *SDeployService {
|
|
deployer := &SDeployService{}
|
|
deployer.SServiceBase = service.NewBaseService(deployer)
|
|
return deployer
|
|
}
|
|
|
|
func (s *SDeployService) RunService() {
|
|
if DeployOption.DeployAction != "" {
|
|
res, err := StartLocalDeploy(DeployOption.DeployAction)
|
|
if err != nil {
|
|
if e := fileutils2.FilePutContents("/error", err.Error(), false); e != nil {
|
|
log.Errorf("failed put errors to file: %s", e)
|
|
}
|
|
}
|
|
if res != nil {
|
|
resStr, _ := json.Marshal(res)
|
|
if e := fileutils2.FilePutContents("/response", string(resStr), false); e != nil {
|
|
log.Errorf("failed put response to file: %s", e)
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
s.grpcServer = grpc.NewServer()
|
|
deployapi.RegisterDeployAgentServer(s.grpcServer, &DeployerServer{})
|
|
if fileutils2.Exists(DeployOption.DeployServerSocketPath) {
|
|
if conn, err := net.Dial("unix", DeployOption.DeployServerSocketPath); err == nil {
|
|
conn.Close()
|
|
log.Fatalf("socket %s already listening", DeployOption.DeployServerSocketPath)
|
|
}
|
|
|
|
if err := os.Remove(DeployOption.DeployServerSocketPath); err != nil {
|
|
log.Fatalln(err)
|
|
}
|
|
}
|
|
listener, err := net.Listen("unix", DeployOption.DeployServerSocketPath)
|
|
if err != nil {
|
|
log.Fatalln(err)
|
|
}
|
|
defer listener.Close()
|
|
log.Infof("Init net listener on %s succ", DeployOption.DeployServerSocketPath)
|
|
s.grpcServer.Serve(listener)
|
|
}
|
|
|
|
func (s *SDeployService) FixPathEnv() error {
|
|
var paths = []string{
|
|
"/usr/bin",
|
|
"/usr/local/sbin",
|
|
"/usr/local/bin",
|
|
"/sbin",
|
|
"/bin",
|
|
"/usr/sbin",
|
|
}
|
|
return os.Setenv("PATH", strings.Join(paths, ":"))
|
|
}
|
|
|
|
func (s *SDeployService) PrepareEnv() error {
|
|
if !fileutils2.Exists(DeployOption.DeployTempDir) {
|
|
err := os.MkdirAll(DeployOption.DeployTempDir, 0755)
|
|
if err != nil {
|
|
return errors.Errorf("fail to create %s: %s", DeployOption.DeployTempDir, err)
|
|
}
|
|
}
|
|
commonconsts.SetDeployTempDir(DeployOption.DeployTempDir)
|
|
commonconsts.SetAllowVmSELinux(DeployOption.AllowVmSELinux)
|
|
|
|
if err := s.FixPathEnv(); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := fsdriver.Init(DeployOption.PrivatePrefixes, DeployOption.CloudrootDir); err != nil {
|
|
log.Fatalln(err)
|
|
}
|
|
if DeployOption.ImageDeployDriver == consts.DEPLOY_DRIVER_LIBGUESTFS {
|
|
if err := libguestfs.Init(3); err != nil {
|
|
log.Fatalln(err)
|
|
}
|
|
}
|
|
|
|
if DeployOption.ImageDeployDriver != consts.DEPLOY_DRIVER_QEMU_KVM {
|
|
if err := nbd.Init(); err != nil {
|
|
return errors.Wrap(err, "nbd.Init")
|
|
}
|
|
} else {
|
|
// prepare for yunionos don't have but necessary files
|
|
out, err := procutils.NewCommand("cp", "-rf", "/usr/bin/chntpw.static", "/opt/yunion/bin/chntpw.static").Output()
|
|
if err != nil {
|
|
return errors.Wrapf(err, "cp files failed %s", out)
|
|
}
|
|
out, err = procutils.NewCommand("cp", "-rf", "/usr/bin/.chntpw.static.bin", "/opt/yunion/bin/.chntpw.static.bin").Output()
|
|
if err != nil {
|
|
return errors.Wrapf(err, "cp files failed %s", out)
|
|
}
|
|
out, err = procutils.NewCommand("mkdir", "-p", "/opt/yunion/bin/bundles").Output()
|
|
if err != nil {
|
|
return errors.Wrapf(err, "cp files failed %s", out)
|
|
}
|
|
out, err = procutils.NewCommand("cp", "-rf", "/usr/bin/bundles/chntpw.static", "/opt/yunion/bin/bundles/chntpw.static").Output()
|
|
if err != nil {
|
|
return errors.Wrapf(err, "cp files failed %s", out)
|
|
}
|
|
out, err = procutils.NewCommand("cp", "-rf", "/usr/sbin/zerofree", "/opt/yunion/bin/zerofree").Output()
|
|
if err != nil {
|
|
return errors.Wrapf(err, "cp files failed %s", out)
|
|
}
|
|
|
|
err = procutils.NewCommand("mkdir", "-p", qemu_kvm.RUN_ON_HOST_ROOT_PATH).Run()
|
|
if err != nil {
|
|
return errors.Wrap(err, "Failed to mkdir RUN_ON_HOST_ROOT_PATH: %s")
|
|
}
|
|
|
|
cmd := fmt.Sprintf("mkisofs -l -J -L -R -r -v -hide-rr-moved -o %s -graft-points vmware-vddk=/opt/vmware-vddk yunion=/opt/yunion", qemu_kvm.DEPLOY_ISO)
|
|
out, err = procutils.NewCommand("bash", "-c", cmd).Output()
|
|
if err != nil {
|
|
return errors.Wrapf(err, "mkisofs failed %s", out)
|
|
}
|
|
|
|
cpuArch, err := procutils.NewCommand("uname", "-m").Output()
|
|
if err != nil {
|
|
return errors.Wrap(err, "get cpu architecture")
|
|
}
|
|
|
|
err = qemu_kvm.InitQemuDeployManager(
|
|
strings.TrimSpace(string(cpuArch)),
|
|
DeployOption.DefaultQemuVersion,
|
|
DeployOption.HugepagesOption == "native",
|
|
DeployOption.HugepageSizeMb*1024,
|
|
DeployOption.DeployConcurrent,
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// create /dev/lvm_remote
|
|
if err := s.checkLvmRemote(); err != nil {
|
|
return errors.Wrap(err, "unable to checkLvmRemote")
|
|
}
|
|
|
|
if !winutils.CheckTool(DeployOption.ChntpwPath) {
|
|
if winutils.CheckTool("/usr/bin/chntpw.static") {
|
|
winutils.SetChntpwPath("/usr/bin/chntpw.static")
|
|
} else {
|
|
log.Errorf("Failed to find chntpw tool")
|
|
}
|
|
} else {
|
|
winutils.SetChntpwPath(DeployOption.ChntpwPath)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *SDeployService) checkLvmRemote() error {
|
|
_, err := os.Stat("/dev/lvm_remote")
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
if os.IsNotExist(err) {
|
|
err := os.Mkdir("/dev/lvm_remote", os.ModePerm)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
|
|
func (s *SDeployService) InitService() {
|
|
s.SignalTrap(func() {
|
|
for {
|
|
if len(connectedEsxiDisks) > 0 {
|
|
log.Warningf("Waiting for esxi disks %d disconnect !!!", len(connectedEsxiDisks))
|
|
time.Sleep(time.Second * 1)
|
|
} else {
|
|
if s.grpcServer != nil {
|
|
s.grpcServer.Stop()
|
|
} else {
|
|
os.Exit(0)
|
|
}
|
|
}
|
|
}
|
|
})
|
|
|
|
if DeployOption.EnableRemoteExecutor {
|
|
log.Infof("exec socket path: %s", DeployOption.ExecutorSocketPath)
|
|
execlient.Init(DeployOption.ExecutorSocketPath)
|
|
execlient.SetTimeoutSeconds(DeployOption.ExecutorConnectTimeoutSeconds)
|
|
procutils.SetRemoteExecutor()
|
|
}
|
|
|
|
if DeployOption.DeployAction != "" {
|
|
if err := LocalInitEnv(); err != nil {
|
|
log.Fatalf("local init env %s", err)
|
|
}
|
|
return
|
|
}
|
|
|
|
if err := s.PrepareEnv(); err != nil {
|
|
log.Fatalln(err)
|
|
}
|
|
|
|
s.O = &DeployOption.BaseOptions
|
|
if len(DeployOption.DeployServerSocketPath) == 0 {
|
|
log.Fatalf("missing deploy server socket path")
|
|
}
|
|
}
|
|
|
|
func (s *SDeployService) OnExitService() {}
|