feat(region,host,host-deployer): infiniband nic sriov support (#19127)

pkg/apis/compute/guestnetwork.go

@@ -104,6 +104,7 @@ type GuestnetworkBaseDesc struct {
 	Index          int8                 `json:"index"`
 	RxTrafficLimit int64                `json:"rx_traffic_limit"`
 	TxTrafficLimit int64                `json:"tx_traffic_limit"`
+	NicType        compute.TNicType     `json:"nic_type"`
 
 	// 是否为缺省路由网关
 	IsDefault bool `json:"is_default"`
@@ -141,8 +142,6 @@ type GuestnetworkJsonDesc struct {
 	Rate        int    `json:"rate"`
 	BaremetalId string `json:"baremetal_id"`
 
-	NicType compute.TNicType `json:"nic_type"`
-
 	LinkUp bool `json:"link_up"`
 }
 

pkg/apis/compute/host_const.go

@@ -65,9 +65,10 @@ const (
 	HOST_OFFLINE  = compute.HOST_OFFLINE
 	HOST_DISABLED = "offline"
 
-	NIC_TYPE_IPMI   = compute.NIC_TYPE_IPMI
-	NIC_TYPE_ADMIN  = compute.NIC_TYPE_ADMIN
-	NIC_TYPE_NORMAL = compute.NIC_TYPE_NORMAL
+	NIC_TYPE_IPMI       = compute.NIC_TYPE_IPMI
+	NIC_TYPE_ADMIN      = compute.NIC_TYPE_ADMIN
+	NIC_TYPE_NORMAL     = compute.NIC_TYPE_NORMAL
+	NIC_TYPE_INFINIBAND = compute.TNicType("infiniband")
 
 	BAREMETAL_INIT           = "init"
 	BAREMETAL_PREPARE        = "prepare"

pkg/apis/compute/isolated_device.go

@@ -122,6 +122,7 @@ type IsolatedDeviceJsonDesc struct {
 	VendorDeviceId      string `json:"vendor_device_id"`
 	Vendor              string `json:"vendor"`
 	NetworkIndex        int8   `json:"network_index"`
+	IsInfinibandNic     bool   `json:"is_infiniband_nic"`
 	OvsOffloadInterface string `json:"ovs_offload_interface"`
 	DiskIndex           int8   `json:"disk_index"`
 	NvmeSizeMB          int    `json:"nvme_size_mb"`

pkg/compute/models/guestnetworks.go

@@ -626,9 +626,22 @@ func (gn *SGuestnetwork) getJsonDesc() *api.GuestnetworkJsonDesc {
 	desc.TeamWith = gn.TeamWith
 
 	guest := gn.getGuest()
-	if guest.GetHypervisor() != api.HYPERVISOR_KVM || gn.IsSriovWithoutOffload() {
+	if guest.GetHypervisor() != api.HYPERVISOR_KVM {
 		manual := true
 		desc.Manual = &manual
+	} else {
+		if gn.Driver == api.NETWORK_DRIVER_VFIO {
+			dev, _ := gn.GetIsolatedDevice()
+			if dev != nil {
+				if dev.OvsOffloadInterface == "" {
+					manual := true
+					desc.Manual = &manual
+				}
+				if dev.IsInfinibandNic {
+					desc.NicType = api.NIC_TYPE_INFINIBAND
+				}
+			}
+		}
 	}
 
 	return desc

pkg/compute/models/isolated_devices.go

@@ -102,6 +102,8 @@ type SIsolatedDevice struct {
 	WireId string `width:"36" charset:"ascii" nullable:"true" index:"true" list:"domain" update:"domain" create:"domain_optional"`
 	// Offload interface name
 	OvsOffloadInterface string `width:"16" charset:"ascii" nullable:"true" list:"domain" update:"domain" create:"domain_optional"`
+	// Is infiniband nic
+	IsInfinibandNic bool `nullable:"false" default:"false" list:"user" create:"optional"`
 	// NVME disk size
 	NvmeSizeMB int `nullable:"true" list:"domain" update:"domain" create:"domain_optional"`
 	// guest disk index

pkg/hostman/guestfs/fsdriver/linux.go

@@ -56,6 +56,8 @@ const (
 var (
 	NetDevPrefix   = "eth"
 	NetDevPrefixEN = "en"
+
+	IBNetDevPrefix = "ib"
 )
 
 func GetNetDevPrefix(nics []*types.SServerNic) string {
@@ -66,6 +68,10 @@ func GetNetDevPrefix(nics []*types.SServerNic) string {
 	}
 }
 
+func GetIBNetDevPrefix() string {
+	return IBNetDevPrefix
+}
+
 func NicsHasDifferentDriver(nics []*types.SServerNic) bool {
 	m := make(map[string]int)
 	for i := 0; i < len(nics); i++ {
@@ -397,10 +403,13 @@ func (l *sLinuxRootFs) DeployNetworkingScripts(rootFs IDiskPartition, nics []*ty
 		for _, nic := range nics {
 			nicRules += `KERNEL=="*", SUBSYSTEM=="net", ACTION=="add", `
 			nicRules += `DRIVERS=="?*", `
-			mac := nic.Mac
-			nicRules += fmt.Sprintf(`ATTR{address}=="%s", ATTR{type}=="1", `, strings.ToLower(mac))
-			idx := nic.Index
-			nicRules += fmt.Sprintf("NAME=\"%s%d\"\n", netDevPrefix, idx)
+			if nic.NicType == api.NIC_TYPE_INFINIBAND {
+				nicRules += fmt.Sprintf(`ATTR{address}=="?*%s", ATTR{type}=="32", `, strings.ToLower(nic.Mac))
+				nicRules += fmt.Sprintf("NAME=\"%s%d\"\n", GetIBNetDevPrefix(), nic.Index)
+			} else {
+				nicRules += fmt.Sprintf(`ATTR{address}=="%s", ATTR{type}=="1", `, strings.ToLower(nic.Mac))
+				nicRules += fmt.Sprintf("NAME=\"%s%d\"\n", netDevPrefix, nic.Index)
+			}
 		}
 		if err := rootFs.FilePutContents(path.Join(udevPath, "70-persistent-net.rules"), nicRules, false, false); err != nil {
 			return err
@@ -1348,7 +1357,7 @@ func (r *sRedhatLikeRootFs) deployNetworkingScripts(rootFs IDiskPartition, nics
 		if nicDesc.Mtu > 0 {
 			cmds.WriteString(fmt.Sprintf("MTU=%d\n", nicDesc.Mtu))
 		}
-		if len(nicDesc.Mac) > 0 {
+		if len(nicDesc.Mac) > 0 && nicDesc.NicType != api.NIC_TYPE_INFINIBAND {
 			cmds.WriteString("HWADDR=")
 			cmds.WriteString(nicDesc.Mac)
 			cmds.WriteString("\n")

pkg/hostman/guestfs/fsdriver/nicteaming.go

@@ -20,6 +20,7 @@ import (
 	"yunion.io/x/cloudmux/pkg/apis/compute"
 	"yunion.io/x/jsonutils"
 
+	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/types"
 	deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
 )
@@ -96,7 +97,11 @@ func convertNicConfigs(nics []*types.SServerNic) ([]*types.SServerNic, []*types.
 		if teamNic == nil {
 			// no teaming nic
 			nnic := nics[i]
-			nnic.Name = fmt.Sprintf("%s%d", netDevPrefix, nnic.Index)
+			if nnic.NicType == computeapi.NIC_TYPE_INFINIBAND {
+				nnic.Name = fmt.Sprintf("%s%d", GetIBNetDevPrefix(), nnic.Index)
+			} else {
+				nnic.Name = fmt.Sprintf("%s%d", netDevPrefix, nnic.Index)
+			}
 			allNics = append(allNics, nnic)
 			continue
 		}

pkg/hostman/guestman/qemu-kvm.go

@@ -3047,22 +3047,19 @@ func getVfVlan(vlan int) int {
 	return vlan
 }
 
+func getIbNodeMac(mac string) string {
+	return "00:01:" + mac
+}
+
+func getIbPortMac(mac string) string {
+	return "00:10:" + mac
+}
+
 func (s *SKVMGuestInstance) sriovNicAttachInitScript(networkIndex int8, dev isolated_device.IDevice) (string, error) {
 	for i := range s.Desc.Nics {
 		if s.Desc.Nics[i].Driver == "vfio-pci" && s.Desc.Nics[i].Index == networkIndex {
-			if dev.GetOvsOffloadInterfaceName() != "" {
-				cmd := fmt.Sprintf("ip link set dev %s vf %d mac %s max_tx_rate %d\n",
-					dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac, s.Desc.Nics[i].Bw)
-				cmd += s.getNicUpScriptPath(s.Desc.Nics[i]) + "\n"
-				return cmd, nil
-			} else {
-				cmd := fmt.Sprintf(
-					"sriov_vf_init %s %d %s %d %s %d\n",
-					dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac,
-					getVfVlan(s.Desc.Nics[i].Vlan), srcMacCheckFunc(s.Desc.SrcMacCheck), s.Desc.Nics[i].Bw,
-				)
-				return sriovInitFunc + " && " + cmd, nil
-			}
+			cmd := s.generateSriovInitCmd(i, dev)
+			return sriovInitFunc + " && " + cmd, nil
 		}
 	}
 	return "", errors.Errorf("no nic found for index %d", networkIndex)
@@ -3077,26 +3074,36 @@ func (s *SKVMGuestInstance) generateSRIOVInitScripts() (string, error) {
 			if err != nil {
 				return "", err
 			}
-			if dev.GetOvsOffloadInterfaceName() != "" {
-				cmd += fmt.Sprintf("ip link set dev %s vf %d mac %s max_tx_rate %d\n",
-					dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac, s.Desc.Nics[i].Bw)
-				cmd += s.getNicUpScriptPath(s.Desc.Nics[i]) + "\n"
-			} else {
-				cmd += fmt.Sprintf(
-					"sriov_vf_init %s %d %s %d %s %d\n",
-					dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac,
-					getVfVlan(s.Desc.Nics[i].Vlan), srcMacCheckFunc(s.Desc.SrcMacCheck), s.Desc.Nics[i].Bw,
-				)
-			}
+			cmd += s.generateSriovInitCmd(i, dev)
 		}
 	}
 	if len(cmd) > 0 {
 		cmd = sriovInitFunc + "\n" + cmd
 	}
-
 	return cmd, nil
 }
 
+func (s *SKVMGuestInstance) generateSriovInitCmd(i int, dev isolated_device.IDevice) string {
+	var cmd = ""
+	if dev.GetOvsOffloadInterfaceName() != "" {
+		cmd += fmt.Sprintf("ip link set dev %s vf %d mac %s max_tx_rate %d\n",
+			dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac, s.Desc.Nics[i].Bw)
+		cmd += s.getNicUpScriptPath(s.Desc.Nics[i]) + "\n"
+	} else if dev.IsInfinibandNic() {
+		sriovPath := path.Join("/sys/class/net", dev.GetPfName(), "device/sriov", strconv.Itoa(dev.GetVirtfn()))
+		cmd = fmt.Sprintf("echo Follow > %s/policy\n", sriovPath)
+		cmd += fmt.Sprintf("echo %s > %s/node\n", getIbNodeMac(s.Desc.Nics[i].Mac), sriovPath)
+		cmd += fmt.Sprintf("echo %s > %s/port\n", getIbPortMac(s.Desc.Nics[i].Mac), sriovPath)
+	} else {
+		cmd += fmt.Sprintf(
+			"sriov_vf_init %s %d %s %d %s %d\n",
+			dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac,
+			getVfVlan(s.Desc.Nics[i].Vlan), srcMacCheckFunc(s.Desc.SrcMacCheck), s.Desc.Nics[i].Bw,
+		)
+	}
+	return cmd
+}
+
 func (s *SKVMGuestInstance) reconfigureVfioNicsBandwidth(nicDesc *desc.SGuestNetwork) error {
 	if nicDesc.Driver == "vfio-pci" {
 		dev, err := s.GetSriovDeviceByNetworkIndex(nicDesc.Index)

pkg/hostman/hostinfo/hostinfo.go

@@ -2002,12 +2002,21 @@ func (h *SHostInfo) getNicsInterfaces(nics []string) []isolated_device.HostNic {
 	if len(nics) == 0 {
 		return nil
 	}
+	log.Infof("sriov input nics %v", nics)
 	res := []isolated_device.HostNic{}
-	for i := 0; i < len(h.Nics); i++ {
-		if utils.IsInStringArray(h.Nics[i].Inter, nics) {
-			res = append(res, isolated_device.HostNic{h.Nics[i].Bridge, h.Nics[i].Inter, h.Nics[i].WireId})
+	for i := 0; i < len(nics); i++ {
+		found := false
+		for j := 0; j < len(h.Nics); j++ {
+			if nics[i] == h.Nics[j].Inter {
+				found = true
+				res = append(res, isolated_device.HostNic{h.Nics[j].Bridge, h.Nics[j].Inter, h.Nics[j].WireId})
+			}
+		}
+		if !found {
+			res = append(res, isolated_device.HostNic{h.Nics[0].Bridge, nics[i], h.Nics[0].WireId})
 		}
 	}
+	log.Infof("sriov output nics %v", res)
 	return res
 }
 

pkg/hostman/isolated_device/isolated_device.go

@@ -71,6 +71,7 @@ type IDevice interface {
 	SetHostId(hId string)
 	GetGuestId() string
 	GetWireId() string
+	IsInfinibandNic() bool
 	GetOvsOffloadInterfaceName() string
 	GetVendorDeviceId() string
 	GetAddr() string
@@ -506,6 +507,10 @@ func (dev *sBaseDevice) GetOvsOffloadInterfaceName() string {
 	return ""
 }
 
+func (dev *sBaseDevice) IsInfinibandNic() bool {
+	return false
+}
+
 func (dev *sBaseDevice) GetNVMESizeMB() int {
 	return -1
 }
@@ -554,6 +559,9 @@ func GetApiResourceData(dev IDevice) *jsonutils.JSONDict {
 	if len(dev.GetWireId()) != 0 {
 		data["wire_id"] = dev.GetWireId()
 	}
+	if dev.IsInfinibandNic() {
+		data["is_infiniband_nic"] = true
+	}
 	if len(dev.GetOvsOffloadInterfaceName()) != 0 {
 		data["ovs_offload_interface"] = dev.GetOvsOffloadInterfaceName()
 	}

pkg/hostman/isolated_device/nic.go

@@ -56,6 +56,17 @@ func getSRIOVNics(hostNics []HostNic) ([]*sSRIOVNicDevice, error) {
 	log.Infof("host nics %s detected support sriov nics %v", hostNics, nics)
 	sriovNics := make([]*sSRIOVNicDevice, 0)
 	for i := 0; i < len(nics); i++ {
+		nicType, err := fileutils2.FileGetContents(path.Join(sysfsNetDir, nics[i].Interface, "type"))
+		if err != nil {
+			return nil, errors.Wrap(err, "failed get nic type")
+		}
+		var isInfinibandNic = false
+		if strings.TrimSpace(nicType) == "32" {
+			// include/uapi/linux/if_arp.h
+			// #define ARPHRD_INFINIBAND 32		/* InfiniBand			*/
+			isInfinibandNic = true
+		}
+
 		nicDir := path.Join(sysfsNetDir, nics[i].Interface, "device")
 		err = ensureNumvfsEqualTotalvfs(nicDir)
 		if err != nil {
@@ -80,34 +91,40 @@ func getSRIOVNics(hostNics []HostNic) ([]*sSRIOVNicDevice, error) {
 				if err != nil {
 					return nil, err
 				}
-				sriovNics = append(sriovNics, NewSRIOVNicDevice(vfDev, api.NIC_TYPE, nics[i].Wire, nics[i].Interface, virtfn))
+				sriovNics = append(sriovNics, NewSRIOVNicDevice(vfDev, api.NIC_TYPE, nics[i].Wire, nics[i].Interface, virtfn, isInfinibandNic))
 			}
 		}
 	}
 	return sriovNics, nil
 }
 
-func NewSRIOVNicDevice(dev *PCIDevice, devType, wireId, pfName string, virtfn int) *sSRIOVNicDevice {
+func NewSRIOVNicDevice(dev *PCIDevice, devType, wireId, pfName string, virtfn int, isInfinibandNic bool) *sSRIOVNicDevice {
 	return &sSRIOVNicDevice{
 		sSRIOVBaseDevice: newSRIOVBaseDevice(dev, devType),
 		WireId:           wireId,
 		pfName:           pfName,
 		virtfn:           virtfn,
+		isInfinibandNic:  isInfinibandNic,
 	}
 }
 
 type sSRIOVNicDevice struct {
 	*sSRIOVBaseDevice
 
-	WireId string
-	pfName string
-	virtfn int
+	WireId          string
+	pfName          string
+	virtfn          int
+	isInfinibandNic bool
 }
 
 func (dev *sSRIOVNicDevice) GetPfName() string {
 	return dev.pfName
 }
 
+func (dev *sSRIOVNicDevice) IsInfinibandNic() bool {
+	return dev.isInfinibandNic
+}
+
 func (dev *sSRIOVNicDevice) GetVirtfn() int {
 	return dev.virtfn
 }
@@ -270,7 +287,7 @@ type sOvsOffloadNicDevice struct {
 
 func NewSRIOVOffloadNicDevice(dev *PCIDevice, devType, wireId, pfName string, virtfn int, ifname string) *sOvsOffloadNicDevice {
 	return &sOvsOffloadNicDevice{
-		sSRIOVNicDevice: NewSRIOVNicDevice(dev, devType, wireId, pfName, virtfn),
+		sSRIOVNicDevice: NewSRIOVNicDevice(dev, devType, wireId, pfName, virtfn, false),
 		interfaceName:   ifname,
 	}
 }

pkg/hostman/isolated_device/nvidia_vgpu.go

@@ -45,6 +45,10 @@ func (dev *sNVIDIAVgpuDevice) String() string {
 	return jsonutils.Marshal(dev).String()
 }
 
+func (dev *sNVIDIAVgpuDevice) IsInfinibandNic() bool {
+	return false
+}
+
 func (dev *sNVIDIAVgpuDevice) GetCloudId() string {
 	return dev.cloudId
 }