diff --git a/pkg/apis/compute/guestnetwork.go b/pkg/apis/compute/guestnetwork.go index ed7eff5b1c..b80ef766f9 100644 --- a/pkg/apis/compute/guestnetwork.go +++ b/pkg/apis/compute/guestnetwork.go @@ -104,6 +104,7 @@ type GuestnetworkBaseDesc struct { Index int8 `json:"index"` RxTrafficLimit int64 `json:"rx_traffic_limit"` TxTrafficLimit int64 `json:"tx_traffic_limit"` + NicType compute.TNicType `json:"nic_type"` // 是否为缺省路由网关 IsDefault bool `json:"is_default"` @@ -141,8 +142,6 @@ type GuestnetworkJsonDesc struct { Rate int `json:"rate"` BaremetalId string `json:"baremetal_id"` - NicType compute.TNicType `json:"nic_type"` - LinkUp bool `json:"link_up"` } diff --git a/pkg/apis/compute/host_const.go b/pkg/apis/compute/host_const.go index 6730e30226..9d5ea4a450 100644 --- a/pkg/apis/compute/host_const.go +++ b/pkg/apis/compute/host_const.go @@ -65,9 +65,10 @@ const ( HOST_OFFLINE = compute.HOST_OFFLINE HOST_DISABLED = "offline" - NIC_TYPE_IPMI = compute.NIC_TYPE_IPMI - NIC_TYPE_ADMIN = compute.NIC_TYPE_ADMIN - NIC_TYPE_NORMAL = compute.NIC_TYPE_NORMAL + NIC_TYPE_IPMI = compute.NIC_TYPE_IPMI + NIC_TYPE_ADMIN = compute.NIC_TYPE_ADMIN + NIC_TYPE_NORMAL = compute.NIC_TYPE_NORMAL + NIC_TYPE_INFINIBAND = compute.TNicType("infiniband") BAREMETAL_INIT = "init" BAREMETAL_PREPARE = "prepare" diff --git a/pkg/apis/compute/isolated_device.go b/pkg/apis/compute/isolated_device.go index 6145f5f906..91533d66c9 100644 --- a/pkg/apis/compute/isolated_device.go +++ b/pkg/apis/compute/isolated_device.go @@ -122,6 +122,7 @@ type IsolatedDeviceJsonDesc struct { VendorDeviceId string `json:"vendor_device_id"` Vendor string `json:"vendor"` NetworkIndex int8 `json:"network_index"` + IsInfinibandNic bool `json:"is_infiniband_nic"` OvsOffloadInterface string `json:"ovs_offload_interface"` DiskIndex int8 `json:"disk_index"` NvmeSizeMB int `json:"nvme_size_mb"` diff --git a/pkg/compute/models/guestnetworks.go b/pkg/compute/models/guestnetworks.go index 86e4d851e5..bc9067478e 100644 --- a/pkg/compute/models/guestnetworks.go +++ b/pkg/compute/models/guestnetworks.go @@ -626,9 +626,22 @@ func (gn *SGuestnetwork) getJsonDesc() *api.GuestnetworkJsonDesc { desc.TeamWith = gn.TeamWith guest := gn.getGuest() - if guest.GetHypervisor() != api.HYPERVISOR_KVM || gn.IsSriovWithoutOffload() { + if guest.GetHypervisor() != api.HYPERVISOR_KVM { manual := true desc.Manual = &manual + } else { + if gn.Driver == api.NETWORK_DRIVER_VFIO { + dev, _ := gn.GetIsolatedDevice() + if dev != nil { + if dev.OvsOffloadInterface == "" { + manual := true + desc.Manual = &manual + } + if dev.IsInfinibandNic { + desc.NicType = api.NIC_TYPE_INFINIBAND + } + } + } } return desc diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go index 5a54431557..e70acfa8da 100644 --- a/pkg/compute/models/isolated_devices.go +++ b/pkg/compute/models/isolated_devices.go @@ -102,6 +102,8 @@ type SIsolatedDevice struct { WireId string `width:"36" charset:"ascii" nullable:"true" index:"true" list:"domain" update:"domain" create:"domain_optional"` // Offload interface name OvsOffloadInterface string `width:"16" charset:"ascii" nullable:"true" list:"domain" update:"domain" create:"domain_optional"` + // Is infiniband nic + IsInfinibandNic bool `nullable:"false" default:"false" list:"user" create:"optional"` // NVME disk size NvmeSizeMB int `nullable:"true" list:"domain" update:"domain" create:"domain_optional"` // guest disk index diff --git a/pkg/hostman/guestfs/fsdriver/linux.go b/pkg/hostman/guestfs/fsdriver/linux.go index 3b9d82da95..0e89bc4b72 100644 --- a/pkg/hostman/guestfs/fsdriver/linux.go +++ b/pkg/hostman/guestfs/fsdriver/linux.go @@ -56,6 +56,8 @@ const ( var ( NetDevPrefix = "eth" NetDevPrefixEN = "en" + + IBNetDevPrefix = "ib" ) func GetNetDevPrefix(nics []*types.SServerNic) string { @@ -66,6 +68,10 @@ func GetNetDevPrefix(nics []*types.SServerNic) string { } } +func GetIBNetDevPrefix() string { + return IBNetDevPrefix +} + func NicsHasDifferentDriver(nics []*types.SServerNic) bool { m := make(map[string]int) for i := 0; i < len(nics); i++ { @@ -397,10 +403,13 @@ func (l *sLinuxRootFs) DeployNetworkingScripts(rootFs IDiskPartition, nics []*ty for _, nic := range nics { nicRules += `KERNEL=="*", SUBSYSTEM=="net", ACTION=="add", ` nicRules += `DRIVERS=="?*", ` - mac := nic.Mac - nicRules += fmt.Sprintf(`ATTR{address}=="%s", ATTR{type}=="1", `, strings.ToLower(mac)) - idx := nic.Index - nicRules += fmt.Sprintf("NAME=\"%s%d\"\n", netDevPrefix, idx) + if nic.NicType == api.NIC_TYPE_INFINIBAND { + nicRules += fmt.Sprintf(`ATTR{address}=="?*%s", ATTR{type}=="32", `, strings.ToLower(nic.Mac)) + nicRules += fmt.Sprintf("NAME=\"%s%d\"\n", GetIBNetDevPrefix(), nic.Index) + } else { + nicRules += fmt.Sprintf(`ATTR{address}=="%s", ATTR{type}=="1", `, strings.ToLower(nic.Mac)) + nicRules += fmt.Sprintf("NAME=\"%s%d\"\n", netDevPrefix, nic.Index) + } } if err := rootFs.FilePutContents(path.Join(udevPath, "70-persistent-net.rules"), nicRules, false, false); err != nil { return err @@ -1348,7 +1357,7 @@ func (r *sRedhatLikeRootFs) deployNetworkingScripts(rootFs IDiskPartition, nics if nicDesc.Mtu > 0 { cmds.WriteString(fmt.Sprintf("MTU=%d\n", nicDesc.Mtu)) } - if len(nicDesc.Mac) > 0 { + if len(nicDesc.Mac) > 0 && nicDesc.NicType != api.NIC_TYPE_INFINIBAND { cmds.WriteString("HWADDR=") cmds.WriteString(nicDesc.Mac) cmds.WriteString("\n") diff --git a/pkg/hostman/guestfs/fsdriver/nicteaming.go b/pkg/hostman/guestfs/fsdriver/nicteaming.go index 481e5657b2..5221e238b7 100644 --- a/pkg/hostman/guestfs/fsdriver/nicteaming.go +++ b/pkg/hostman/guestfs/fsdriver/nicteaming.go @@ -20,6 +20,7 @@ import ( "yunion.io/x/cloudmux/pkg/apis/compute" "yunion.io/x/jsonutils" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/types" deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis" ) @@ -96,7 +97,11 @@ func convertNicConfigs(nics []*types.SServerNic) ([]*types.SServerNic, []*types. if teamNic == nil { // no teaming nic nnic := nics[i] - nnic.Name = fmt.Sprintf("%s%d", netDevPrefix, nnic.Index) + if nnic.NicType == computeapi.NIC_TYPE_INFINIBAND { + nnic.Name = fmt.Sprintf("%s%d", GetIBNetDevPrefix(), nnic.Index) + } else { + nnic.Name = fmt.Sprintf("%s%d", netDevPrefix, nnic.Index) + } allNics = append(allNics, nnic) continue } diff --git a/pkg/hostman/guestman/qemu-kvm.go b/pkg/hostman/guestman/qemu-kvm.go index f12b2ed5be..5b71a0f4ff 100644 --- a/pkg/hostman/guestman/qemu-kvm.go +++ b/pkg/hostman/guestman/qemu-kvm.go @@ -3047,22 +3047,19 @@ func getVfVlan(vlan int) int { return vlan } +func getIbNodeMac(mac string) string { + return "00:01:" + mac +} + +func getIbPortMac(mac string) string { + return "00:10:" + mac +} + func (s *SKVMGuestInstance) sriovNicAttachInitScript(networkIndex int8, dev isolated_device.IDevice) (string, error) { for i := range s.Desc.Nics { if s.Desc.Nics[i].Driver == "vfio-pci" && s.Desc.Nics[i].Index == networkIndex { - if dev.GetOvsOffloadInterfaceName() != "" { - cmd := fmt.Sprintf("ip link set dev %s vf %d mac %s max_tx_rate %d\n", - dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac, s.Desc.Nics[i].Bw) - cmd += s.getNicUpScriptPath(s.Desc.Nics[i]) + "\n" - return cmd, nil - } else { - cmd := fmt.Sprintf( - "sriov_vf_init %s %d %s %d %s %d\n", - dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac, - getVfVlan(s.Desc.Nics[i].Vlan), srcMacCheckFunc(s.Desc.SrcMacCheck), s.Desc.Nics[i].Bw, - ) - return sriovInitFunc + " && " + cmd, nil - } + cmd := s.generateSriovInitCmd(i, dev) + return sriovInitFunc + " && " + cmd, nil } } return "", errors.Errorf("no nic found for index %d", networkIndex) @@ -3077,26 +3074,36 @@ func (s *SKVMGuestInstance) generateSRIOVInitScripts() (string, error) { if err != nil { return "", err } - if dev.GetOvsOffloadInterfaceName() != "" { - cmd += fmt.Sprintf("ip link set dev %s vf %d mac %s max_tx_rate %d\n", - dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac, s.Desc.Nics[i].Bw) - cmd += s.getNicUpScriptPath(s.Desc.Nics[i]) + "\n" - } else { - cmd += fmt.Sprintf( - "sriov_vf_init %s %d %s %d %s %d\n", - dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac, - getVfVlan(s.Desc.Nics[i].Vlan), srcMacCheckFunc(s.Desc.SrcMacCheck), s.Desc.Nics[i].Bw, - ) - } + cmd += s.generateSriovInitCmd(i, dev) } } if len(cmd) > 0 { cmd = sriovInitFunc + "\n" + cmd } - return cmd, nil } +func (s *SKVMGuestInstance) generateSriovInitCmd(i int, dev isolated_device.IDevice) string { + var cmd = "" + if dev.GetOvsOffloadInterfaceName() != "" { + cmd += fmt.Sprintf("ip link set dev %s vf %d mac %s max_tx_rate %d\n", + dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac, s.Desc.Nics[i].Bw) + cmd += s.getNicUpScriptPath(s.Desc.Nics[i]) + "\n" + } else if dev.IsInfinibandNic() { + sriovPath := path.Join("/sys/class/net", dev.GetPfName(), "device/sriov", strconv.Itoa(dev.GetVirtfn())) + cmd = fmt.Sprintf("echo Follow > %s/policy\n", sriovPath) + cmd += fmt.Sprintf("echo %s > %s/node\n", getIbNodeMac(s.Desc.Nics[i].Mac), sriovPath) + cmd += fmt.Sprintf("echo %s > %s/port\n", getIbPortMac(s.Desc.Nics[i].Mac), sriovPath) + } else { + cmd += fmt.Sprintf( + "sriov_vf_init %s %d %s %d %s %d\n", + dev.GetPfName(), dev.GetVirtfn(), s.Desc.Nics[i].Mac, + getVfVlan(s.Desc.Nics[i].Vlan), srcMacCheckFunc(s.Desc.SrcMacCheck), s.Desc.Nics[i].Bw, + ) + } + return cmd +} + func (s *SKVMGuestInstance) reconfigureVfioNicsBandwidth(nicDesc *desc.SGuestNetwork) error { if nicDesc.Driver == "vfio-pci" { dev, err := s.GetSriovDeviceByNetworkIndex(nicDesc.Index) diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index 0fc7799d85..935e9a4d30 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -2002,12 +2002,21 @@ func (h *SHostInfo) getNicsInterfaces(nics []string) []isolated_device.HostNic { if len(nics) == 0 { return nil } + log.Infof("sriov input nics %v", nics) res := []isolated_device.HostNic{} - for i := 0; i < len(h.Nics); i++ { - if utils.IsInStringArray(h.Nics[i].Inter, nics) { - res = append(res, isolated_device.HostNic{h.Nics[i].Bridge, h.Nics[i].Inter, h.Nics[i].WireId}) + for i := 0; i < len(nics); i++ { + found := false + for j := 0; j < len(h.Nics); j++ { + if nics[i] == h.Nics[j].Inter { + found = true + res = append(res, isolated_device.HostNic{h.Nics[j].Bridge, h.Nics[j].Inter, h.Nics[j].WireId}) + } + } + if !found { + res = append(res, isolated_device.HostNic{h.Nics[0].Bridge, nics[i], h.Nics[0].WireId}) } } + log.Infof("sriov output nics %v", res) return res } diff --git a/pkg/hostman/isolated_device/isolated_device.go b/pkg/hostman/isolated_device/isolated_device.go index ae5e53329b..b1cc32219e 100644 --- a/pkg/hostman/isolated_device/isolated_device.go +++ b/pkg/hostman/isolated_device/isolated_device.go @@ -71,6 +71,7 @@ type IDevice interface { SetHostId(hId string) GetGuestId() string GetWireId() string + IsInfinibandNic() bool GetOvsOffloadInterfaceName() string GetVendorDeviceId() string GetAddr() string @@ -506,6 +507,10 @@ func (dev *sBaseDevice) GetOvsOffloadInterfaceName() string { return "" } +func (dev *sBaseDevice) IsInfinibandNic() bool { + return false +} + func (dev *sBaseDevice) GetNVMESizeMB() int { return -1 } @@ -554,6 +559,9 @@ func GetApiResourceData(dev IDevice) *jsonutils.JSONDict { if len(dev.GetWireId()) != 0 { data["wire_id"] = dev.GetWireId() } + if dev.IsInfinibandNic() { + data["is_infiniband_nic"] = true + } if len(dev.GetOvsOffloadInterfaceName()) != 0 { data["ovs_offload_interface"] = dev.GetOvsOffloadInterfaceName() } diff --git a/pkg/hostman/isolated_device/nic.go b/pkg/hostman/isolated_device/nic.go index dbc56cf445..6c76d80a43 100644 --- a/pkg/hostman/isolated_device/nic.go +++ b/pkg/hostman/isolated_device/nic.go @@ -56,6 +56,17 @@ func getSRIOVNics(hostNics []HostNic) ([]*sSRIOVNicDevice, error) { log.Infof("host nics %s detected support sriov nics %v", hostNics, nics) sriovNics := make([]*sSRIOVNicDevice, 0) for i := 0; i < len(nics); i++ { + nicType, err := fileutils2.FileGetContents(path.Join(sysfsNetDir, nics[i].Interface, "type")) + if err != nil { + return nil, errors.Wrap(err, "failed get nic type") + } + var isInfinibandNic = false + if strings.TrimSpace(nicType) == "32" { + // include/uapi/linux/if_arp.h + // #define ARPHRD_INFINIBAND 32 /* InfiniBand */ + isInfinibandNic = true + } + nicDir := path.Join(sysfsNetDir, nics[i].Interface, "device") err = ensureNumvfsEqualTotalvfs(nicDir) if err != nil { @@ -80,34 +91,40 @@ func getSRIOVNics(hostNics []HostNic) ([]*sSRIOVNicDevice, error) { if err != nil { return nil, err } - sriovNics = append(sriovNics, NewSRIOVNicDevice(vfDev, api.NIC_TYPE, nics[i].Wire, nics[i].Interface, virtfn)) + sriovNics = append(sriovNics, NewSRIOVNicDevice(vfDev, api.NIC_TYPE, nics[i].Wire, nics[i].Interface, virtfn, isInfinibandNic)) } } } return sriovNics, nil } -func NewSRIOVNicDevice(dev *PCIDevice, devType, wireId, pfName string, virtfn int) *sSRIOVNicDevice { +func NewSRIOVNicDevice(dev *PCIDevice, devType, wireId, pfName string, virtfn int, isInfinibandNic bool) *sSRIOVNicDevice { return &sSRIOVNicDevice{ sSRIOVBaseDevice: newSRIOVBaseDevice(dev, devType), WireId: wireId, pfName: pfName, virtfn: virtfn, + isInfinibandNic: isInfinibandNic, } } type sSRIOVNicDevice struct { *sSRIOVBaseDevice - WireId string - pfName string - virtfn int + WireId string + pfName string + virtfn int + isInfinibandNic bool } func (dev *sSRIOVNicDevice) GetPfName() string { return dev.pfName } +func (dev *sSRIOVNicDevice) IsInfinibandNic() bool { + return dev.isInfinibandNic +} + func (dev *sSRIOVNicDevice) GetVirtfn() int { return dev.virtfn } @@ -270,7 +287,7 @@ type sOvsOffloadNicDevice struct { func NewSRIOVOffloadNicDevice(dev *PCIDevice, devType, wireId, pfName string, virtfn int, ifname string) *sOvsOffloadNicDevice { return &sOvsOffloadNicDevice{ - sSRIOVNicDevice: NewSRIOVNicDevice(dev, devType, wireId, pfName, virtfn), + sSRIOVNicDevice: NewSRIOVNicDevice(dev, devType, wireId, pfName, virtfn, false), interfaceName: ifname, } } diff --git a/pkg/hostman/isolated_device/nvidia_vgpu.go b/pkg/hostman/isolated_device/nvidia_vgpu.go index a5eec31dba..dbb1d5fbb3 100644 --- a/pkg/hostman/isolated_device/nvidia_vgpu.go +++ b/pkg/hostman/isolated_device/nvidia_vgpu.go @@ -45,6 +45,10 @@ func (dev *sNVIDIAVgpuDevice) String() string { return jsonutils.Marshal(dev).String() } +func (dev *sNVIDIAVgpuDevice) IsInfinibandNic() bool { + return false +} + func (dev *sNVIDIAVgpuDevice) GetCloudId() string { return dev.cloudId }