diff --git a/cmd/virt-prerunner/main.go b/cmd/virt-prerunner/main.go index d11248c..63516fb 100644 --- a/cmd/virt-prerunner/main.go +++ b/cmd/virt-prerunner/main.go @@ -252,6 +252,18 @@ func buildVMConfig(ctx context.Context, vm *virtv1alpha1.VirtualMachine) (*cloud vmConfig.Devices = append(vmConfig.Devices, &sriovDeviceConfig) } } + case iface.VDPA != nil: + for _, networkStatus := range networkStatusList { + if networkStatus.Interface == linkName && networkStatus.DeviceInfo != nil && networkStatus.DeviceInfo.Vdpa != nil { + vdpaDeviceConfig := cloudhypervisor.VdpaConfig{ + Id: iface.Name, + NumQueues: iface.VDPA.NumQueues, + Iommu: iface.VDPA.IOMMU, + Path: networkStatus.DeviceInfo.Vdpa.Path, + } + vmConfig.Vdpa = append(vmConfig.Vdpa, &vdpaDeviceConfig) + } + } case iface.VhostUser != nil: netConfig := cloudhypervisor.NetConfig{ Id: iface.Name, diff --git a/deploy/crd/virt.virtink.smartx.com_virtualmachines.yaml b/deploy/crd/virt.virtink.smartx.com_virtualmachines.yaml index 9d1117f..a3e2604 100644 --- a/deploy/crd/virt.virtink.smartx.com_virtualmachines.yaml +++ b/deploy/crd/virt.virtink.smartx.com_virtualmachines.yaml @@ -910,6 +910,13 @@ spec: type: string sriov: type: object + vdpa: + properties: + iommu: + type: boolean + numQueues: + type: integer + type: object vhostUser: type: object required: diff --git a/docs/interfaces_and_networks.md b/docs/interfaces_and_networks.md index ba04321..1c6b13f 100644 --- a/docs/interfaces_and_networks.md +++ b/docs/interfaces_and_networks.md @@ -83,6 +83,7 @@ Each interface should declare its type by defining one of the following fields: | `bridge` | Connect using a linux bridge | | `masquerade` | Connect using iptables rules to NAT the traffic | | `sriov` | Passthrough a SR-IOV PCI device via VFIO | +| `vdpa` | Add a vhost-vdpa device | Each interface may also have additional configuration fields that modify properties "seen" inside guest instances, as listed below: @@ -224,3 +225,63 @@ spec: multus: networkName: mellanox-sriov-25g ``` +### `vdpa` Mode +Similar to the sriov mode, the following components need to be installed: +- [Multus CNI](https://github.com/k8snetworkplumbingwg/multus-cni) +- A vDPA device plugin +- A vDPA CNI Plugin + +If your hardware vendor support vDPA over SR-IOV, you can use + the [SR-IOV Network Device Plugin](https://github.com/k8snetworkplumbingwg/sriov-network-device-plugin), here is a sample configuration: + ```yaml + apiVersion: v1 +kind: ConfigMap +metadata: + name: sriovdp-config + namespace: kube-system +data: + config.json: | + { + "resourceList": [ + { + "resourceName": "jaguar", + "resourcePrefix": "jaguarmicro.com", + "selectors": { + "vendors": ["1f53"], + "devices": ["1000"], + "drivers": ["jaguar"], + "vdpaType": "vhost" + } + } + ] + } + ``` + If you don't need complex network configuration, the changes to enable the SR-IOV CNI to also manage vDPA interfaces are in this repository: + + https://github.com/amorenoz/sriov-cni/tree/rfe/vdpa + +#### Start an vDPA VM + +To create a VM that will attach to the aforementioned network, refer to the following VM spec: + +```yaml +apiVersion: virt.virtink.smartx.com/v1alpha1 +kind: VirtualMachine +spec: + instance: + interfaces: + - name: pod + - name: vdpa1 + vdpa: { numQueues: 9, iommu: False } + - name: vdpa2 + vdpa: { numQueues: 9, iommu: False } + networks: + - name: pod + pod: {} + - name: vdpa1 + multus: + networkName: offload-ovn1 + - name: vdpa2 + multus: + networkName: offload-ovn2 +``` diff --git a/pkg/apis/virt/v1alpha1/types.go b/pkg/apis/virt/v1alpha1/types.go index 30765a4..59df5b1 100644 --- a/pkg/apis/virt/v1alpha1/types.go +++ b/pkg/apis/virt/v1alpha1/types.go @@ -102,6 +102,7 @@ type InterfaceBindingMethod struct { Bridge *InterfaceBridge `json:"bridge,omitempty"` Masquerade *InterfaceMasquerade `json:"masquerade,omitempty"` SRIOV *InterfaceSRIOV `json:"sriov,omitempty"` + VDPA *InterfaceVDPA `json:"vdpa,omitempty"` VhostUser *InterfaceVhostUser `json:"vhostUser,omitempty"` } @@ -115,6 +116,11 @@ type InterfaceMasquerade struct { type InterfaceSRIOV struct { } +type InterfaceVDPA struct { + NumQueues int `json:"numQueues,omitempty"` + IOMMU bool `json:"iommu,omitempty"` +} + type InterfaceVhostUser struct { } diff --git a/pkg/apis/virt/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/virt/v1alpha1/zz_generated.deepcopy.go index 5b80684..af88547 100644 --- a/pkg/apis/virt/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/virt/v1alpha1/zz_generated.deepcopy.go @@ -238,6 +238,11 @@ func (in *InterfaceBindingMethod) DeepCopyInto(out *InterfaceBindingMethod) { *out = new(InterfaceSRIOV) **out = **in } + if in.VDPA != nil { + in, out := &in.VDPA, &out.VDPA + *out = new(InterfaceVDPA) + **out = **in + } if in.VhostUser != nil { in, out := &in.VhostUser, &out.VhostUser *out = new(InterfaceVhostUser) @@ -304,6 +309,22 @@ func (in *InterfaceSRIOV) DeepCopy() *InterfaceSRIOV { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InterfaceVDPA) DeepCopyInto(out *InterfaceVDPA) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InterfaceVDPA. +func (in *InterfaceVDPA) DeepCopy() *InterfaceVDPA { + if in == nil { + return nil + } + out := new(InterfaceVDPA) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InterfaceVhostUser) DeepCopyInto(out *InterfaceVhostUser) { *out = *in diff --git a/pkg/controller/vm_webhook.go b/pkg/controller/vm_webhook.go index 16fe869..7c5f94f 100644 --- a/pkg/controller/vm_webhook.go +++ b/pkg/controller/vm_webhook.go @@ -165,7 +165,7 @@ func MutateVM(ctx context.Context, vm *virtv1alpha1.VirtualMachine, oldVM *virtv vm.Spec.Instance.Interfaces[i].MAC = macStr } - if vm.Spec.Instance.Interfaces[i].Bridge == nil && vm.Spec.Instance.Interfaces[i].Masquerade == nil && vm.Spec.Instance.Interfaces[i].SRIOV == nil && vm.Spec.Instance.Interfaces[i].VhostUser == nil { + if vm.Spec.Instance.Interfaces[i].Bridge == nil && vm.Spec.Instance.Interfaces[i].Masquerade == nil && vm.Spec.Instance.Interfaces[i].SRIOV == nil && vm.Spec.Instance.Interfaces[i].VDPA == nil && vm.Spec.Instance.Interfaces[i].VhostUser == nil { vm.Spec.Instance.Interfaces[i].InterfaceBindingMethod = virtv1alpha1.InterfaceBindingMethod{ Bridge: &virtv1alpha1.InterfaceBridge{}, } @@ -504,6 +504,12 @@ func ValidateInterfaceBindingMethod(ctx context.Context, bindingMethod *virtv1al errs = append(errs, field.Forbidden(fieldPath.Child("sriov"), "may not specify more than 1 binding method")) } } + if bindingMethod.VDPA != nil { + cnt++ + if cnt > 1 { + errs = append(errs, field.Forbidden(fieldPath.Child("vdpa"), "may not specify more than 1 binding method")) + } + } if bindingMethod.VhostUser != nil { cnt++ if cnt > 1 { diff --git a/pkg/controller/vm_webhook_test.go b/pkg/controller/vm_webhook_test.go index 963765a..684c220 100644 --- a/pkg/controller/vm_webhook_test.go +++ b/pkg/controller/vm_webhook_test.go @@ -365,6 +365,17 @@ func TestValidateVM(t *testing.T) { return vm }(), invalidFields: []string{"spec.instance.interfaces[0].sriov"}, + }, { + vm: func() *virtv1alpha1.VirtualMachine { + vm := validVM.DeepCopy() + vm.Spec.Instance.Interfaces[0].InterfaceBindingMethod.Bridge = nil + vm.Spec.Instance.Interfaces[0].InterfaceBindingMethod.VDPA = &virtv1alpha1.InterfaceVDPA{ + NumQueues: 9, + IOMMU: false, + } + return vm + }(), + invalidFields: []string{"spec.instance.interfaces[0].vdpa.numQueues", "spec.instance.interfaces[0].vdpa.iommu"}, }, { vm: func() *virtv1alpha1.VirtualMachine { vm := validVM.DeepCopy() diff --git a/pkg/daemon/vm_controller.go b/pkg/daemon/vm_controller.go index 8009dca..b679411 100644 --- a/pkg/daemon/vm_controller.go +++ b/pkg/daemon/vm_controller.go @@ -145,7 +145,7 @@ func (r *VMReconciler) reconcile(ctx context.Context, vm *virtv1alpha1.VirtualMa return err } - if len(vmConfig.Devices) > 0 { + if len(vmConfig.Devices) > 0 || len(vmConfig.Vdpa) > 0 { cloudHypervisorPID, err := pid.GetPIDBySocket(filepath.Join(getVMDataDirPath(vm), "ch.sock")) if err != nil { return fmt.Errorf("get cloud-hypervisor process pid: %s", err)