Skip to content

Commit 5eb1908

Browse files
capolrikcapolrik
committed
Adding the support of Linux bridge interface for VM setup
Use the --bridge option to create a NAD that will be attached to the VMs
1 parent 0d99424 commit 5eb1908

File tree

4 files changed

+131
-10
lines changed

4 files changed

+131
-10
lines changed

‎cmd/k8s-netperf/k8s-netperf.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ var (
4444
full bool
4545
vm bool
4646
debug bool
47+
bridge string
4748
promURL string
4849
id string
4950
searchURL string
@@ -182,6 +183,14 @@ var rootCmd = &cobra.Command{
182183
log.Error(err)
183184
}
184185
s.KClient = kclient
186+
s.DClient = dynClient
187+
if len(bridge) > 0 {
188+
err := k8s.DeployNADBridge(s.DClient, bridge)
189+
if err != nil {
190+
log.Error(err)
191+
}
192+
s.Bridge = true
193+
}
185194
}
186195

187196
// Build the SUT (Deployments)
@@ -415,6 +424,9 @@ func executeWorkload(nc config.Config,
415424
if err != nil {
416425
log.Fatal(err)
417426
}
427+
//when using a bridge, ip is static
428+
} else if s.Bridge {
429+
serverIP = "10.10.10.14"
418430
} else {
419431
if hostNet {
420432
serverIP = s.ServerHost.Items[0].Status.PodIP
@@ -513,6 +525,7 @@ func main() {
513525
rootCmd.Flags().BoolVar(&full, "all", false, "Run all tests scenarios - hostNet and podNetwork (if possible)")
514526
rootCmd.Flags().BoolVar(&debug, "debug", false, "Enable debug log")
515527
rootCmd.Flags().BoolVar(&udn, "udn", false, "Create and use a UDN called 'udn-l2-primary' as primary network.")
528+
rootCmd.Flags().StringVar(&bridge, "bridge", "", "Name of the NNCP to be used for creating bridge interface - VM only.")
516529
rootCmd.Flags().StringVar(&promURL, "prom", "", "Prometheus URL")
517530
rootCmd.Flags().StringVar(&id, "uuid", "", "User provided UUID")
518531
rootCmd.Flags().StringVar(&searchURL, "search", "", "OpenSearch URL, if you have auth, pass in the format of https://user:pass@url:port")

‎pkg/config/config.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ type PerfScenarios struct {
3939
VM bool
4040
VMHost string
4141
Udn bool
42+
Bridge bool
4243
ServerNodeInfo metrics.NodeInfo
4344
ClientNodeInfo metrics.NodeInfo
4445
Client apiv1.PodList

‎pkg/k8s/kubernetes.go

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,36 @@ func DeployL2Udn(dynamicClient *dynamic.DynamicClient) error {
171171
return nil
172172
}
173173

174+
// Create a NetworkAttachcmentDefinition object for a bridge connection
175+
func DeployNADBridge(dyn *dynamic.DynamicClient, bridgeName string) error {
176+
nadBridge := &unstructured.Unstructured{
177+
Object: map[string]interface{}{
178+
"apiVersion": "k8s.cni.cncf.io/v1",
179+
"kind": "NetworkAttachmentDefinition",
180+
"metadata": map[string]interface{}{
181+
"name": "br-netperf",
182+
"namespace": "netperf",
183+
"annotations": map[string]interface{}{
184+
"k8s.v1.cni.cncf.io/resourceName": "bridge.network.kubevirt.io/" + bridgeName,
185+
},
186+
},
187+
"spec": map[string]interface{}{
188+
"config": `{"cniVersion": "0.3.1", "type": "bridge", "name": "br-netperf", "bridge": "` + bridgeName + `"}`,
189+
},
190+
},
191+
}
192+
gvr := schema.GroupVersionResource{
193+
Group: "k8s.cni.cncf.io",
194+
Version: "v1",
195+
Resource: "network-attachment-definitions",
196+
}
197+
_, err := dyn.Resource(gvr).Namespace(namespace).Create(context.TODO(), nadBridge, metav1.CreateOptions{})
198+
if err != nil {
199+
return err
200+
}
201+
return nil
202+
}
203+
174204
// BuildSUT Build the k8s env to run network performance tests
175205
func BuildSUT(client *kubernetes.Clientset, s *config.PerfScenarios) error {
176206
var netperfDataPorts []int32
@@ -527,7 +557,7 @@ func ExtractUdnIp(s config.PerfScenarios) (string, error) {
527557

528558
// launchServerVM will create the ServerVM with the specific node and pod affinity.
529559
func launchServerVM(perf *config.PerfScenarios, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) error {
530-
_, err := CreateVMServer(perf.KClient, serverRole, serverRole, *podAff, *nodeAff)
560+
_, err := CreateVMServer(perf.KClient, serverRole, serverRole, *podAff, *nodeAff, perf.Bridge)
531561
if err != nil {
532562
return err
533563
}
@@ -552,7 +582,7 @@ func launchServerVM(perf *config.PerfScenarios, name string, podAff *corev1.PodA
552582

553583
// launchClientVM will create the ClientVM with the specific node and pod affinity.
554584
func launchClientVM(perf *config.PerfScenarios, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) error {
555-
host, err := CreateVMClient(perf.KClient, perf.ClientSet, perf.DClient, name, podAff, nodeAff)
585+
host, err := CreateVMClient(perf.KClient, perf.ClientSet, perf.DClient, name, podAff, nodeAff, perf.Bridge)
556586
if err != nil {
557587
return err
558588
}

‎pkg/k8s/kubevirt.go

Lines changed: 85 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ func exposeService(client *kubernetes.Clientset, dynamicClient *dynamic.DynamicC
158158

159159
// CreateVMClient takes in the affinity rules and deploys the VMI
160160
func CreateVMClient(kclient *kubevirtv1.KubevirtV1Client, client *kubernetes.Clientset,
161-
dyn *dynamic.DynamicClient, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) (string, error) {
161+
dyn *dynamic.DynamicClient, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity, bridge bool) (string, error) {
162162
label := map[string]string{
163163
"app": name,
164164
"role": name,
@@ -171,6 +171,7 @@ func CreateVMClient(kclient *kubevirtv1.KubevirtV1Client, client *kubernetes.Cli
171171
if err != nil {
172172
return "", err
173173
}
174+
netData := "{}"
174175
data := fmt.Sprintf(`#cloud-config
175176
users:
176177
- name: fedora
@@ -184,7 +185,7 @@ chpasswd: { expire: False }
184185
runcmd:
185186
- export HOME=/home/fedora
186187
- dnf install -y --nodocs uperf iperf3 git ethtool automake gcc bc lksctp-tools-devel texinfo --enablerepo=*
187-
- git clone https://github.com/HewlettPackard/netperf
188+
- git clone https://github.com/HewlettPackard/netperf.git
188189
- cd netperf
189190
- git reset --hard 3bc455b23f901dae377ca0a558e1e32aa56b31c4
190191
- curl -o netperf.diff https://raw.githubusercontent.com/cloud-bulldozer/k8s-netperf/main/containers/netperf.diff
@@ -196,7 +197,43 @@ runcmd:
196197
- curl -o /usr/bin/super-netperf https://raw.githubusercontent.com/cloud-bulldozer/k8s-netperf/main/containers/super-netperf
197198
- chmod 0777 /usr/bin/super-netperf
198199
`, ssh)
199-
_, err = CreateVMI(kclient, name, label, b64.StdEncoding.EncodeToString([]byte(data)), *podAff, *nodeAff)
200+
interfaces := []v1.Interface{
201+
{
202+
Name: "default",
203+
InterfaceBindingMethod: v1.InterfaceBindingMethod{
204+
Bridge: &v1.InterfaceBridge{},
205+
},
206+
},
207+
}
208+
networks := []v1.Network{
209+
{
210+
Name: "default",
211+
NetworkSource: v1.NetworkSource{
212+
Pod: &v1.PodNetwork{},
213+
},
214+
},
215+
}
216+
if bridge {
217+
interfaces = append(interfaces, v1.Interface{
218+
Name: "br-netperf",
219+
InterfaceBindingMethod: v1.InterfaceBindingMethod{
220+
Bridge: &v1.InterfaceBridge{},
221+
},
222+
})
223+
networks = append(networks, v1.Network{
224+
Name: "br-netperf",
225+
NetworkSource: v1.NetworkSource{
226+
Multus: &v1.MultusNetwork{
227+
NetworkName: "netperf/br-netperf",
228+
},
229+
},
230+
})
231+
netData = `version: 2
232+
ethernets:
233+
eth1:
234+
addresses: [ 10.10.10.12/24 ]`
235+
}
236+
_, err = CreateVMI(kclient, name, label, b64.StdEncoding.EncodeToString([]byte(data)), *podAff, *nodeAff, interfaces, networks, b64.StdEncoding.EncodeToString([]byte(netData)))
200237
if err != nil {
201238
return "", err
202239
}
@@ -213,11 +250,12 @@ runcmd:
213250

214251
// CreateVMServer will take the pod and node affinity and deploy the VMI
215252
func CreateVMServer(client *kubevirtv1.KubevirtV1Client, name string, role string, podAff corev1.PodAntiAffinity,
216-
nodeAff corev1.NodeAffinity) (*v1.VirtualMachineInstance, error) {
253+
nodeAff corev1.NodeAffinity, bridge bool) (*v1.VirtualMachineInstance, error) {
217254
label := map[string]string{
218255
"app": name,
219256
"role": role,
220257
}
258+
netData := "{}"
221259
dirname, err := os.UserHomeDir()
222260
if err != nil {
223261
return nil, err
@@ -239,7 +277,7 @@ chpasswd: { expire: False }
239277
runcmd:
240278
- dnf install -y --nodocs uperf iperf3 git ethtool
241279
- dnf install -y --nodocs automake gcc bc lksctp-tools-devel texinfo --enablerepo=*
242-
- git clone https://github.com/HewlettPackard/netperf
280+
- git clone https://github.com/HewlettPackard/netperf.git
243281
- cd netperf
244282
- git reset --hard 3bc455b23f901dae377ca0a558e1e32aa56b31c4
245283
- curl -o netperf.diff https://raw.githubusercontent.com/cloud-bulldozer/k8s-netperf/main/containers/netperf.diff
@@ -252,12 +290,48 @@ runcmd:
252290
- iperf3 -s -p %d &
253291
- netserver &
254292
`, string(ssh), UperfServerCtlPort, IperfServerCtlPort)
255-
return CreateVMI(client, name, label, b64.StdEncoding.EncodeToString([]byte(data)), podAff, nodeAff)
293+
interfaces := []v1.Interface{
294+
{
295+
Name: "default",
296+
InterfaceBindingMethod: v1.InterfaceBindingMethod{
297+
Bridge: &v1.InterfaceBridge{},
298+
},
299+
},
300+
}
301+
networks := []v1.Network{
302+
{
303+
Name: "default",
304+
NetworkSource: v1.NetworkSource{
305+
Pod: &v1.PodNetwork{},
306+
},
307+
},
308+
}
309+
if bridge {
310+
interfaces = append(interfaces, v1.Interface{
311+
Name: "br-netperf",
312+
InterfaceBindingMethod: v1.InterfaceBindingMethod{
313+
Bridge: &v1.InterfaceBridge{},
314+
},
315+
})
316+
networks = append(networks, v1.Network{
317+
Name: "br-netperf",
318+
NetworkSource: v1.NetworkSource{
319+
Multus: &v1.MultusNetwork{
320+
NetworkName: "netperf/br-netperf",
321+
},
322+
},
323+
})
324+
netData = `version: 2
325+
ethernets:
326+
eth1:
327+
addresses: [ 10.10.10.14/24 ]`
328+
}
329+
return CreateVMI(client, name, label, b64.StdEncoding.EncodeToString([]byte(data)), podAff, nodeAff, interfaces, networks, b64.StdEncoding.EncodeToString([]byte(netData)))
256330
}
257331

258332
// CreateVMI creates the desired Virtual Machine instance with the cloud-init config with affinity.
259333
func CreateVMI(client *kubevirtv1.KubevirtV1Client, name string, label map[string]string, b64data string, podAff corev1.PodAntiAffinity,
260-
nodeAff corev1.NodeAffinity) (*v1.VirtualMachineInstance, error) {
334+
nodeAff corev1.NodeAffinity, interfaces []v1.Interface, networks []v1.Network, netDatab64 string) (*v1.VirtualMachineInstance, error) {
261335
delSeconds := int64(0)
262336
mutliQ := true
263337
vmi, err := client.VirtualMachineInstances(namespace).Create(context.TODO(), &v1.VirtualMachineInstance{
@@ -300,8 +374,10 @@ func CreateVMI(client *kubevirtv1.KubevirtV1Client, name string, label map[strin
300374
},
301375
},
302376
},
377+
Interfaces: interfaces,
303378
},
304379
},
380+
Networks: networks,
305381
Volumes: []v1.Volume{
306382
v1.Volume{
307383
Name: "disk0",
@@ -315,7 +391,8 @@ func CreateVMI(client *kubevirtv1.KubevirtV1Client, name string, label map[strin
315391
Name: "cloudinit",
316392
VolumeSource: v1.VolumeSource{
317393
CloudInitNoCloud: &v1.CloudInitNoCloudSource{
318-
UserDataBase64: b64data,
394+
UserDataBase64: b64data,
395+
NetworkDataBase64: netDatab64,
319396
},
320397
},
321398
},

0 commit comments

Comments
 (0)