From 6d15b29ce90c055fd2ba98d28b50578997997f5b Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Thu, 9 Nov 2023 14:56:29 +0100 Subject: [PATCH 1/2] kubeadm: install helm via Ignition this can be used to ease the testing of complexe kubernetes deployment Signed-off-by: Mathieu Tortuyaux --- kola/tests/kubeadm/kubeadm.go | 3 +++ kola/tests/kubeadm/kubeadm_test.go | 2 ++ kola/tests/kubeadm/templates.go | 21 +++++++++++++++++++ .../testdata/master-cilium-amd64-config.yml | 21 +++++++++++++++++++ .../testdata/master-cilium-arm64-config.yml | 21 +++++++++++++++++++ 5 files changed, 68 insertions(+) diff --git a/kola/tests/kubeadm/kubeadm.go b/kola/tests/kubeadm/kubeadm.go index 3a0c4d44b..e1e6c8818 100644 --- a/kola/tests/kubeadm/kubeadm.go +++ b/kola/tests/kubeadm/kubeadm.go @@ -76,6 +76,7 @@ var ( // and the nested params are used to render script templates testConfig = map[string]map[string]interface{}{ "v1.28.1": map[string]interface{}{ + "HelmVersion": "v3.13.2", "MinMajorVersion": 3374, // from https://github.com/flannel-io/flannel/releases "FlannelVersion": "v0.22.0", @@ -108,6 +109,7 @@ var ( "cgroupv1": false, }, "v1.27.2": map[string]interface{}{ + "HelmVersion": "v3.13.2", "MinMajorVersion": 3374, // from https://github.com/flannel-io/flannel/releases "FlannelVersion": "v0.22.0", @@ -140,6 +142,7 @@ var ( "cgroupv1": false, }, "v1.26.5": map[string]interface{}{ + "HelmVersion": "v3.13.2", "MinMajorVersion": 3374, // from https://github.com/flannel-io/flannel/releases "FlannelVersion": "v0.20.2", diff --git a/kola/tests/kubeadm/kubeadm_test.go b/kola/tests/kubeadm/kubeadm_test.go index bde20e478..d333ac2d6 100644 --- a/kola/tests/kubeadm/kubeadm_test.go +++ b/kola/tests/kubeadm/kubeadm_test.go @@ -54,6 +54,7 @@ func TestRenderTemplate(t *testing.T) { res, err := render( masterScript, map[string]interface{}{ + "HelmVersion": "1.2.3", "CiliumVersion": "v0.11.1", "FlannelVersion": "v0.14.0", "CNI": CNI, @@ -84,6 +85,7 @@ func TestRenderTemplate(t *testing.T) { res, err := render( masterConfig, map[string]interface{}{ + "HelmVersion": "1.2.3", "CiliumVersion": "v0.11.1", "CNI": "cilium", "CiliumCLIVersion": "v0.9.0", diff --git a/kola/tests/kubeadm/templates.go b/kola/tests/kubeadm/templates.go index 81366b863..6eb4d9970 100644 --- a/kola/tests/kubeadm/templates.go +++ b/kola/tests/kubeadm/templates.go @@ -162,6 +162,21 @@ storage: ExecStart=/usr/bin/rm "/opt/crictl-${CRICTL_VERSION}-linux-{{ .Arch }}.tar.gz" [Install] WantedBy=multi-user.target + - name: prepare-helm.service + enabled: true + contents: | + [Unit] + Description=Unpack helm to /opt/bin + ConditionPathExists=!/opt/bin/helm + [Service] + Type=oneshot + RemainAfterExit=true + Restart=on-failure + ExecStartPre=/usr/bin/mkdir --parents "{{ .DownloadDir }}" + ExecStartPre=/usr/bin/tar -v --extract --file "/opt/helm-{{ .HelmVersion }}-linux-{{ .Arch }}.tar.gz" --directory "{{ .DownloadDir }}" --strip-components=1 --no-same-owner + ExecStart=/usr/bin/rm "/opt/helm-{{ .HelmVersion }}-linux-{{ .Arch }}.tar.gz" + [Install] + WantedBy=multi-user.target storage: files:{{ if .cgroupv1 }} - path: /etc/flatcar-cgroupv1 @@ -216,6 +231,12 @@ storage: hash: function: sha512 sum: {{ index (index . .Arch) "KubectlSum" }} + - path: /opt/helm-{{ .HelmVersion }}-linux-{{ .Arch }}.tar.gz + filesystem: root + mode: 0755 + contents: + remote: + url: https://get.helm.sh/helm-{{ .HelmVersion }}-linux-{{ .Arch }}.tar.gz - path: /etc/docker/daemon.json filesystem: root mode: 0644 diff --git a/kola/tests/kubeadm/testdata/master-cilium-amd64-config.yml b/kola/tests/kubeadm/testdata/master-cilium-amd64-config.yml index 21c2da38b..9c1749719 100644 --- a/kola/tests/kubeadm/testdata/master-cilium-amd64-config.yml +++ b/kola/tests/kubeadm/testdata/master-cilium-amd64-config.yml @@ -34,6 +34,21 @@ systemd: ExecStart=/usr/bin/rm "/opt/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" [Install] WantedBy=multi-user.target + - name: prepare-helm.service + enabled: true + contents: | + [Unit] + Description=Unpack helm to /opt/bin + ConditionPathExists=!/opt/bin/helm + [Service] + Type=oneshot + RemainAfterExit=true + Restart=on-failure + ExecStartPre=/usr/bin/mkdir --parents "/opt/bin" + ExecStartPre=/usr/bin/tar -v --extract --file "/opt/helm-1.2.3-linux-amd64.tar.gz" --directory "/opt/bin" --strip-components=1 --no-same-owner + ExecStart=/usr/bin/rm "/opt/helm-1.2.3-linux-amd64.tar.gz" + [Install] + WantedBy=multi-user.target storage: files: - path: /opt/cni-plugins-linux-amd64-v0.8.7.tgz @@ -86,6 +101,12 @@ storage: hash: function: sha512 sum: a93b2ca067629cb1fe9cbf1af1a195c12126488ed321e3652200d4dbfee9a577865647b7ef6bb673e1bdf08f03108b5dcb4b05812a649a0de5c7c9efc1407810 + - path: /opt/helm-1.2.3-linux-amd64.tar.gz + filesystem: root + mode: 0755 + contents: + remote: + url: https://get.helm.sh/helm-1.2.3-linux-amd64.tar.gz - path: /etc/docker/daemon.json filesystem: root mode: 0644 diff --git a/kola/tests/kubeadm/testdata/master-cilium-arm64-config.yml b/kola/tests/kubeadm/testdata/master-cilium-arm64-config.yml index 4d2a0396c..2ed77405c 100644 --- a/kola/tests/kubeadm/testdata/master-cilium-arm64-config.yml +++ b/kola/tests/kubeadm/testdata/master-cilium-arm64-config.yml @@ -34,6 +34,21 @@ systemd: ExecStart=/usr/bin/rm "/opt/crictl-${CRICTL_VERSION}-linux-arm64.tar.gz" [Install] WantedBy=multi-user.target + - name: prepare-helm.service + enabled: true + contents: | + [Unit] + Description=Unpack helm to /opt/bin + ConditionPathExists=!/opt/bin/helm + [Service] + Type=oneshot + RemainAfterExit=true + Restart=on-failure + ExecStartPre=/usr/bin/mkdir --parents "/opt/bin" + ExecStartPre=/usr/bin/tar -v --extract --file "/opt/helm-1.2.3-linux-arm64.tar.gz" --directory "/opt/bin" --strip-components=1 --no-same-owner + ExecStart=/usr/bin/rm "/opt/helm-1.2.3-linux-arm64.tar.gz" + [Install] + WantedBy=multi-user.target storage: files: - path: /opt/cni-plugins-linux-arm64-v0.8.7.tgz @@ -86,6 +101,12 @@ storage: hash: function: sha512 sum: b990b81d5a885a9d131aabcc3a5ca9c37dfaff701470f2beb896682a8643c7e0c833e479a26f21129b598ac981732bf52eecdbe73896fe0ff2d9c1ffd082d1fd + - path: /opt/helm-1.2.3-linux-arm64.tar.gz + filesystem: root + mode: 0755 + contents: + remote: + url: https://get.helm.sh/helm-1.2.3-linux-arm64.tar.gz - path: /etc/docker/daemon.json filesystem: root mode: 0644 From 6c88fd65a7d5b19bc7838a9ee1cff42018351107 Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Thu, 9 Nov 2023 15:03:10 +0100 Subject: [PATCH 2/2] kubeadm: add NFS storage class tests This deploys an NFS storage class and tests that we can deploy a pod consuming this storage using NFS vers 4.2 mounting options. Signed-off-by: Mathieu Tortuyaux --- kola/tests/kubeadm/kubeadm.go | 32 +++++++++++++++++ kola/tests/kubeadm/templates.go | 36 +++++++++++++++++++ .../testdata/master-cilium-amd64-config.yml | 36 +++++++++++++++++++ .../testdata/master-cilium-arm64-config.yml | 36 +++++++++++++++++++ 4 files changed, 140 insertions(+) diff --git a/kola/tests/kubeadm/kubeadm.go b/kola/tests/kubeadm/kubeadm.go index e1e6c8818..daa509705 100644 --- a/kola/tests/kubeadm/kubeadm.go +++ b/kola/tests/kubeadm/kubeadm.go @@ -292,6 +292,38 @@ func kubeadmBaseTest(c cluster.TestCluster, params map[string]interface{}) { } }) + c.Run("NFS deployment", func(c cluster.TestCluster) { + if _, err := c.SSH(kubectl, "/opt/bin/helm repo add nfs-ganesha-server-and-external-provisioner https://kubernetes-sigs.github.io/nfs-ganesha-server-and-external-provisioner/"); err != nil { + c.Fatalf("unable to add helm NFS repo: %v", err) + } + + if _, err := c.SSH(kubectl, "/opt/bin/helm install nfs-server-provisioner nfs-ganesha-server-and-external-provisioner/nfs-server-provisioner --set 'storageClass.mountOptions={nfsvers=4.2}'"); err != nil { + c.Fatalf("unable to install NFS Helm Chart: %v", err) + } + + // Manifests have been deployed through Ignition + if _, err := c.SSH(kubectl, "/opt/bin/kubectl apply -f nfs-pod.yaml -f nfs-pvc.yaml"); err != nil { + c.Fatalf("unable to create NFS pod and pvc: %v", err) + } + + // Wait up to 3 min (36*5 = 180s). The test can be flaky on overcommitted platforms. + if err := util.Retry(36, 5*time.Second, func() error { + out, err := c.SSH(kubectl, `/opt/bin/kubectl get pod/test-pod-1 -o json | jq '.status.containerStatuses[] | select (.name == "test") | .ready'`) + if err != nil { + return fmt.Errorf("getting container status: %v", err) + } + + ready := string(out) + if ready != "true" { + return fmt.Errorf("'test' pod should be ready, got: %s", ready) + } + + return nil + }); err != nil { + c.Fatalf("nginx pod with NFS is not deployed: %v", err) + } + }) + // this should not fail, we always have the CNI present at this step. cni, ok := params["CNI"] if !ok { diff --git a/kola/tests/kubeadm/templates.go b/kola/tests/kubeadm/templates.go index 6eb4d9970..09492f441 100644 --- a/kola/tests/kubeadm/templates.go +++ b/kola/tests/kubeadm/templates.go @@ -285,6 +285,42 @@ storage: image: ghcr.io/flatcar/nginx ports: - containerPort: 80 + - path: /home/core/nfs-pod.yaml + filesystem: root + mode: 0644 + contents: + inline: | + apiVersion: v1 + kind: Pod + metadata: + name: test-pod-1 + spec: + containers: + - name: test + image: ghcr.io/flatcar/nginx + volumeMounts: + - name: config + mountPath: /test + volumes: + - name: config + persistentVolumeClaim: + claimName: test-dynamic-volume-claim + - path: /home/core/nfs-pvc.yaml + filesystem: root + mode: 0644 + contents: + inline: | + kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: test-dynamic-volume-claim + spec: + storageClassName: "nfs" + accessModes: + - ReadWriteMany + resources: + requests: + storage: 100Mi ` masterScript = `#!/bin/bash diff --git a/kola/tests/kubeadm/testdata/master-cilium-amd64-config.yml b/kola/tests/kubeadm/testdata/master-cilium-amd64-config.yml index 9c1749719..4c1d1e532 100644 --- a/kola/tests/kubeadm/testdata/master-cilium-amd64-config.yml +++ b/kola/tests/kubeadm/testdata/master-cilium-amd64-config.yml @@ -155,3 +155,39 @@ storage: image: ghcr.io/flatcar/nginx ports: - containerPort: 80 + - path: /home/core/nfs-pod.yaml + filesystem: root + mode: 0644 + contents: + inline: | + apiVersion: v1 + kind: Pod + metadata: + name: test-pod-1 + spec: + containers: + - name: test + image: ghcr.io/flatcar/nginx + volumeMounts: + - name: config + mountPath: /test + volumes: + - name: config + persistentVolumeClaim: + claimName: test-dynamic-volume-claim + - path: /home/core/nfs-pvc.yaml + filesystem: root + mode: 0644 + contents: + inline: | + kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: test-dynamic-volume-claim + spec: + storageClassName: "nfs" + accessModes: + - ReadWriteMany + resources: + requests: + storage: 100Mi diff --git a/kola/tests/kubeadm/testdata/master-cilium-arm64-config.yml b/kola/tests/kubeadm/testdata/master-cilium-arm64-config.yml index 2ed77405c..fd691729a 100644 --- a/kola/tests/kubeadm/testdata/master-cilium-arm64-config.yml +++ b/kola/tests/kubeadm/testdata/master-cilium-arm64-config.yml @@ -155,3 +155,39 @@ storage: image: ghcr.io/flatcar/nginx ports: - containerPort: 80 + - path: /home/core/nfs-pod.yaml + filesystem: root + mode: 0644 + contents: + inline: | + apiVersion: v1 + kind: Pod + metadata: + name: test-pod-1 + spec: + containers: + - name: test + image: ghcr.io/flatcar/nginx + volumeMounts: + - name: config + mountPath: /test + volumes: + - name: config + persistentVolumeClaim: + claimName: test-dynamic-volume-claim + - path: /home/core/nfs-pvc.yaml + filesystem: root + mode: 0644 + contents: + inline: | + kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: test-dynamic-volume-claim + spec: + storageClassName: "nfs" + accessModes: + - ReadWriteMany + resources: + requests: + storage: 100Mi