From 3d63bf1d887d364779fc888e241888c81e168605 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 12 Nov 2020 14:15:06 +0100 Subject: [PATCH] deploy: document NFD usage and test with it We should encourage the usage of NFD for labeling nodes because it is simpler and more scalable. To ensure that it works, we set up the test cluster accordingly. The default node selector is the same as before, to avoid breaking installations during an upgrade. --- deploy/common/pmem-app-block-volume.yaml | 2 -- ...-csi.intel.com_v1alpha1_deployment_cr.yaml | 5 ++- docs/autotest.md | 7 +++-- docs/install.md | 31 ++++++++++++++----- test/e2e/deploy/deploy.go | 4 +++ test/e2e/operator/deployment_api.go | 4 +++ test/setup-deployment.sh | 16 ++++++++++ test/start-kubernetes.sh | 8 +++++ test/test-config.sh | 4 +++ 9 files changed, 69 insertions(+), 12 deletions(-) diff --git a/deploy/common/pmem-app-block-volume.yaml b/deploy/common/pmem-app-block-volume.yaml index 9a2ec34349..8fa7933c73 100644 --- a/deploy/common/pmem-app-block-volume.yaml +++ b/deploy/common/pmem-app-block-volume.yaml @@ -30,8 +30,6 @@ spec: volumeMounts: - name: data mountPath: /data - nodeSelector: - storage: pmem volumes: - name: my-csi-device persistentVolumeClaim: diff --git a/deploy/common/pmem-csi.intel.com_v1alpha1_deployment_cr.yaml b/deploy/common/pmem-csi.intel.com_v1alpha1_deployment_cr.yaml index f6b2358b30..7d38c5c138 100644 --- a/deploy/common/pmem-csi.intel.com_v1alpha1_deployment_cr.yaml +++ b/deploy/common/pmem-csi.intel.com_v1alpha1_deployment_cr.yaml @@ -5,5 +5,8 @@ metadata: spec: deviceMode: "lvm" nodeSelector: - storage: "pmem" + # When using Node Feature Discovery (NFD): + feature.node.kubernetes.io/memory-nv.dax: "true" + # When using manual node labeling with that label: + # storage: pmem diff --git a/docs/autotest.md b/docs/autotest.md index 2f2e114c44..62d3f5e3cd 100644 --- a/docs/autotest.md +++ b/docs/autotest.md @@ -56,8 +56,11 @@ virtual machines. The first node is the Kubernetes master without persistent memory. The other three nodes are worker nodes with one emulated 32GB NVDIMM each. -After the cluster has been formed, `make start` adds `storage=pmem` label -to the worker nodes and deploys the PMEM-CSI driver. +After the cluster has been formed, `make start` installs [NFD](https://kubernetes-sigs.github.io/node-feature-discovery/stable/get-started/index.html) to label +the worker nodes. The PMEM-CSI driver can be installed with +`test/setup-deployment.sh`, but will also be installed as needed by +the E2E test suite. + Once `make start` completes, the cluster is ready for interactive use via `kubectl` inside the virtual machine. Alternatively, you can also set `KUBECONFIG` as shown at the end of the `make start` output diff --git a/docs/install.md b/docs/install.md index cc9e60e7c5..f83f1f2665 100644 --- a/docs/install.md +++ b/docs/install.md @@ -209,10 +209,25 @@ on the Kubernetes version. - **Label the cluster nodes that provide persistent memory device(s)** +PMEM-CSI manages PMEM on those nodes that have a certain label. For +historic reasons, the default in the YAML files and the operator +settings is to use a label `storage` with the value `pmem`. + +Such a label can be set for each node manually with: + ``` console $ kubectl label node storage=pmem ``` +Alternatively, the [Node Feature +Discovery (NFD)](https://kubernetes-sigs.github.io/node-feature-discovery/stable/get-started/index.html) +add-on can be used to label nodes automatically. In that case, the +default PMEM-CSI node selector has to be changed to +`"feature.node.kubernetes.io/memory-nv.dax": "true"`. The operator has +the [`nodeSelector` +field](https://kubernetes-sigs.github.io/node-feature-discovery/stable/get-started/index.html) +for that. For the YAML files a kustomize patch can be used. + ### Install PMEM-CSI driver PMEM-CSI driver can be deployed to a Kubernetes cluster either using the @@ -267,12 +282,12 @@ metadata: spec: deviceMode: lvm nodeSelector: - storage: pmem + feature.node.kubernetes.io/memory-nv.dax: "true" EOF ``` This uses the same `pmem-csi.intel.com` driver name as the YAML files -in [`deploy`](/deploy) and the node label from the [hardware +in [`deploy`](/deploy) and the node label created by NFD (see the [hardware installation and setup section](#installation-and-setup). Once the above deployment installation is successful, we can see all the driver @@ -448,13 +463,16 @@ verify that the node labels have been configured correctly $ kubectl get nodes --show-labels ``` -The command output must indicate that every node with PMEM has these two labels: +The command output must indicate that every node with PMEM has at least two labels: ``` console pmem-csi.intel.com/node=,storage=pmem ``` -If **storage=pmem** is missing, label manually as described above. If -**pmem-csi.intel.com/node** is missing, then double-check that the +**storage=pmem** is the label that has to be added manually as +described above. When using NFD, the node should have the +`feature.node.kubernetes.io/memory-nv.dax=true` label. + +If **pmem-csi.intel.com/node** is missing, then double-check that the alpha feature gates are enabled, that the CSI driver is running on the node, and that the driver's log output doesn't contain errors. @@ -486,8 +504,7 @@ pmem-csi-pvc-xfs Bound pvc-f7101fd2-6b36-11e9-bf09-deadbeef0100 4Gi $ kubectl create -f deploy/common/pmem-app.yaml ``` -These applications use **storage: pmem** in the nodeSelector -list to ensure scheduling to a node supporting pmem device, and each requests a mount of a volume, +These applications each request a mount of a volume, one with ext4-format and another with xfs-format file system. - **Verify two application pods reach 'Running' status** diff --git a/test/e2e/deploy/deploy.go b/test/e2e/deploy/deploy.go index 62ba98c1aa..d927d7c85a 100644 --- a/test/e2e/deploy/deploy.go +++ b/test/e2e/deploy/deploy.go @@ -932,6 +932,10 @@ func (d *Deployment) GetDriverDeployment() api.Deployment { // PMEM must be used for LVM, otherwise other tests cannot // run after the LVM driver was deployed once. PMEMPercentage: 50, + NodeSelector: map[string]string{ + // Provided by NFD. + "feature.node.kubernetes.io/memory-nv.dax": "true", + }, }, } } diff --git a/test/e2e/operator/deployment_api.go b/test/e2e/operator/deployment_api.go index aa3ac2ee31..314850ab43 100644 --- a/test/e2e/operator/deployment_api.go +++ b/test/e2e/operator/deployment_api.go @@ -385,6 +385,10 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { Spec: api.DeploymentSpec{ DeviceMode: from, PMEMPercentage: 50, + NodeSelector: map[string]string{ + // Provided by NFD. + "feature.node.kubernetes.io/memory-nv.dax": "true", + }, }, } diff --git a/test/setup-deployment.sh b/test/setup-deployment.sh index 98f7d2b7b4..29cc03ea29 100755 --- a/test/setup-deployment.sh +++ b/test/setup-deployment.sh @@ -157,6 +157,22 @@ EOF value: "--pmemPercentage=50" EOF fi + + # Always use the configured label for selecting nodes. + ${SSH} "cat >>'$tmpdir/my-deployment/kustomization.yaml'" <>'$tmpdir/my-deployment/node-label-patch.yaml'" <