From 263dd310d24c39357bcf846ac86152ea334cd9a0 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 4 Oct 2022 13:01:29 +0200 Subject: [PATCH 1/6] test: update WeaveNet installation. The installation of WeaveNet stopped working end of September when cloud.weave.net was turned off. The latest upstream documentation (https://www.weave.works/docs/net/latest/kubernetes/kube-addon/#-installation) describes installation from GitHub. --- test/setup-kubernetes.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/setup-kubernetes.sh b/test/setup-kubernetes.sh index b9a3db1d0..66fe02580 100755 --- a/test/setup-kubernetes.sh +++ b/test/setup-kubernetes.sh @@ -276,7 +276,7 @@ kubectl version ${TEST_CONFIGURE_POST_MASTER} -kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')" +kubectl apply -f https://github.com/weaveworks/weave/releases/download/v2.8.1/weave-daemonset-k8s.yaml # Install addon storage CRDs, needed if certain feature gates are enabled. # Only applicable to Kubernetes 1.13 and older. 1.14 will have them as builtin APIs. From e529813ebaae5cc20eed19b6cecbc92237991c79 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 7 Dec 2022 10:05:28 +0100 Subject: [PATCH 2/6] build: work around GitHub "releases/latest" URL changes A GET used to return some JSON that described the latest URL. Not anymore, now one has to follow the redirect to find the latest tag. --- Dockerfile | 2 +- Dockerfile.UBI | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1aa7e5203..613d4786f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,7 +39,7 @@ RUN equivs-build python3-fake-debian-package RUN set -x && \ git clone https://github.com/intel/ipmctl.git && \ cd ipmctl && \ - tag=$(curl --silent https://github.com/intel/ipmctl/releases/latest | sed -e 's;.*tag/\([^"]*\).*;\1;') && \ + tag=$(basename $(curl -Ls -o /dev/null -w %{url_effective} https://github.com/intel/ipmctl/releases/latest)) && \ git checkout $tag && \ mkdir build && \ cd build && \ diff --git a/Dockerfile.UBI b/Dockerfile.UBI index fe5a51e36..394bbe58f 100644 --- a/Dockerfile.UBI +++ b/Dockerfile.UBI @@ -59,7 +59,7 @@ RUN set -x && \ mkdir -p /usr/local/share/package-licenses && \ cp LICENSE /usr/local/share/package-licenses/ipmctl.LICENSE && \ cd ipmctl && \ - tag=$(curl --silent https://github.com/intel/ipmctl/releases/latest | sed -e 's;.*tag/\([^"]*\).*;\1;') && \ + tag=$(basename $(curl -Ls -o /dev/null -w %{url_effective} https://github.com/intel/ipmctl/releases/latest)) && \ git checkout $tag && \ mkdir build && \ cd build && \ From 5a9e5305d90e279024f0d3c65c9054b254637fec Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 29 Sep 2022 21:04:53 +0200 Subject: [PATCH 3/6] remove support for scheduler extender and mutating pod webhook As all supported Kubernetes releases have storage capacity tracking, the previous solution is no longer needed. Removing it gets rid of technical debt and simplifies the installation instructions. --- conf.json | 3 +- ...pmem-csi.intel.com_pmemcsideployments.yaml | 17 +- deploy/kubernetes-1.21/direct/pmem-csi.yaml | 11 - .../direct/testing/pmem-csi.yaml | 11 - deploy/kubernetes-1.21/lvm/pmem-csi.yaml | 11 - .../kubernetes-1.21/lvm/testing/pmem-csi.yaml | 11 - .../pmem-csi-direct-testing.yaml | 11 - deploy/kubernetes-1.21/pmem-csi-direct.yaml | 11 - .../kubernetes-1.21/pmem-csi-lvm-testing.yaml | 11 - deploy/kubernetes-1.21/pmem-csi-lvm.yaml | 11 - deploy/kubernetes-1.22/direct/pmem-csi.yaml | 11 - .../direct/testing/pmem-csi.yaml | 11 - deploy/kubernetes-1.22/lvm/pmem-csi.yaml | 11 - .../kubernetes-1.22/lvm/testing/pmem-csi.yaml | 11 - .../pmem-csi-direct-testing.yaml | 11 - deploy/kubernetes-1.22/pmem-csi-direct.yaml | 11 - .../kubernetes-1.22/pmem-csi-lvm-testing.yaml | 11 - deploy/kubernetes-1.22/pmem-csi-lvm.yaml | 11 - deploy/kubernetes-1.23/direct/pmem-csi.yaml | 11 - .../direct/testing/pmem-csi.yaml | 11 - deploy/kubernetes-1.23/lvm/pmem-csi.yaml | 11 - .../kubernetes-1.23/lvm/testing/pmem-csi.yaml | 11 - .../pmem-csi-direct-testing.yaml | 11 - deploy/kubernetes-1.23/pmem-csi-direct.yaml | 11 - .../kubernetes-1.23/pmem-csi-lvm-testing.yaml | 11 - deploy/kubernetes-1.23/pmem-csi-lvm.yaml | 11 - deploy/kubernetes-1.24/direct/pmem-csi.yaml | 11 - .../direct/testing/pmem-csi.yaml | 11 - deploy/kubernetes-1.24/lvm/pmem-csi.yaml | 11 - .../kubernetes-1.24/lvm/testing/pmem-csi.yaml | 11 - .../pmem-csi-direct-testing.yaml | 11 - deploy/kubernetes-1.24/pmem-csi-direct.yaml | 11 - .../kubernetes-1.24/pmem-csi-lvm-testing.yaml | 11 - deploy/kubernetes-1.24/pmem-csi-lvm.yaml | 11 - deploy/kubernetes-1.25/direct/pmem-csi.yaml | 11 - .../direct/testing/pmem-csi.yaml | 11 - deploy/kubernetes-1.25/lvm/pmem-csi.yaml | 11 - .../kubernetes-1.25/lvm/testing/pmem-csi.yaml | 11 - .../pmem-csi-direct-testing.yaml | 11 - deploy/kubernetes-1.25/pmem-csi-direct.yaml | 11 - .../kubernetes-1.25/pmem-csi-lvm-testing.yaml | 11 - deploy/kubernetes-1.25/pmem-csi-lvm.yaml | 11 - deploy/kustomize/driver/pmem-csi.yaml | 13 - deploy/kustomize/driver/webhooks-rbac.yaml | 0 .../scheduler/openshift-configmap.yaml | 23 - .../scheduler/scheduler-service.yaml | 13 - .../testing/controller-coverage-patch.yaml | 16 +- deploy/yamls.go | 3 - docs/DEVELOPMENT.md | 1 - docs/autotest.md | 8 - docs/design.md | 98 --- docs/install.md | 544 +------------ go.mod | 1 - go.sum | 2 - operator/operator.make | 2 + pkg/apis/pmemcsi/v1beta1/deployment_types.go | 59 +- pkg/deployments/load.go | 115 --- pkg/deployments/load_test.go | 2 +- pkg/pmem-csi-driver/main.go | 20 +- pkg/pmem-csi-driver/pmem-csi-driver.go | 91 +-- pkg/pmem-csi-driver/pmem-csi-driver_test.go | 12 +- .../deployment/controller_driver.go | 207 ----- .../deployment/deployment_controller.go | 33 - .../deployment/deployment_controller_test.go | 70 +- .../deployment/testcases/testcases.go | 20 - pkg/scheduler/README.md | 8 - pkg/scheduler/capacity.go | 138 ---- pkg/scheduler/capacity_test.go | 209 ----- pkg/scheduler/mutate_pod.go | 180 ----- pkg/scheduler/scheduler.go | 366 --------- pkg/scheduler/scheduler_test.go | 763 ------------------ runtime-deps.csv | 1 - test/e2e/deploy/deploy.go | 23 - test/e2e/operator/deployment_api.go | 100 +-- test/e2e/operator/validate/validate.go | 29 +- test/e2e/storage/pmem_csi.go | 6 - test/e2e/storage/scheduler/scheduler.go | 199 ----- test/e2e/tls/tls.go | 2 +- test/setup-ca-kubernetes.sh | 50 -- test/setup-ca.sh | 75 -- test/setup-deployment.sh | 129 +-- test/setup-kubernetes.sh | 115 --- test/start-kubernetes.sh | 1 - test/start-operator.sh | 6 - test/start-stop.make | 2 +- test/test-config.sh | 11 - test/test.make | 10 +- 87 files changed, 88 insertions(+), 4148 deletions(-) delete mode 100644 deploy/kustomize/driver/webhooks-rbac.yaml delete mode 100644 deploy/kustomize/scheduler/openshift-configmap.yaml delete mode 100644 deploy/kustomize/scheduler/scheduler-service.yaml delete mode 100644 pkg/scheduler/README.md delete mode 100644 pkg/scheduler/capacity.go delete mode 100644 pkg/scheduler/capacity_test.go delete mode 100644 pkg/scheduler/mutate_pod.go delete mode 100644 pkg/scheduler/scheduler.go delete mode 100644 pkg/scheduler/scheduler_test.go delete mode 100644 test/e2e/storage/scheduler/scheduler.go delete mode 100755 test/setup-ca-kubernetes.sh delete mode 100755 test/setup-ca.sh diff --git a/conf.json b/conf.json index 364302af3..64fe67c43 100644 --- a/conf.json +++ b/conf.json @@ -9,8 +9,7 @@ "_work", "deploy/kustomize", "third-party", - "test/test-config.d", - "pkg/scheduler" + "test/test-config.d" ], "extensions": [ "linkcheck2", diff --git a/deploy/crd/pmem-csi.intel.com_pmemcsideployments.yaml b/deploy/crd/pmem-csi.intel.com_pmemcsideployments.yaml index a7079dd33..d5a16527a 100644 --- a/deploy/crd/pmem-csi.intel.com_pmemcsideployments.yaml +++ b/deploy/crd/pmem-csi.intel.com_pmemcsideployments.yaml @@ -89,12 +89,9 @@ spec: type: object type: object controllerTLSSecret: - description: ControllerTLSSecret is the name of a secret which contains - ca.crt, tls.crt and tls.key data for the scheduler extender and - pod mutation webhook. A controller is started if (and only if) this - secret is specified. The special string "-openshift-" enables the - usage of https://docs.openshift.com/container-platform/4.6/security/certificates/service-serving-certificate.html - to create certificates. + description: "ControllerTLSSecret used to be the name of a secret + which contains ca.crt, tls.crt and tls.key data for the scheduler + extender and pod mutation webhook. It is now unused. \n DEPRECATED" type: string deviceMode: description: DeviceMode to use to manage PMEM devices. @@ -135,9 +132,9 @@ spec: increased with this setting, either with a higher integer or a percentage. x-kubernetes-int-or-string: true mutatePods: - description: MutatePod defines how a mutating pod webhook is configured + description: "MutatePod defines how a mutating pod webhook is configured if a controller is started. The field is ignored if the controller - is not enabled. The default is "Try". + is not enabled. The default is \"Try\". \n DEPRECATED" enum: - Always - Try @@ -246,12 +243,12 @@ spec: type: object type: object schedulerNodePort: - description: SchedulerNodePort, if non-zero, ensures that the "scheduler" + description: "SchedulerNodePort, if non-zero, ensures that the \"scheduler\" service is created as a NodeService with that fixed port number. Otherwise that service is created as a cluster service. The number must be from the range reserved by Kubernetes for node ports. This is useful if the kube-scheduler cannot reach the scheduler extender - via a cluster service. + via a cluster service. \n DEPRECATED" format: int32 type: integer type: object diff --git a/deploy/kubernetes-1.21/direct/pmem-csi.yaml b/deploy/kubernetes-1.21/direct/pmem-csi.yaml index bb5841a52..7c7f6fe52 100644 --- a/deploy/kubernetes-1.21/direct/pmem-csi.yaml +++ b/deploy/kubernetes-1.21/direct/pmem-csi.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 env: - name: TERMINATION_LOG_PATH @@ -397,9 +393,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -407,10 +400,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.21/direct/testing/pmem-csi.yaml b/deploy/kubernetes-1.21/direct/testing/pmem-csi.yaml index 661d3cff6..53473738f 100644 --- a/deploy/kubernetes-1.21/direct/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.21/direct/testing/pmem-csi.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 env: @@ -398,9 +394,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -408,10 +401,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.21/lvm/pmem-csi.yaml b/deploy/kubernetes-1.21/lvm/pmem-csi.yaml index 2cb71513f..9d7b10bc9 100644 --- a/deploy/kubernetes-1.21/lvm/pmem-csi.yaml +++ b/deploy/kubernetes-1.21/lvm/pmem-csi.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 env: - name: TERMINATION_LOG_PATH @@ -397,9 +393,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -407,10 +400,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.21/lvm/testing/pmem-csi.yaml b/deploy/kubernetes-1.21/lvm/testing/pmem-csi.yaml index 44e94397d..8eda1e6dc 100644 --- a/deploy/kubernetes-1.21/lvm/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.21/lvm/testing/pmem-csi.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 env: @@ -398,9 +394,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -408,10 +401,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.21/pmem-csi-direct-testing.yaml b/deploy/kubernetes-1.21/pmem-csi-direct-testing.yaml index 661d3cff6..53473738f 100644 --- a/deploy/kubernetes-1.21/pmem-csi-direct-testing.yaml +++ b/deploy/kubernetes-1.21/pmem-csi-direct-testing.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 env: @@ -398,9 +394,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -408,10 +401,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.21/pmem-csi-direct.yaml b/deploy/kubernetes-1.21/pmem-csi-direct.yaml index bb5841a52..7c7f6fe52 100644 --- a/deploy/kubernetes-1.21/pmem-csi-direct.yaml +++ b/deploy/kubernetes-1.21/pmem-csi-direct.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 env: - name: TERMINATION_LOG_PATH @@ -397,9 +393,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -407,10 +400,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.21/pmem-csi-lvm-testing.yaml b/deploy/kubernetes-1.21/pmem-csi-lvm-testing.yaml index 44e94397d..8eda1e6dc 100644 --- a/deploy/kubernetes-1.21/pmem-csi-lvm-testing.yaml +++ b/deploy/kubernetes-1.21/pmem-csi-lvm-testing.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 env: @@ -398,9 +394,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -408,10 +401,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.21/pmem-csi-lvm.yaml b/deploy/kubernetes-1.21/pmem-csi-lvm.yaml index 2cb71513f..9d7b10bc9 100644 --- a/deploy/kubernetes-1.21/pmem-csi-lvm.yaml +++ b/deploy/kubernetes-1.21/pmem-csi-lvm.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 env: - name: TERMINATION_LOG_PATH @@ -397,9 +393,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -407,10 +400,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.22/direct/pmem-csi.yaml b/deploy/kubernetes-1.22/direct/pmem-csi.yaml index bb5841a52..7c7f6fe52 100644 --- a/deploy/kubernetes-1.22/direct/pmem-csi.yaml +++ b/deploy/kubernetes-1.22/direct/pmem-csi.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 env: - name: TERMINATION_LOG_PATH @@ -397,9 +393,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -407,10 +400,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.22/direct/testing/pmem-csi.yaml b/deploy/kubernetes-1.22/direct/testing/pmem-csi.yaml index 661d3cff6..53473738f 100644 --- a/deploy/kubernetes-1.22/direct/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.22/direct/testing/pmem-csi.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 env: @@ -398,9 +394,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -408,10 +401,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.22/lvm/pmem-csi.yaml b/deploy/kubernetes-1.22/lvm/pmem-csi.yaml index 2cb71513f..9d7b10bc9 100644 --- a/deploy/kubernetes-1.22/lvm/pmem-csi.yaml +++ b/deploy/kubernetes-1.22/lvm/pmem-csi.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 env: - name: TERMINATION_LOG_PATH @@ -397,9 +393,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -407,10 +400,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.22/lvm/testing/pmem-csi.yaml b/deploy/kubernetes-1.22/lvm/testing/pmem-csi.yaml index 44e94397d..8eda1e6dc 100644 --- a/deploy/kubernetes-1.22/lvm/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.22/lvm/testing/pmem-csi.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 env: @@ -398,9 +394,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -408,10 +401,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.22/pmem-csi-direct-testing.yaml b/deploy/kubernetes-1.22/pmem-csi-direct-testing.yaml index 661d3cff6..53473738f 100644 --- a/deploy/kubernetes-1.22/pmem-csi-direct-testing.yaml +++ b/deploy/kubernetes-1.22/pmem-csi-direct-testing.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 env: @@ -398,9 +394,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -408,10 +401,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.22/pmem-csi-direct.yaml b/deploy/kubernetes-1.22/pmem-csi-direct.yaml index bb5841a52..7c7f6fe52 100644 --- a/deploy/kubernetes-1.22/pmem-csi-direct.yaml +++ b/deploy/kubernetes-1.22/pmem-csi-direct.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 env: - name: TERMINATION_LOG_PATH @@ -397,9 +393,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -407,10 +400,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.22/pmem-csi-lvm-testing.yaml b/deploy/kubernetes-1.22/pmem-csi-lvm-testing.yaml index 44e94397d..8eda1e6dc 100644 --- a/deploy/kubernetes-1.22/pmem-csi-lvm-testing.yaml +++ b/deploy/kubernetes-1.22/pmem-csi-lvm-testing.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 env: @@ -398,9 +394,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -408,10 +401,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.22/pmem-csi-lvm.yaml b/deploy/kubernetes-1.22/pmem-csi-lvm.yaml index 2cb71513f..9d7b10bc9 100644 --- a/deploy/kubernetes-1.22/pmem-csi-lvm.yaml +++ b/deploy/kubernetes-1.22/pmem-csi-lvm.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 env: - name: TERMINATION_LOG_PATH @@ -397,9 +393,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -407,10 +400,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.23/direct/pmem-csi.yaml b/deploy/kubernetes-1.23/direct/pmem-csi.yaml index bb5841a52..7c7f6fe52 100644 --- a/deploy/kubernetes-1.23/direct/pmem-csi.yaml +++ b/deploy/kubernetes-1.23/direct/pmem-csi.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 env: - name: TERMINATION_LOG_PATH @@ -397,9 +393,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -407,10 +400,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.23/direct/testing/pmem-csi.yaml b/deploy/kubernetes-1.23/direct/testing/pmem-csi.yaml index 661d3cff6..53473738f 100644 --- a/deploy/kubernetes-1.23/direct/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.23/direct/testing/pmem-csi.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 env: @@ -398,9 +394,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -408,10 +401,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.23/lvm/pmem-csi.yaml b/deploy/kubernetes-1.23/lvm/pmem-csi.yaml index 2cb71513f..9d7b10bc9 100644 --- a/deploy/kubernetes-1.23/lvm/pmem-csi.yaml +++ b/deploy/kubernetes-1.23/lvm/pmem-csi.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 env: - name: TERMINATION_LOG_PATH @@ -397,9 +393,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -407,10 +400,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.23/lvm/testing/pmem-csi.yaml b/deploy/kubernetes-1.23/lvm/testing/pmem-csi.yaml index 44e94397d..8eda1e6dc 100644 --- a/deploy/kubernetes-1.23/lvm/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.23/lvm/testing/pmem-csi.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 env: @@ -398,9 +394,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -408,10 +401,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.23/pmem-csi-direct-testing.yaml b/deploy/kubernetes-1.23/pmem-csi-direct-testing.yaml index 661d3cff6..53473738f 100644 --- a/deploy/kubernetes-1.23/pmem-csi-direct-testing.yaml +++ b/deploy/kubernetes-1.23/pmem-csi-direct-testing.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 env: @@ -398,9 +394,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -408,10 +401,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.23/pmem-csi-direct.yaml b/deploy/kubernetes-1.23/pmem-csi-direct.yaml index bb5841a52..7c7f6fe52 100644 --- a/deploy/kubernetes-1.23/pmem-csi-direct.yaml +++ b/deploy/kubernetes-1.23/pmem-csi-direct.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 env: - name: TERMINATION_LOG_PATH @@ -397,9 +393,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -407,10 +400,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.23/pmem-csi-lvm-testing.yaml b/deploy/kubernetes-1.23/pmem-csi-lvm-testing.yaml index 44e94397d..8eda1e6dc 100644 --- a/deploy/kubernetes-1.23/pmem-csi-lvm-testing.yaml +++ b/deploy/kubernetes-1.23/pmem-csi-lvm-testing.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 env: @@ -398,9 +394,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -408,10 +401,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.23/pmem-csi-lvm.yaml b/deploy/kubernetes-1.23/pmem-csi-lvm.yaml index 2cb71513f..9d7b10bc9 100644 --- a/deploy/kubernetes-1.23/pmem-csi-lvm.yaml +++ b/deploy/kubernetes-1.23/pmem-csi-lvm.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 env: - name: TERMINATION_LOG_PATH @@ -397,9 +393,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -407,10 +400,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.24/direct/pmem-csi.yaml b/deploy/kubernetes-1.24/direct/pmem-csi.yaml index bb5841a52..7c7f6fe52 100644 --- a/deploy/kubernetes-1.24/direct/pmem-csi.yaml +++ b/deploy/kubernetes-1.24/direct/pmem-csi.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 env: - name: TERMINATION_LOG_PATH @@ -397,9 +393,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -407,10 +400,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.24/direct/testing/pmem-csi.yaml b/deploy/kubernetes-1.24/direct/testing/pmem-csi.yaml index 661d3cff6..53473738f 100644 --- a/deploy/kubernetes-1.24/direct/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.24/direct/testing/pmem-csi.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 env: @@ -398,9 +394,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -408,10 +401,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.24/lvm/pmem-csi.yaml b/deploy/kubernetes-1.24/lvm/pmem-csi.yaml index 2cb71513f..9d7b10bc9 100644 --- a/deploy/kubernetes-1.24/lvm/pmem-csi.yaml +++ b/deploy/kubernetes-1.24/lvm/pmem-csi.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 env: - name: TERMINATION_LOG_PATH @@ -397,9 +393,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -407,10 +400,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.24/lvm/testing/pmem-csi.yaml b/deploy/kubernetes-1.24/lvm/testing/pmem-csi.yaml index 44e94397d..8eda1e6dc 100644 --- a/deploy/kubernetes-1.24/lvm/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.24/lvm/testing/pmem-csi.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 env: @@ -398,9 +394,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -408,10 +401,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.24/pmem-csi-direct-testing.yaml b/deploy/kubernetes-1.24/pmem-csi-direct-testing.yaml index 661d3cff6..53473738f 100644 --- a/deploy/kubernetes-1.24/pmem-csi-direct-testing.yaml +++ b/deploy/kubernetes-1.24/pmem-csi-direct-testing.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 env: @@ -398,9 +394,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -408,10 +401,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.24/pmem-csi-direct.yaml b/deploy/kubernetes-1.24/pmem-csi-direct.yaml index bb5841a52..7c7f6fe52 100644 --- a/deploy/kubernetes-1.24/pmem-csi-direct.yaml +++ b/deploy/kubernetes-1.24/pmem-csi-direct.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 env: - name: TERMINATION_LOG_PATH @@ -397,9 +393,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -407,10 +400,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.24/pmem-csi-lvm-testing.yaml b/deploy/kubernetes-1.24/pmem-csi-lvm-testing.yaml index 44e94397d..8eda1e6dc 100644 --- a/deploy/kubernetes-1.24/pmem-csi-lvm-testing.yaml +++ b/deploy/kubernetes-1.24/pmem-csi-lvm-testing.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 env: @@ -398,9 +394,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -408,10 +401,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.24/pmem-csi-lvm.yaml b/deploy/kubernetes-1.24/pmem-csi-lvm.yaml index 2cb71513f..9d7b10bc9 100644 --- a/deploy/kubernetes-1.24/pmem-csi-lvm.yaml +++ b/deploy/kubernetes-1.24/pmem-csi-lvm.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 env: - name: TERMINATION_LOG_PATH @@ -397,9 +393,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -407,10 +400,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.25/direct/pmem-csi.yaml b/deploy/kubernetes-1.25/direct/pmem-csi.yaml index bb5841a52..7c7f6fe52 100644 --- a/deploy/kubernetes-1.25/direct/pmem-csi.yaml +++ b/deploy/kubernetes-1.25/direct/pmem-csi.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 env: - name: TERMINATION_LOG_PATH @@ -397,9 +393,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -407,10 +400,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.25/direct/testing/pmem-csi.yaml b/deploy/kubernetes-1.25/direct/testing/pmem-csi.yaml index 661d3cff6..53473738f 100644 --- a/deploy/kubernetes-1.25/direct/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.25/direct/testing/pmem-csi.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 env: @@ -398,9 +394,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -408,10 +401,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.25/lvm/pmem-csi.yaml b/deploy/kubernetes-1.25/lvm/pmem-csi.yaml index 2cb71513f..9d7b10bc9 100644 --- a/deploy/kubernetes-1.25/lvm/pmem-csi.yaml +++ b/deploy/kubernetes-1.25/lvm/pmem-csi.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 env: - name: TERMINATION_LOG_PATH @@ -397,9 +393,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -407,10 +400,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.25/lvm/testing/pmem-csi.yaml b/deploy/kubernetes-1.25/lvm/testing/pmem-csi.yaml index 44e94397d..8eda1e6dc 100644 --- a/deploy/kubernetes-1.25/lvm/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.25/lvm/testing/pmem-csi.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 env: @@ -398,9 +394,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -408,10 +401,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.25/pmem-csi-direct-testing.yaml b/deploy/kubernetes-1.25/pmem-csi-direct-testing.yaml index 661d3cff6..53473738f 100644 --- a/deploy/kubernetes-1.25/pmem-csi-direct-testing.yaml +++ b/deploy/kubernetes-1.25/pmem-csi-direct-testing.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 env: @@ -398,9 +394,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -408,10 +401,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.25/pmem-csi-direct.yaml b/deploy/kubernetes-1.25/pmem-csi-direct.yaml index bb5841a52..7c7f6fe52 100644 --- a/deploy/kubernetes-1.25/pmem-csi-direct.yaml +++ b/deploy/kubernetes-1.25/pmem-csi-direct.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 env: - name: TERMINATION_LOG_PATH @@ -397,9 +393,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -407,10 +400,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.25/pmem-csi-lvm-testing.yaml b/deploy/kubernetes-1.25/pmem-csi-lvm-testing.yaml index 44e94397d..8eda1e6dc 100644 --- a/deploy/kubernetes-1.25/pmem-csi-lvm-testing.yaml +++ b/deploy/kubernetes-1.25/pmem-csi-lvm-testing.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 - -v=5 env: @@ -398,9 +394,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -408,10 +401,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kubernetes-1.25/pmem-csi-lvm.yaml b/deploy/kubernetes-1.25/pmem-csi-lvm.yaml index 2cb71513f..9d7b10bc9 100644 --- a/deploy/kubernetes-1.25/pmem-csi-lvm.yaml +++ b/deploy/kubernetes-1.25/pmem-csi-lvm.yaml @@ -352,10 +352,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 - -metricsListen=:10010 env: - name: TERMINATION_LOG_PATH @@ -397,9 +393,6 @@ spec: successThreshold: 1 timeoutSeconds: 5 terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /certs - name: webhook-cert priorityClassName: system-cluster-critical serviceAccountName: pmem-csi-intel-com-webhooks tolerations: @@ -407,10 +400,6 @@ spec: operator: Exists - effect: NoExecute operator: Exists - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- apiVersion: apps/v1 kind: DaemonSet diff --git a/deploy/kustomize/driver/pmem-csi.yaml b/deploy/kustomize/driver/pmem-csi.yaml index 401d48963..64ff0c3ca 100644 --- a/deploy/kustomize/driver/pmem-csi.yaml +++ b/deploy/kustomize/driver/pmem-csi.yaml @@ -162,12 +162,6 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - # ca.crt is present in pmem-csi-intel-com-controller-secret but not required for anything at - # the moment. - - -caFile= - - -certFile=/certs/tls.crt - - -keyFile=/certs/tls.key - - -schedulerListen=:8000 securityContext: readOnlyRootFilesystem: true resources: @@ -175,9 +169,6 @@ spec: memory: 128Mi cpu: 12m terminationMessagePath: /dev/termination-log - volumeMounts: - - name: webhook-cert - mountPath: /certs env: - name: TERMINATION_LOG_PATH value: /dev/termination-log @@ -187,10 +178,6 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - volumes: - - name: webhook-cert - secret: - secretName: pmem-csi-intel-com-controller-secret --- kind: DaemonSet apiVersion: apps/v1 diff --git a/deploy/kustomize/driver/webhooks-rbac.yaml b/deploy/kustomize/driver/webhooks-rbac.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/deploy/kustomize/scheduler/openshift-configmap.yaml b/deploy/kustomize/scheduler/openshift-configmap.yaml deleted file mode 100644 index 620efee91..000000000 --- a/deploy/kustomize/scheduler/openshift-configmap.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: scheduler-policy - namespace: openshift-config -data: - policy.cfg: | - { - "kind" : "Policy", - "apiVersion" : "v1", - "extenders" : [ - {"urlPrefix": "https://127.0.0.1:30674", - "filterVerb": "filter", - "prioritizeVerb": "prioritize", - "nodeCacheCapable": true, - "weight": 1, - "managedResources": [ { - "name": "pmem-csi.intel.com/scheduler", - "ignoredByScheduler": true - } ] - } - ] - } diff --git a/deploy/kustomize/scheduler/scheduler-service.yaml b/deploy/kustomize/scheduler/scheduler-service.yaml deleted file mode 100644 index 7d44c76f6..000000000 --- a/deploy/kustomize/scheduler/scheduler-service.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: pmem-csi-intel-com-scheduler - namespace: pmem-csi -spec: - selector: - app.kubernetes.io/name: pmem-csi-controller - app.kubernetes.io/instance: pmem-csi.intel.com - type: ClusterIP - ports: - - targetPort: 8000 - port: 443 diff --git a/deploy/kustomize/testing/controller-coverage-patch.yaml b/deploy/kustomize/testing/controller-coverage-patch.yaml index 518cb7ed2..4be718870 100644 --- a/deploy/kustomize/testing/controller-coverage-patch.yaml +++ b/deploy/kustomize/testing/controller-coverage-patch.yaml @@ -3,14 +3,14 @@ path: /spec/template/spec/containers/0/command/- value: -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out - op: add - path: /spec/template/spec/containers/0/volumeMounts/- + path: /spec/template/spec/containers/0/volumeMounts value: - mountPath: /var/lib/pmem-csi-coverage - name: coverage-dir + - mountPath: /var/lib/pmem-csi-coverage + name: coverage-dir - op: add - path: /spec/template/spec/volumes/- + path: /spec/template/spec/volumes value: - name: coverage-dir - hostPath: - path: /var/lib/pmem-csi-coverage - type: DirectoryOrCreate + - name: coverage-dir + hostPath: + path: /var/lib/pmem-csi-coverage + type: DirectoryOrCreate diff --git a/deploy/yamls.go b/deploy/yamls.go index f0a45e2da..869d00f14 100644 --- a/deploy/yamls.go +++ b/deploy/yamls.go @@ -17,9 +17,6 @@ import ( ) //go:embed kubernetes-*/pmem-csi-*.yaml -//go:embed kustomize/webhook/webhook.yaml -//go:embed kustomize/scheduler/scheduler-service.yaml -//go:embed kustomize/webhook/webhook-service.yaml var assets embed.FS // YamlFile contains all objects of a certain deployment. diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index c72507d28..e2fd881f2 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -177,7 +177,6 @@ Network ports are opened as configured in manifest files: - metrics endpoint: typical port values 10010 (PMEM-CSI) and 10011 (external-provisioner) -- webhook endpoint: disabled by default, port chosen when [enabling the scheduler extensions](install.md#enable-scheduler-extensions) ### Local sockets diff --git a/docs/autotest.md b/docs/autotest.md index ed30320d3..49283a1e2 100644 --- a/docs/autotest.md +++ b/docs/autotest.md @@ -170,14 +170,6 @@ ssh igk-1 "$@" - Label all worker nodes with PMEM with `storage=pmem` and `feature.node.kubernetes.io/memory-nv.dax=true`. - Delete all namespaces. -- On OpenShift 4.6 and 4.7: create the `scheduler-policy` config map with a fixed - host port (default: `TEST_SCHEDULER_EXTENDER_NODE_PORT=32000`) and reconfigure - `scheduler/cluster` as explained - in [OpenShift scheduler configuration](install.md#openshift-scheduler-configuration). - There is one crucial difference for the config map: in the `managedResources` array, - it must also include an entry for `second.pmem-csi.intel.com/scheduler` (use by - operator-lvm-production). The corresponding service will be created when deploying - PMEM-CSI as part of the tests. - Symlink `_work/` to the directory. - Push a pmem-csi-driver image to a registry that the cluster can pull from. The normal `make push-images` and `make test_e2e` work when overriding the test config diff --git a/docs/design.md b/docs/design.md index 56a35938a..fec6cb220 100644 --- a/docs/design.md +++ b/docs/design.md @@ -186,13 +186,6 @@ provisioning"](https://github.com/kubernetes-csi/external-provisioner/tree/v2.1. have been created on a node which has insufficient RAM and CPU resources for a pod. -PMEM-CSI also has a central component which implements the [scheduler -extender](#scheduler-extender) webhook. That component needs to know -on which nodes the PMEM-CSI driver is running and how much capacity is -available there. This information is retrieved by dynamically -discovering PMEM-CSI pods and connecting to their [metrics -endpoint](/docs/install.md#metrics-support). - ## Communication between components The following diagram illustrates the communication channels between driver components: @@ -206,26 +199,6 @@ confidential and therefore offered without access control via HTTP. This also simplifies scraping that data with tools like Prometheus. -The communication between Kubernetes and the scheduler extender -webhook is protected by TLS because this is encouraged and supported -by Kubernetes. But as the webhook only exposes information that is -already available, it accepts all incoming connection without -checking the client certificate. - -The [`test/setup-ca.sh`](/test/setup-ca.sh) -script shows how to generate self-signed certificates. The test cluster is set -up using certificates created by that script, with secrets prepared by -[`test/setup-deployment.sh`](/test/setup-deployment.sh) before -deploying the driver using the provided [deployment files](/deploy/). - -Beware that these are just examples. Administrators of a cluster must -ensure that they choose key lengths and algorithms of sufficient -strength for their purposes and manage certificate distribution. - -A production deployment can improve upon that by using some other key -delivery mechanism, like for example -[Vault](https://www.vaultproject.io/). - The PMEM-CSI controller runs with the default security context. On upstream Kubernetes, this means that it runs as root. The expectation is that actual production deployments of PMEM-CSI will avoid that, for @@ -339,77 +312,6 @@ pod scheduling with late binding of volumes. Until that feature becomes generally available, PMEM-CSI provides two components that help with pod scheduling: -### Scheduler extender - -When a pod requests a special [extended -resource](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#extended-resources) -, the Kubernetes scheduler calls -a [scheduler -extender](https://github.com/kubernetes/design-proposals-archive/blob/main/scheduling/scheduler_extender.md) -provided by PMEM-CSI with a list of nodes that a pod might run -on. - -The name of that special resource is `/scheduler`, -i.e. `pmem-csi.intel.com/scheduler` when the default PMEM-CSI driver -name is used. It is possible to configure one extender per PMEM-CSI -deployment because each deployment has its own unique driver name. - -This extender is implemented in the PMEM-CSI controller and retrieves -metrics data from each PMEM-CSI node driver instance to filter out all -nodes which currently do not -have enough storage left for the volumes that still need to be -created. This considers inline ephemeral volumes and all unbound -volumes, regardless whether they use late binding or immediate -binding. - -This special scheduling can be requested manually by adding this snippet -to one container in the pod spec: -``` -containers: -- name: some-container - ... - resources: - limits: - pmem-csi.intel.com/scheduler: "1" - requests: - pmem-csi.intel.com/scheduler: "1" -``` - -This scheduler extender is optional and not necessarily installed in -all clusters that have PMEM-CSI. Don't add this extended resource -unless the scheduler extender is installed, otherwise the pod won't -start! - -See our [implementation](http://github.com/intel/pmem-csi/tree/devel/pkg/scheduler) of a scheduler extender. - -### Pod admission webhook - -Having to add the `/scheduler` extended resource manually is not -user-friendly. To simplify this, PMEM-CSI provides a [mutating -admission -webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/) -which intercepts the creation of all pods. If that pod uses inline -ephemeral volumes or volumes with late binding that are provided by -PMEM-CSI, the webhook transparently adds the extended resource -request. PMEM-CSI volumes with immediate binding are ignored because -for those the normal topology support ensures that unsuitable nodes -are filtered out. - -The webhook can only do that if the persistent volume claim (PVC) and -its storage class have been created already. This is normally not -required: it's okay to create the pod first, then later add the -PVC. The pod simply won't start in the meantime. - -The webhook deals with this uncertainty by allowing the creation of -the pod without adding the extended resource when it lacks the -necessary information. The alternative would be to reject the pod, but -that would be a change of behavior of the cluster that may affect also pods -that don't use PMEM-CSI at all. - -Users must take care to create PVCs first, then the pods if they want -to use the webhook. In practice, that is often already done because it -is more natural, so it is not a big limitation. - ## PMEM-CSI Operator PMEM-CSI operator facilitates deploying and managing the [PMEM-CSI driver](https://github.com/intel/pmem-csi) diff --git a/docs/install.md b/docs/install.md index 1fb1cabb0..7e1e37334 100644 --- a/docs/install.md +++ b/docs/install.md @@ -41,26 +41,15 @@ installation by name, which indirectly determines the device mode. A storage class also chooses which filesystem is used (xfs or ext4) and enables [Kata Containers support](#kata-containers-support). -Optionally, the administrator can enable [the scheduler -extensions](#enable-scheduler-extensions) and monitoring of resource +Optionally, the administrator can enable monitoring of resource usage via the [metrics support](#metrics-support). It is [recommended](./design.md#dynamic-provisioning-of-local-volumes) -to enable the scheduler extensions and use +to use `volumeBindingMode: WaitForFirstConsumer` as in the [`pmem-storageclass-late-binding.yaml`](/deploy/common/pmem-storageclass-late-binding.yaml) example. This ensures that pods get scheduled onto nodes that have -sufficient RAM, CPU and PMEM. Without the scheduler extensions, it is -random whether the scheduler picks a node that has PMEM available and -immediate binding (the default volume binding mode) might work -better. However, then pods might not be able to run when the node -where volumes were created are overloaded. - -Starting with Kubernetes 1.21, PMEM-CSI uses [storage capacity -tracking](https://kubernetes.io/docs/concepts/storage/storage-capacity/) -to handle Pod scheduling and the scheduler extensions are not needed -anymore. `WaitForFirstConsumer` still is the recommended volume -binding mode. +sufficient RAM, CPU and PMEM. Optionally, the log output format can be changed from the default "text" format (= the traditional glog format) to "json" (= output via @@ -348,11 +337,7 @@ for a complete list of supported properties. Here is a minimal example driver deployment created with a custom resource: **NOTE**: `nodeSelector` must match the node label that was set in the -[installation and setup](#installation-and-setup) section. The PMEM-CSI -[scheduler extender](design.md#scheduler-extender) and -[webhook](design.md#pod-admission-webhook) are not enabled in this basic -installation. See [below](#enable-scheduler-extensions) for -instructions about that. +[installation and setup](#installation-and-setup) section. ``` ShellSession $ kubectl create -f - <= 1.14 includes some support for that. The sub-directories @@ -701,71 +648,6 @@ This can be checked with `kubectl get pods --all-namespaces -o wide`. Have nodes been labeled as expected by the driver deployment? Check with `kubectl get nodes -o yaml`. -#### Example Pod getting assigned to a node with no PMEM - -This can happen on clusters where only some worker nodes have PMEM and -the [PMEM-CSI scheduler extensions](#enable-scheduler-extensions) are -not enabled. This can be checked by looking at the `selected-node` -annotation of the PVC: -``` console -$ kubectl get pvc/pmem-csi-pvc-late-binding -o yaml | grep ' volume.kubernetes.io/selected-node:' - volume.kubernetes.io/selected-node: pmem-csi-pmem-govm-worker2 -``` - -The PMEM-CSI controller pod will detect this and ask the scheduler to -pick a node anew by removing that annotation, but it is random whether -the next choice is better and starting the Pod may get delayed. - -To avoid this, enable the scheduler extensions. - -#### Example Pod getting assigned to a node with insufficient PMEM - -This also can only happen when the [PMEM-CSI scheduler -extensions](#enable-scheduler-extensions) are not enabled. Then volume -creation is attempted repeatedly, potentially on different nodes, but -fails with `not enough space` errors: - -``` console -$ kubectl describe pvc/pmem-csi-pvc-late-binding -Name: pmem-csi-pvc-late-binding -Namespace: default -StorageClass: pmem-csi-sc-late-binding -Status: Pending -Volume: -Labels: -Annotations: volume.beta.kubernetes.io/storage-provisioner: pmem-csi.intel.com -Finalizers: [kubernetes.io/pvc-protection] -Capacity: -Access Modes: -VolumeMode: Filesystem -Used By: my-csi-app -Events: - Type Reason Age From Message - ---- ------ ---- ---- ------- - Normal WaitForFirstConsumer 7m30s persistentvolume-controller waiting for first consumer to be created before binding - Normal WaitForPodScheduled 6m (x15 over 7m19s) persistentvolume-controller waiting for pod my-csi-app to be scheduled - Warning ProvisioningFailed 3m59s (x12 over 7m19s) pmem-csi.intel.com_pmem-csi-intel-com-node-nwkqv_cc2984e6-915f-4cf2-93a0-e143da407917 failed to provision volume with StorageClass "pmem-csi-sc-late-binding": rpc error: code = ResourceExhausted desc = Node CreateVolume: device creation failed: not enough space - Warning ProvisioningFailed 2m47s (x12 over 7m18s) pmem-csi.intel.com_pmem-csi-intel-com-node-9vlhf_6ac47898-58bf-45e1-b601-5d8f39d21f4e failed to provision volume with StorageClass "pmem-csi-sc-late-binding": rpc error: code = ResourceExhausted desc = Node CreateVolume: device creation failed: not enough space - Normal ExternalProvisioning 2m23s (x28 over 7m19s) persistentvolume-controller waiting for a volume to be created, either by external provisioner "pmem-csi.intel.com" or manually created by system administrator - Normal Provisioning 2m11s (x14 over 7m18s) pmem-csi.intel.com_pmem-csi-intel-com-node-9vlhf_6ac47898-58bf-45e1-b601-5d8f39d21f4e External provisioner is provisioning volume for claim "default/pmem-csi-pvc-late-binding" - Normal Provisioning 107s (x16 over 7m19s) pmem-csi.intel.com_pmem-csi-intel-com-node-nwkqv_cc2984e6-915f-4cf2-93a0-e143da407917 External provisioner is provisioning volume for claim "default/pmem-csi-pvc-late-binding" -``` - -The scheduler extensions prevent these useless attempts on nodes with -insufficient PMEM. When none of the available nodes have sufficient -PMEM, the attempt to schedule the example Pod fails: - -``` console -$ kubectl describe pod/my-csi-app -Name: my-csi-app -Namespace: default -... -Events: - Type Reason Age From Message - ---- ------ ---- ---- ------- - Warning FailedScheduling 12s (x2 over 12s) default-scheduler 0/4 nodes are available: 1 node(s) had taint {node-role.kubernetes.io/master: }, that the pod didn't tolerate, 3 only 63484MiB of PMEM available, need 400GiB. -``` - #### Less PMEM available than expected This is usually the result of not preparing the node(s) as describe in @@ -872,7 +754,6 @@ Pod Template: app.kubernetes.io/name=pmem-csi-node-setup app.kubernetes.io/part-of=pmem-csi pmem-csi.intel.com/deployment=lvm-production - pmem-csi.intel.com/webhook=ignore Service Account: pmem-csi-intel-com-node-setup Containers: pmem-driver: @@ -1007,11 +888,6 @@ cluster has the feature enabled. See [`pmem-app-generic-ephemeral.yaml`](/deploy/common/pmem-app-generic-ephemeral.yaml) for an example. -When using generic ephemeral inline volumes together with [storage -capacity tracking](#storage-capacity-tracking), the [PMEM-CSI -scheduler extensions](#enable-scheduler-extensions) are not needed -anymore. - ### Raw block volumes Applications can use volumes provisioned by PMEM-CSI as [raw block @@ -1035,403 +911,6 @@ That example demonstrates how to handle some details: - [Kubernetes bug #85624](https://github.com/kubernetes/kubernetes/issues/85624) must be worked around to format and mount the raw block device. -### Enable scheduler extensions - -#### Manual scheduler configuration - -**NOTE**: this sections provides an in-depth explanation that makes no -assumptions about how the cluster works. For simpler install -instructions on OpenShift see [below](#openshift-scheduler-configuration). - -The PMEM-CSI scheduler extender and admission webhook are provided by -the PMEM-CSI controller. They need to be enabled during deployment via -the `--schedulerListen=[]:` parameter. The -listen address is optional and can be left out. The port is where a -HTTPS server will run. The YAML files already enable this. The -operator has the `controllerTLSSecret` and `mutatePods` properties in -the [`DeploymentSpec`](#deploymentspec). - -The controller needs TLS certificates which must be created in -advance. The YAML files expects them in a secret called -`pmem-csi-intel-com-controller-secret` and will not work without one. -The operator is more flexible and creates a driver without the -controller by default. This can be changed by setting the -`controllerTLSSecret` field in the `PmemCSIDeployment` API. - -That secret must contain the following data items: -- `ca.crt`: root CA certificate -- `tls.key`: secret key of the webhook -- `tls.crt`: public key of the webhook - -The webhook certificate must include host names that match how the -webhooks are going to be called by the `kube-apiserver` -(i.e. `pmem-csi-intel-com-scheduler.pmem-csi.svc` for a -deployment with the `pmem-csi.intel.com` driver name in the `pmem-csi` -namespace) and by the `kube-scheduler` (might be the same service name, through some external -load balancer or `127.0.0.1` when using the node port workaround -described below). - -To enable the PMEM-CSI scheduler extender, a configuration file and an -additional `--config` parameter for `kube-scheduler` must be added to -the cluster control plane, or, if there is already such a -configuration file, one new entry must be added to the `extenders` -array. A full example is presented below. - -The `kube-scheduler` must be able to connect to the PMEM-CSI -controller via the `urlPrefix` in its configuration. In some clusters -it is possible to use cluster DNS and thus a symbolic service name. If -that is the case, then deploy the [scheduler -service](/deploy/kustomize/scheduler/scheduler-service.yaml) as-is -and use `https://pmem-csi-scheduler.default.svc` as `urlPrefix`. If -the PMEM-CSI driver is deployed in a namespace, replace `default` with -the name of that namespace. - -In a cluster created with kubeadm, `kube-scheduler` is unable to use -cluster DNS because the pod it runs in is configured with -`hostNetwork: true` and without `dnsPolicy`. Therefore the cluster DNS -servers are ignored. There also is no special dialer as in other -clusters. As a workaround, the PMEM-CSI service can be exposed via a -fixed node port like 32000 on all nodes. Then -`https://127.0.0.1:32000` needs to be used as `urlPrefix`. Here's how -the service can be created with that node port: - -``` ShellSession -$ mkdir my-scheduler - -$ cat >my-scheduler/kustomization.yaml <my-scheduler/node-port-patch.yaml </var/lib/scheduler/scheduler-policy.cfg' <:", - "filterVerb": "filter", - "prioritizeVerb": "prioritize", - "nodeCacheCapable": true, - "weight": 1, - "managedResources": - [{ - "name": "pmem-csi.intel.com/scheduler", - "ignoredByScheduler": true - }] - }] -} -EOF - -# https://github.com/kubernetes/kubernetes/blob/52d7614a8ca5b8aebc45333b6dc8fbf86a5e7ddf/staging/src/k8s.io/kube-scheduler/config/v1alpha1/types.go#L38-L107 -$ sudo sh -c 'cat >/var/lib/scheduler/scheduler-config.yaml' <kubeadm.config <= 1.19 are: - -``` ShellSession -$ sudo mkdir -p /var/lib/scheduler/ -$ sudo cp _work/pmem-ca/ca.pem /var/lib/scheduler/ca.crt - -# https://github.com/kubernetes/kubernetes/blob/1afc53514032a44d091ae4a9f6e092171db9fe10/staging/src/k8s.io/kube-scheduler/config/v1beta1/types.go#L44-L96 -$ sudo sh -c 'cat >/var/lib/scheduler/scheduler-config.yaml' <: - filterVerb: filter - prioritizeVerb: prioritize - nodeCacheCapable: true - weight: 1 - managedResources: - - name: pmem-csi.intel.com/scheduler - ignoredByScheduler: true -EOF - -$ cat >kubeadm.config <= -1.15, it can also be used to let individual pods bypass the webhook by -adding that label. The CA gets configured explicitly, which is -supported for webhooks. - -``` ShellSession -$ mkdir my-webhook - -$ cat >my-webhook/kustomization.yaml <my-webhook/webhook-patch.yaml <3 | | deviceMode | string | Device management mode to use. Supports one of `lvm` or `direct` | `lvm` -| controllerTLSSecret | string | Name of an existing secret in the driver's namespace which contains ca.crt, tls.crt and tls.key data for the scheduler extender and pod mutation webhook. A controller is started if (and only if) this secret is specified.
Alternatively, the special string `-openshift-` can be used on OpenShift to let OpenShift create the necessary secrets. | empty | controllerReplicas | int | Number of concurrently running controller pods. | 1 -| mutatePods | Always/Try/Never | Defines how a mutating pod webhook is configured if a controller is started. The field is ignored if the controller is not enabled. "Never" disables pod mutation. "Try" configured it so that pod creation is allowed to proceed even when the webhook fails. "Always" requires that the webhook gets invoked successfully before creating a pod. | Try -| schedulerNodePort | int or string | If non-zero, the scheduler service is created as a NodeService with that fixed port number. Otherwise that service is created as a cluster service. The number must be from the range reserved by Kubernetes for node ports. This is useful if the kube-scheduler cannot reach the scheduler extender via a cluster service. | 0 | controllerResources | [ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#resourcerequirements-v1-core) | Describes the compute resource requirements for controller pod.
4_Deprecated and only available in `v1alpha1`._ | | nodeResources | [ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#resourcerequirements-v1-core) | Describes the compute resource requirements for the pods running on node(s).
_4Deprecated and only available in `v1alpha1`._ | | controllerDriverResources | [ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#resourcerequirements-v1-core) | Describes the compute resource requirements for controller driver container running on master node. Available since `v1beta1`. | diff --git a/go.mod b/go.mod index c8c0cc5c2..9550c8cb7 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,6 @@ require ( k8s.io/client-go v12.0.0+incompatible k8s.io/component-base v0.25.0 k8s.io/klog/v2 v2.70.1 - k8s.io/kube-scheduler v0.25.0 k8s.io/kubectl v1.25.0 k8s.io/kubernetes v1.25.0 k8s.io/pod-security-admission v0.0.0 diff --git a/go.sum b/go.sum index 21e8a3b72..6d0f110bf 100644 --- a/go.sum +++ b/go.sum @@ -1179,8 +1179,6 @@ k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ= k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA= k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= -k8s.io/kube-scheduler v0.25.0 h1:Up2rW+1H3JsgcpfdMcj/kVbYtgoxpiwxKLg5L4PaZ98= -k8s.io/kube-scheduler v0.25.0/go.mod h1:cwiyJeImgFbhmbnImzvuhbiJayNngRNEe3FJkZDPw9Y= k8s.io/kubectl v0.25.0 h1:/Wn1cFqo8ik3iee1EvpxYre3bkWsGLXzLQI6uCCAkQc= k8s.io/kubectl v0.25.0/go.mod h1:n16ULWsOl2jmQpzt2o7Dud1t4o0+Y186ICb4O+GwKAU= k8s.io/kubelet v0.25.0 h1:eTS5B1u1o63ndExAHKLJytzz/GBy86ROcxYtu0VK3RA= diff --git a/operator/operator.make b/operator/operator.make index 16cbd2327..3f605e8c5 100644 --- a/operator/operator.make +++ b/operator/operator.make @@ -5,6 +5,8 @@ CONTROLLER_GEN_VERSION=v0.8.0 CONTROLLER_GEN=_work/bin/controller-gen-$(CONTROLLER_GEN_VERSION) # download operator-sdk binary +_work/.operator-sdk-stamp: _work/bin/operator-sdk-$(OPERATOR_SDK_VERSION) + touch $@ _work/bin/operator-sdk-$(OPERATOR_SDK_VERSION): mkdir -p $(@D) curl -L https://github.com/operator-framework/operator-sdk/releases/download/v$(OPERATOR_SDK_VERSION)/operator-sdk_linux_amd64 -o $@ diff --git a/pkg/apis/pmemcsi/v1beta1/deployment_types.go b/pkg/apis/pmemcsi/v1beta1/deployment_types.go index 0a16c2016..ef35afcac 100644 --- a/pkg/apis/pmemcsi/v1beta1/deployment_types.go +++ b/pkg/apis/pmemcsi/v1beta1/deployment_types.go @@ -103,12 +103,11 @@ type DeploymentSpec struct { NodeDriverResources *corev1.ResourceRequirements `json:"nodeDriverResources,omitempty"` // ControllerDriverResources Compute resources required by central driver container ControllerDriverResources *corev1.ResourceRequirements `json:"controllerDriverResources,omitempty"` - // ControllerTLSSecret is the name of a secret which contains ca.crt, tls.crt and tls.key data - // for the scheduler extender and pod mutation webhook. A controller is started if (and only if) - // this secret is specified. The special string "-openshift-" enables the usage of - // https://docs.openshift.com/container-platform/4.6/security/certificates/service-serving-certificate.html - // to create certificates. - ControllerTLSSecret string `json:"controllerTLSSecret,omitempty"` + // ControllerTLSSecret used to be the name of a secret which contains ca.crt, tls.crt and tls.key data + // for the scheduler extender and pod mutation webhook. It is now unused. + // + // DEPRECATED + DeprecatedControllerTLSSecret string `json:"controllerTLSSecret,omitempty"` // ControllerReplicas determines how many copys of the controller Pod run concurrently. // Zero (= unset) selects the builtin default, which is currently 1. // +kubebuilder:validation:Minimum=0 @@ -116,15 +115,19 @@ type DeploymentSpec struct { // MutatePod defines how a mutating pod webhook is configured if a controller // is started. The field is ignored if the controller is not enabled. // The default is "Try". + // + // DEPRECATED // +kubebuilder:validation:Enum=Always;Try;Never - MutatePods MutatePods `json:"mutatePods,omitempty"` + DeprecatedMutatePods MutatePods `json:"mutatePods,omitempty"` // SchedulerNodePort, if non-zero, ensures that the "scheduler" service // is created as a NodeService with that fixed port number. Otherwise // that service is created as a cluster service. The number must be // from the range reserved by Kubernetes for // node ports. This is useful if the kube-scheduler cannot reach the scheduler // extender via a cluster service. - SchedulerNodePort int32 `json:"schedulerNodePort,omitempty"` + // + // DEPRECATED + DeprecatedSchedulerNodePort int32 `json:"schedulerNodePort,omitempty"` // DeviceMode to use to manage PMEM devices. // +kubebuilder:validation:Enum=lvm;direct DeviceMode DeviceMode `json:"deviceMode,omitempty"` @@ -277,8 +280,6 @@ const ( // DefaultDriverImage default PMEM-CSI driver docker image DefaultDriverImage = defaultDriverImageName + ":" + defaultDriverImageTag - DefaultMutatePods = MutatePodsTry - // The sidecar versions must be kept in sync with the // deploy/kustomize YAML files! hack/bump-image-versions.sh // can be used to update both. @@ -389,14 +390,6 @@ func (d *PmemCSIDeployment) EnsureDefaults(operatorImage string) error { return fmt.Errorf("invalid device mode %q", d.Spec.DeviceMode) } - switch d.Spec.MutatePods { - case "": - d.Spec.MutatePods = DefaultMutatePods - case MutatePodsAlways, MutatePodsTry, MutatePodsNever: - default: - return fmt.Errorf("invalid MutatePods value: %s", d.Spec.MutatePods) - } - if d.Spec.Image == "" { // If provided use operatorImage if operatorImage != "" { @@ -482,24 +475,6 @@ func (d *PmemCSIDeployment) GetHyphenedName() string { return strings.ReplaceAll(d.GetName(), ".", "-") } -// ControllerTLSSecretOpenshiftName returns the name of the secret that -// we want OpenShift to create for the controller service. -func (d *PmemCSIDeployment) ControllerTLSSecretOpenshiftName() string { - return d.GetHyphenedName() + "-openshift-controller-tls" -} - -// RegistrySecretName returns the name of the registry -// Secret object used by the deployment -func (d *PmemCSIDeployment) RegistrySecretName() string { - return d.GetHyphenedName() + "-registry-secrets" -} - -// NodeSecretName returns the name of the node-controller -// Secret object used by the deployment -func (d *PmemCSIDeployment) NodeSecretName() string { - return d.GetHyphenedName() + "-node-secrets" -} - // CSIDriverName returns the name of the CSIDriver // object name for the deployment func (d *PmemCSIDeployment) CSIDriverName() string { @@ -512,12 +487,6 @@ func (d *PmemCSIDeployment) MetricsServiceName() string { return d.GetHyphenedName() + "-metrics" } -// SchedulerServiceName returns the name of the controller's -// Service object for the scheduler extender. -func (d *PmemCSIDeployment) SchedulerServiceName() string { - return d.GetHyphenedName() + "-scheduler" -} - // SchedulerServiceName returns the name of the controller's // Service object for the webhooks. func (d *PmemCSIDeployment) WebhooksServiceName() string { @@ -554,12 +523,6 @@ func (d *PmemCSIDeployment) WebhooksClusterRoleBindingName() string { return d.GetHyphenedName() + "-webhooks-role" } -// MutatingWebhookName returns the name of the -// MutatingWebhookConfiguration -func (d *PmemCSIDeployment) MutatingWebhookName() string { - return d.GetHyphenedName() + "-hook" -} - // NodeServiceAccountName returns the name of the service account // used by the DaemonSet with the external-provisioner func (d *PmemCSIDeployment) ProvisionerServiceAccountName() string { diff --git a/pkg/deployments/load.go b/pkg/deployments/load.go index 817655fa4..7e8245398 100644 --- a/pkg/deployments/load.go +++ b/pkg/deployments/load.go @@ -8,7 +8,6 @@ package deployments import ( "bytes" - "encoding/base64" "encoding/json" "fmt" "regexp" @@ -38,7 +37,6 @@ var driverNameRegex = regexp.MustCompile(`(?m)(name|app\.kubernetes.io/instance) // and updates them on-the-fly according to the deployment spec, namespace and name. func LoadAndCustomizeObjects(kubernetes version.Version, deviceMode api.DeviceMode, namespace string, deployment api.PmemCSIDeployment, - controllerCABundle []byte, ) ([]unstructured.Unstructured, error) { // Conceptually this function is similar to calling "kustomize" for @@ -153,47 +151,6 @@ func LoadAndCustomizeObjects(kubernetes version.Version, deviceMode api.DeviceMo spec["nodeSelector"] = selector } } - case "MutatingWebhookConfiguration": - webhooks := obj.Object["webhooks"].([]interface{}) - failurePolicy := "Ignore" - if deployment.Spec.MutatePods == api.MutatePodsAlways { - failurePolicy = "Fail" - } - webhook := webhooks[0].(map[string]interface{}) - webhook["failurePolicy"] = failurePolicy - clientConfig := webhook["clientConfig"].(map[string]interface{}) - if controllerCABundle != nil { - clientConfig["caBundle"] = base64.StdEncoding.EncodeToString(controllerCABundle) - - } - if deployment.Spec.ControllerTLSSecret == api.ControllerTLSSecretOpenshift { - meta := obj.Object["metadata"].(map[string]interface{}) - meta["annotations"] = map[string]string{ - "service.beta.openshift.io/inject-cabundle": "true", - } - } - case "Service": - switch obj.GetName() { - case deployment.SchedulerServiceName(): - spec := obj.Object["spec"].(map[string]interface{}) - ports := spec["ports"].([]interface{}) - port0 := ports[0].(map[string]interface{}) - if deployment.Spec.SchedulerNodePort != 0 { - spec["type"] = "NodePort" - port0["nodePort"] = deployment.Spec.SchedulerNodePort - } - if deployment.Spec.ControllerTLSSecret == api.ControllerTLSSecretOpenshift { - port0["targetPort"] = 8001 - port0["port"] = 80 - } - case deployment.WebhooksServiceName(): - if deployment.Spec.ControllerTLSSecret == api.ControllerTLSSecretOpenshift { - meta := obj.Object["metadata"].(map[string]interface{}) - meta["annotations"] = map[string]string{ - "service.beta.openshift.io/serving-cert-secret-name": deployment.ControllerTLSSecretOpenshiftName(), - } - } - } } } @@ -202,25 +159,6 @@ func LoadAndCustomizeObjects(kubernetes version.Version, deviceMode api.DeviceMo return nil, err } - scheduler, err := loadYAML("kustomize/scheduler/scheduler-service.yaml", patchYAML, enabled, patchUnstructured) - if err != nil { - return nil, err - } - objects = append(objects, scheduler...) - - if deployment.Spec.ControllerTLSSecret != "" && deployment.Spec.MutatePods != api.MutatePodsNever { - webhook, err := loadYAML("kustomize/webhook/webhook.yaml", patchYAML, enabled, patchUnstructured) - if err != nil { - return nil, err - } - objects = append(objects, webhook...) - service, err := loadYAML("kustomize/webhook/webhook-service.yaml", patchYAML, enabled, patchUnstructured) - if err != nil { - return nil, err - } - objects = append(objects, service...) - } - return objects, nil } @@ -230,10 +168,6 @@ func patchPodTemplate(obj *unstructured.Unstructured, deployment api.PmemCSIDepl spec := template["spec"].(map[string]interface{}) metadata := template["metadata"].(map[string]interface{}) - isController := strings.Contains(obj.GetName(), "controller") - stripTLS := isController && deployment.Spec.ControllerTLSSecret == "" - openshiftTLS := isController && deployment.Spec.ControllerTLSSecret == api.ControllerTLSSecretOpenshift - if deployment.Spec.Labels != nil { labels := metadata["labels"] var labelsMap map[string]interface{} @@ -265,10 +199,6 @@ func patchPodTemplate(obj *unstructured.Unstructured, deployment api.PmemCSIDepl return obj, nil } - if stripTLS { - spec["volumes"] = nil - } - containers := spec["containers"].([]interface{}) for _, container := range containers { container := container.(map[string]interface{}) @@ -279,36 +209,6 @@ func patchPodTemplate(obj *unstructured.Unstructured, deployment api.PmemCSIDepl } container["resources"] = obj - if stripTLS && container["name"].(string) == "pmem-driver" { - container["volumeMounts"] = nil - var command []interface{} - for _, arg := range container["command"].([]interface{}) { - switch strings.Split(arg.(string), "=")[0] { - case "-caFile", - "-certFile", - "-keyFile", - "-schedulerListen": - // remove these parameters - default: - command = append(command, arg) - } - } - container["command"] = command - } - - if openshiftTLS && container["name"].(string) == "pmem-driver" { - var command []interface{} - for _, arg := range container["command"].([]interface{}) { - switch arg.(string) { - case "-schedulerListen=:8000": - command = append(command, arg, "-insecureSchedulerListen=:8001") - default: - command = append(command, arg) - } - } - container["command"] = command - } - // Override driver name in env var. env := container["env"] if env != nil { @@ -342,21 +242,6 @@ func patchPodTemplate(obj *unstructured.Unstructured, deployment api.PmemCSIDepl container["image"] = image } } - - if deployment.Spec.ControllerTLSSecret != "" { - volumes := spec["volumes"].([]interface{}) - for _, volume := range volumes { - volume := volume.(map[string]interface{}) - volumeName := volume["name"].(string) - if volumeName == "webhook-cert" { - name := deployment.Spec.ControllerTLSSecret - if name == api.ControllerTLSSecretOpenshift { - name = deployment.ControllerTLSSecretOpenshiftName() - } - volume["secret"].(map[string]interface{})["secretName"] = name - } - } - } return nil } diff --git a/pkg/deployments/load_test.go b/pkg/deployments/load_test.go index bc635d1dc..4c9ed82d4 100644 --- a/pkg/deployments/load_test.go +++ b/pkg/deployments/load_test.go @@ -51,7 +51,7 @@ func TestLoadObjects(t *testing.T) { Name: "pmem-csi.example.org", }, } - objects, err = deployments.LoadAndCustomizeObjects(testCase.Kubernetes, testCase.DeviceMode, namespace, deployment, nil) + objects, err = deployments.LoadAndCustomizeObjects(testCase.Kubernetes, testCase.DeviceMode, namespace, deployment) if assert.NoError(t, err, "load and customize yaml") { assert.NotEmpty(t, objects, "have customized objects") diff --git a/pkg/pmem-csi-driver/main.go b/pkg/pmem-csi-driver/main.go index 18ae8212f..2e0011cf8 100644 --- a/pkg/pmem-csi-driver/main.go +++ b/pkg/pmem-csi-driver/main.go @@ -9,7 +9,6 @@ package pmemcsidriver import ( "context" - "errors" "flag" "fmt" @@ -36,10 +35,6 @@ func init() { flag.StringVar(&config.NodeID, "nodeid", "nodeid", "node id") flag.StringVar(&config.Endpoint, "endpoint", "unix:///tmp/pmem-csi.sock", "PMEM CSI endpoint") flag.Var(&config.Mode, "mode", "driver run mode") - flag.StringVar(&config.CAFile, "caFile", "ca.pem", "Root CA certificate file to use for verifying clients (optional, can be empty)") - flag.StringVar(&config.CertFile, "certFile", "pmem-controller.pem", "SSL certificate file to be used by the PMEM-CSI controller") - flag.StringVar(&config.KeyFile, "keyFile", "pmem-controller-key.pem", "Private key file associated with the certificate") - flag.Float64Var(&config.KubeAPIQPS, "kube-api-qps", 5, "QPS to use while communicating with the Kubernetes apiserver. Defaults to 5.0.") flag.IntVar(&config.KubeAPIBurst, "kube-api-burst", 10, "Burst to use while communicating with the Kubernetes apiserver. Defaults to 10.") @@ -48,8 +43,6 @@ func init() { flag.StringVar(&config.metricsPath, "metricsPath", "/metrics", "The HTTP path where prometheus metrics will be exposed. Default is `/metrics`.") /* Controller mode options */ - flag.StringVar(&config.schedulerListen, "schedulerListen", "", "controller: HTTPS listen address (like :8000) for scheduler extender and mutating webhook, disabled by default (needs caFile, certFile, keyFile)") - flag.StringVar(&config.insecureSchedulerListen, "insecureSchedulerListen", "", "controller: HTTP listen address (like :8001) for scheduler extender and mutating webhook, disabled by default (does not use TLS config)") flag.Var(&config.nodeSelector, "nodeSelector", "controller: reschedule PVCs with a selected node where PMEM-CSI is not meant to run because the node does not have these labels (represented as JSON map)") /* Node mode options */ @@ -57,6 +50,14 @@ func init() { flag.StringVar(&config.StateBasePath, "statePath", "", "node: directory path where to persist the state of the driver, defaults to /var/lib/") flag.UintVar(&config.PmemPercentage, "pmemPercentage", 100, "node: percentage of space to be used by the driver in each PMEM region") + // These options no longer have an effect. They don't get removed to + // keep old deployments working when upgrading only the image. + flag.String("caFile", "ca.pem", "Root CA certificate file to use for verifying clients (optional, can be empty) - DEPRECATED!") + flag.String("certFile", "pmem-controller.pem", "SSL certificate file to be used by the PMEM-CSI controller - DEPRECATED!") + flag.String("keyFile", "pmem-controller-key.pem", "Private key file associated with the certificate - DEPRECATED!") + flag.String("schedulerListen", "", "controller: HTTPS listen address (like :8000) for scheduler extender and mutating webhook, disabled by default (needs caFile, certFile, keyFile) - DEPRECATED!") + flag.String("insecureSchedulerListen", "", "controller: HTTP listen address (like :8001) for scheduler extender and mutating webhook, disabled by default (does not use TLS config) - DEPRECATED!") + klog.InitFlags(nil) } @@ -76,11 +77,6 @@ func Main() int { logger.Info("PMEM-CSI started.", "version", version) defer logger.Info("PMEM-CSI stopped.") - if config.schedulerListen != "" && config.Mode != Webhooks { - pmemcommon.ExitError("scheduler listening", errors.New("only supported in the controller")) - return 1 - } - config.Version = version driver, err := GetCSIDriver(config) if err != nil { diff --git a/pkg/pmem-csi-driver/pmem-csi-driver.go b/pkg/pmem-csi-driver/pmem-csi-driver.go index 06372cd09..cea37c482 100644 --- a/pkg/pmem-csi-driver/pmem-csi-driver.go +++ b/pkg/pmem-csi-driver/pmem-csi-driver.go @@ -20,21 +20,19 @@ import ( "syscall" "time" + "k8s.io/client-go/informers" "k8s.io/klog/v2" api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" grpcserver "github.com/intel/pmem-csi/pkg/grpc-server" "github.com/intel/pmem-csi/pkg/k8sutil" pmdmanager "github.com/intel/pmem-csi/pkg/pmem-device-manager" - pmemgrpc "github.com/intel/pmem-csi/pkg/pmem-grpc" pmemstate "github.com/intel/pmem-csi/pkg/pmem-state" - "github.com/intel/pmem-csi/pkg/scheduler" "github.com/intel/pmem-csi/pkg/types" "github.com/kubernetes-csi/csi-lib-utils/metrics" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - "k8s.io/client-go/informers" ) const ( @@ -47,7 +45,7 @@ type DriverMode string func (mode *DriverMode) Set(value string) error { switch value { - case string(Node), string(Webhooks), string(ForceConvertRawNamespaces): + case string(Node), string(Controller), string(ForceConvertRawNamespaces): *mode = DriverMode(value) default: // The flag package will add the value to the final output, no need to do it here. @@ -65,8 +63,8 @@ func (mode *DriverMode) String() string { const ( // Node driver with support for provisioning. Node DriverMode = "node" - // Just the webhooks, using metrics instead of gRPC over TCP. - Webhooks DriverMode = "webhooks" + // The controller with the rescheduler. For historic reasons this is called "webhooks". + Controller DriverMode = "webhooks" // Convert each raw namespace into fsdax. ForceConvertRawNamespaces = "force-convert-raw-namespaces" ) @@ -103,12 +101,6 @@ type Config struct { Endpoint string //Mode mode fo the driver Mode DriverMode - //CAFile Root certificate authority certificate file - CAFile string - //CertFile certificate for server authentication - CertFile string - //KeyFile server private key file - KeyFile string //DeviceManager device manager to use DeviceManager api.DeviceMode //Directory where to persist the node driver state @@ -126,10 +118,6 @@ type Config struct { // allowed to send above the average rate of request. KubeAPIBurst int - // parameters for Kubernetes scheduler extender - schedulerListen string - insecureSchedulerListen string - // parameters for rescheduler and raw namespace conversion nodeSelector types.NodeSelector @@ -188,25 +176,15 @@ func (csid *csiDriver) Run(ctx context.Context) error { logger := klog.FromContext(ctx) switch csid.cfg.Mode { - case Webhooks: + case Controller: client, err := k8sutil.NewClient(config.KubeAPIQPS, config.KubeAPIBurst) if err != nil { return fmt.Errorf("connect to apiserver: %v", err) } - // A factory for all namespaces. Some of these are only needed by - // scheduler webhooks or deprovisioner, but because the normal - // setup is to have both enabled, the logic here is simplified so that - // everything gets initialized. - // - // The PV informer is not really needed, but there is no good way to - // tell the lib that it should watch PVs. An informer for a fake client - // did not work: - // Failed to watch *v1.PersistentVolume: unhandled watch: testing.WatchActionImpl + // A factory for all namespaces. globalFactory := informers.NewSharedInformerFactory(client, resyncPeriod) pvcInformer := globalFactory.Core().V1().PersistentVolumeClaims().Informer() - pvcLister := globalFactory.Core().V1().PersistentVolumeClaims().Lister() - scLister := globalFactory.Storage().V1().StorageClasses().Lister() scInformer := globalFactory.Storage().V1().StorageClasses().Informer() pvInformer := globalFactory.Core().V1().PersistentVolumes().Informer() csiNodeLister := globalFactory.Storage().V1().CSINodes().Lister() @@ -247,41 +225,6 @@ func (csid *csiDriver) Run(ctx context.Context) error { } } - if csid.cfg.schedulerListen != "" || csid.cfg.insecureSchedulerListen != "" { - // Factory for the driver's namespace. - namespace := os.Getenv("POD_NAMESPACE") - if namespace == "" { - return errors.New("POD_NAMESPACE env variable is not set") - } - localFactory := informers.NewSharedInformerFactoryWithOptions(client, resyncPeriod, - informers.WithNamespace(namespace), - ) - podLister := localFactory.Core().V1().Pods().Lister() - c := scheduler.CapacityViaMetrics(namespace, csid.cfg.DriverName, podLister) - localFactory.Start(ctx.Done()) - - sched, err := scheduler.NewScheduler( - csid.cfg.DriverName, - c, - client, - pvcLister, - scLister, - ) - if err != nil { - return fmt.Errorf("create scheduler: %v", err) - } - if csid.cfg.schedulerListen != "" { - if _, err := csid.startHTTPSServer(ctx, cancel, csid.cfg.schedulerListen, sched, true /* TLS */); err != nil { - return err - } - } - if csid.cfg.insecureSchedulerListen != "" { - if _, err := csid.startHTTPSServer(ctx, cancel, csid.cfg.insecureSchedulerListen, sched, false /* not TLS */); err != nil { - return err - } - } - } - if pcp != nil { pcp.startRescheduler(ctx, cancel) } @@ -389,27 +332,17 @@ func (csid *csiDriver) startMetrics(ctx context.Context, cancel func()) (string, ), ) mux.Handle(csid.cfg.metricsPath+"/simple", promhttp.HandlerFor(simpleMetrics, promhttp.HandlerOpts{})) - return csid.startHTTPSServer(ctx, cancel, csid.cfg.metricsListen, mux, false /* no TLS */) + return csid.startHTTPSServer(ctx, cancel, csid.cfg.metricsListen, mux) } // startHTTPSServer contains the common logic for starting and // stopping an HTTPS server. Returns an error or the address that can // be used in Dial("tcp") to reach the server (useful for testing when // "listen" does not include a port). -func (csid *csiDriver) startHTTPSServer(ctx context.Context, cancel func(), listen string, handler http.Handler, useTLS bool) (string, error) { +func (csid *csiDriver) startHTTPSServer(ctx context.Context, cancel func(), listen string, handler http.Handler) (string, error) { name := "HTTP server" - if useTLS { - name = "HTTPS server" - } logger := klog.FromContext(ctx).WithName(name).WithValues("listen", listen) var config *tls.Config - if useTLS { - c, err := pmemgrpc.LoadServerTLS(ctx, csid.cfg.CAFile, csid.cfg.CertFile, csid.cfg.KeyFile, "" /* any peer can connect */) - if err != nil { - return "", fmt.Errorf("initialize HTTPS config: %v", err) - } - config = c - } server := http.Server{ Addr: listen, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -426,13 +359,7 @@ func (csid *csiDriver) startHTTPSServer(ctx context.Context, cancel func(), list go func() { defer tcpListener.Close() - var err error - if useTLS { - err = server.ServeTLS(listener, csid.cfg.CertFile, csid.cfg.KeyFile) - } else { - err = server.Serve(listener) - } - if err != http.ErrServerClosed { + if err := server.Serve(listener); err != http.ErrServerClosed { logger.Error(err, "Failed") } // Also stop main thread. diff --git a/pkg/pmem-csi-driver/pmem-csi-driver_test.go b/pkg/pmem-csi-driver/pmem-csi-driver_test.go index 83d409f63..1bb6082bc 100644 --- a/pkg/pmem-csi-driver/pmem-csi-driver_test.go +++ b/pkg/pmem-csi-driver/pmem-csi-driver_test.go @@ -12,19 +12,12 @@ import ( "fmt" "io/ioutil" "net/http" - "os" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -var ( - caFile = os.ExpandEnv("${TEST_WORK}/pmem-ca/ca.pem") - certFile = os.ExpandEnv("${TEST_WORK}/pmem-ca/pmem-controller.pem") - keyFile = os.ExpandEnv("${TEST_WORK}/pmem-ca/pmem-controller-key.pem") -) - func TestMetrics(t *testing.T) { cases := map[string]struct { path string @@ -50,14 +43,11 @@ build_info{version="foo-bar-test"} 1 t.Run(n, func(t *testing.T) { path := "/metrics2" pmemd, err := GetCSIDriver(Config{ - Mode: Webhooks, + Mode: Controller, DriverName: "pmem-csi", NodeID: "testnode", Endpoint: "unused", Version: "foo-bar-test", - CAFile: caFile, - CertFile: certFile, - KeyFile: keyFile, metricsPath: path, metricsListen: "127.0.0.1:", // port allocated dynamically }) diff --git a/pkg/pmem-csi-operator/controller/deployment/controller_driver.go b/pkg/pmem-csi-operator/controller/deployment/controller_driver.go index da54cbcd2..6bec37ad8 100644 --- a/pkg/pmem-csi-operator/controller/deployment/controller_driver.go +++ b/pkg/pmem-csi-operator/controller/deployment/controller_driver.go @@ -38,8 +38,6 @@ const ( controllerMetricsPort = 10010 nodeMetricsPort = 10010 provisionerMetricsPort = 10011 - schedulerPort = 8000 - insecureSchedulerPort = 8001 ) func typeMeta(gv schema.GroupVersion, kind string) metav1.TypeMeta { @@ -153,8 +151,6 @@ type pmemCSIDeployment struct { // operator's namespace used for creating sub-resources namespace string k8sVersion version.Version - - controllerCABundle []byte } func (d *pmemCSIDeployment) withStorageCapacity() bool { @@ -353,10 +349,6 @@ func (d *pmemCSIDeployment) redeploy(ctx context.Context, r *ReconcileDeployment return o, nil } -func mutatingWebhookEnabled(d *pmemCSIDeployment) bool { - return d.Spec.ControllerTLSSecret != "" && d.Spec.MutatePods != api.MutatePodsNever -} - var subObjectHandlers = map[string]redeployObject{ "node driver": { objType: reflect.TypeOf(&appsv1.DaemonSet{}), @@ -496,47 +488,6 @@ var subObjectHandlers = map[string]redeployObject{ return nil }, }, - "webhooks service": { - objType: reflect.TypeOf(&corev1.Service{}), - enabled: mutatingWebhookEnabled, - object: func(d *pmemCSIDeployment) client.Object { - return &corev1.Service{ - TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, - ObjectMeta: d.getObjectMeta(d.WebhooksServiceName(), false), - } - }, - modify: func(d *pmemCSIDeployment, o client.Object) error { - d.getWebhooksService(o.(*corev1.Service)) - return nil - }, - }, - "mutating webhook configuration": { - objType: reflect.TypeOf(&admissionregistrationv1.MutatingWebhookConfiguration{}), - enabled: mutatingWebhookEnabled, - object: func(d *pmemCSIDeployment) client.Object { - return &admissionregistrationv1.MutatingWebhookConfiguration{ - TypeMeta: metav1.TypeMeta{Kind: "MutatingWebhookConfiguration", APIVersion: "admissionregistration.k8s.io/v1"}, - ObjectMeta: d.getObjectMeta(d.MutatingWebhookName(), true), - } - }, - modify: func(d *pmemCSIDeployment, o client.Object) error { - d.getMutatingWebhookConfig(o.(*admissionregistrationv1.MutatingWebhookConfiguration)) - return nil - }, - }, - "scheduler service": { - objType: reflect.TypeOf(&corev1.Service{}), - object: func(d *pmemCSIDeployment) client.Object { - return &corev1.Service{ - TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, - ObjectMeta: d.getObjectMeta(d.SchedulerServiceName(), false), - } - }, - modify: func(d *pmemCSIDeployment, o client.Object) error { - d.getSchedulerService(o.(*corev1.Service)) - return nil - }, - }, "provisioner role": { objType: reflect.TypeOf(&rbacv1.Role{}), object: func(d *pmemCSIDeployment) client.Object { @@ -881,126 +832,6 @@ func (d *pmemCSIDeployment) getWebhooksClusterRoleBinding(crb *rbacv1.ClusterRol } } -func (d *pmemCSIDeployment) getWebhooksService(service *corev1.Service) { - d.getService(service, corev1.ServiceTypeClusterIP, 443) - service.Spec.Ports[0].TargetPort.IntVal = schedulerPort - if d.Spec.ControllerTLSSecret == api.ControllerTLSSecretOpenshift { - if service.Annotations == nil { - service.Annotations = map[string]string{} - } - service.Annotations["service.beta.openshift.io/serving-cert-secret-name"] = d.ControllerTLSSecretOpenshiftName() - } else { - if service.Annotations != nil { - delete(service.Annotations, "service.beta.openshift.io/serving-cert-secret-name") - } - } -} - -func (d *pmemCSIDeployment) getMutatingWebhookConfig(hook *admissionregistrationv1.MutatingWebhookConfiguration) { - servicePort := int32(443) // default webhook service port - selector := &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "pmem-csi.intel.com/webhook", - Operator: metav1.LabelSelectorOpNotIn, - Values: []string{"ignore"}, - }, - }, - } - failurePolicy := admissionregistrationv1.Ignore - if d.Spec.MutatePods == api.MutatePodsAlways { - failurePolicy = admissionregistrationv1.Fail - } - path := "/pod/mutate" - none := admissionregistrationv1.SideEffectClassNone - controllerCABundle := d.controllerCABundle - // Preserve defaults when updating. - var scope *admissionregistrationv1.ScopeType - var timeoutSeconds *int32 - var matchPolicy *admissionregistrationv1.MatchPolicyType - var reinvocationPolicy *admissionregistrationv1.ReinvocationPolicyType - if len(hook.Webhooks) > 0 { - if d.Spec.ControllerTLSSecret == api.ControllerTLSSecretOpenshift { - // Below we overwrite the entire hook.Webhooks. Before we do that, we must - // retrieve the CABundle that was generated for us by OpenShift. - controllerCABundle = hook.Webhooks[0].ClientConfig.CABundle - } - scope = hook.Webhooks[0].Rules[0].Scope - timeoutSeconds = hook.Webhooks[0].TimeoutSeconds - matchPolicy = hook.Webhooks[0].MatchPolicy - reinvocationPolicy = hook.Webhooks[0].ReinvocationPolicy - } - hook.Webhooks = []admissionregistrationv1.MutatingWebhook{ - { - // Name must be "fully-qualified" (i.e. with domain) but not unique, so - // here "pmem-csi.intel.com" is not the default driver name. - // https://pkg.go.dev/k8s.io/api/admissionregistration/v1#MutatingWebhook - Name: "pod-hook.pmem-csi.intel.com", - NamespaceSelector: selector, - ObjectSelector: selector, - FailurePolicy: &failurePolicy, - MatchPolicy: matchPolicy, - ReinvocationPolicy: reinvocationPolicy, - ClientConfig: admissionregistrationv1.WebhookClientConfig{ - Service: &admissionregistrationv1.ServiceReference{ - Name: d.WebhooksServiceName(), - Namespace: d.namespace, - Path: &path, - Port: &servicePort, - }, - // CABundle set below. - }, - Rules: []admissionregistrationv1.RuleWithOperations{ - { - Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, - Rule: admissionregistrationv1.Rule{ - APIGroups: []string{""}, - APIVersions: []string{"v1"}, - Resources: []string{"pods"}, - Scope: scope, - }, - }, - }, - SideEffects: &none, - AdmissionReviewVersions: []string{"v1"}, - TimeoutSeconds: timeoutSeconds, - }, - } - - switch { - case d.Spec.ControllerTLSSecret == api.ControllerTLSSecretOpenshift: - if hook.Annotations == nil { - hook.Annotations = map[string]string{} - } - hook.Annotations["service.beta.openshift.io/inject-cabundle"] = "true" - case len(controllerCABundle) == 0: - panic("controller CA bundle empty, should have been loaded") - default: - if hook.Annotations != nil { - delete(hook.Annotations, "service.beta.openshift.io/inject-cabundle") - } - } - // Set or preserve the CABundle. - hook.Webhooks[0].ClientConfig.CABundle = controllerCABundle -} - -func (d *pmemCSIDeployment) getSchedulerService(service *corev1.Service) { - targetPort := schedulerPort - port := 443 - if d.Spec.ControllerTLSSecret == api.ControllerTLSSecretOpenshift { - targetPort = insecureSchedulerPort - port = 80 - } - d.getService(service, corev1.ServiceTypeClusterIP, int32(port)) - service.Spec.Ports[0].TargetPort = intstr.IntOrString{ - IntVal: int32(targetPort), - } - service.Spec.Ports[0].NodePort = d.Spec.SchedulerNodePort - if d.Spec.SchedulerNodePort != 0 { - service.Spec.Type = corev1.ServiceTypeNodePort - } -} - func (d *pmemCSIDeployment) getControllerProvisionerRole(role *rbacv1.Role) { role.Rules = []rbacv1.PolicyRule{ { @@ -1195,22 +1026,6 @@ func (d *pmemCSIDeployment) getControllerDeployment(ss *appsv1.Deployment) { // Allow this pod to run on all nodes. setTolerations(&ss.Spec.Template.Spec) ss.Spec.Template.Spec.Volumes = []corev1.Volume{} - if d.Spec.ControllerTLSSecret != "" { - mode := corev1.SecretVolumeSourceDefaultMode - name := d.Spec.ControllerTLSSecret - if name == api.ControllerTLSSecretOpenshift { - name = d.ControllerTLSSecretOpenshiftName() - } - ss.Spec.Template.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{ - Name: "webhook-cert", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: name, - DefaultMode: &mode, - }, - }, - }) - } } func (d *pmemCSIDeployment) getNodeDaemonSet(ds *appsv1.DaemonSet) { @@ -1353,19 +1168,6 @@ func (d *pmemCSIDeployment) getControllerCommand() []string { "-nodeSelector=" + nodeSelector.String(), } - if d.Spec.ControllerTLSSecret != "" { - args = append(args, - "-caFile=", - "-certFile=/certs/tls.crt", - "-keyFile=/certs/tls.key", - fmt.Sprintf("-schedulerListen=:%d", schedulerPort), - ) - if d.Spec.ControllerTLSSecret == api.ControllerTLSSecretOpenshift { - args = append(args, - fmt.Sprintf("-insecureSchedulerListen=:%d", insecureSchedulerPort), - ) - } - } args = append(args, fmt.Sprintf("-metricsListen=:%d", controllerMetricsPort)) return args @@ -1424,15 +1226,6 @@ func (d *pmemCSIDeployment) getControllerContainer() corev1.Container { LivenessProbe: getMetricsProbe(6, 10, "/simple"), StartupProbe: getMetricsProbe(60, 1, "/simple"), } - - if d.Spec.ControllerTLSSecret != "" { - c.VolumeMounts = append(c.VolumeMounts, - corev1.VolumeMount{ - Name: "webhook-cert", - MountPath: "/certs", - }) - } - return c } diff --git a/pkg/pmem-csi-operator/controller/deployment/deployment_controller.go b/pkg/pmem-csi-operator/controller/deployment/deployment_controller.go index 6e39e477c..2549fad82 100644 --- a/pkg/pmem-csi-operator/controller/deployment/deployment_controller.go +++ b/pkg/pmem-csi-operator/controller/deployment/deployment_controller.go @@ -431,8 +431,6 @@ func (r *ReconcileDeployment) getDeploymentFor(ctx context.Context, obj metav1.O // newDeployment prepares for object creation and will modify the PmemCSIDeployment. // Callers who don't want that need to clone it first. func (r *ReconcileDeployment) newDeployment(ctx context.Context, deployment *api.PmemCSIDeployment) (*pmemCSIDeployment, error) { - l := klog.FromContext(ctx).WithName("newDeployment") - if err := deployment.EnsureDefaults(r.containerImage); err != nil { return nil, err } @@ -443,37 +441,6 @@ func (r *ReconcileDeployment) newDeployment(ctx context.Context, deployment *api k8sVersion: r.k8sVersion, } - switch d.Spec.ControllerTLSSecret { - case "": - // Nothing to do. - case api.ControllerTLSSecretOpenshift: - // Nothing to load, we just add annotations. - default: - // Load the specified secret. - secret := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Secret", - }, - } - objKey := client.ObjectKey{ - Namespace: d.namespace, - Name: d.Spec.ControllerTLSSecret, - } - if err := r.client.Get(ctx, objKey, secret); err != nil { - return nil, fmt.Errorf("loading ControllerTLSSecret %s from namespace %s: %v", d.Spec.ControllerTLSSecret, d.namespace, err) - } - ca, ok := secret.Data[api.TLSSecretCA] - if !ok { - return nil, fmt.Errorf("ControllerTLSSecret %s in namespace %s contains no %s", d.Spec.ControllerTLSSecret, d.namespace, api.TLSSecretCA) - } - d.controllerCABundle = ca - if len(d.controllerCABundle) == 0 { - return nil, fmt.Errorf("ControllerTLSSecret %s in namespace %s contains empty %s", d.Spec.ControllerTLSSecret, d.namespace, api.TLSSecretCA) - } - l.V(3).Info("load controller TLS secret", "secret", objKey, "CAlength", len(d.controllerCABundle)) - } - return d, nil } diff --git a/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go b/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go index 4b705f704..6f5b48b4e 100644 --- a/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go +++ b/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go @@ -56,9 +56,6 @@ type pmemDeployment struct { nodeCPU, nodeMemory string provisionerCPU, provisionerMemory string nodeRegistarCPU, nodeRegistrarMemory string - controllerTLSSecret string - mutatePods api.MutatePods - schedulerNodePort int32 kubeletDir string objects []runtime.Object @@ -83,16 +80,13 @@ func getDeployment(d *pmemDeployment) *api.PmemCSIDeployment { // The entire copying block below then collapses into a single line. dep.Spec = api.DeploymentSpec{ - DeviceMode: api.DeviceMode(d.deviceMode), - LogLevel: d.logLevel, - LogFormat: api.LogFormat(d.logFormat), - Image: d.image, - PullPolicy: corev1.PullPolicy(d.pullPolicy), - ProvisionerImage: d.provisionerImage, - NodeRegistrarImage: d.registrarImage, - ControllerTLSSecret: d.controllerTLSSecret, - MutatePods: d.mutatePods, - SchedulerNodePort: d.schedulerNodePort, + DeviceMode: api.DeviceMode(d.deviceMode), + LogLevel: d.logLevel, + LogFormat: api.LogFormat(d.logFormat), + Image: d.image, + PullPolicy: corev1.PullPolicy(d.pullPolicy), + ProvisionerImage: d.provisionerImage, + NodeRegistrarImage: d.registrarImage, } spec := &dep.Spec spec.ControllerReplicas = d.controllerReplicas @@ -320,12 +314,6 @@ func TestDeploymentController(t *testing.T) { t.Parallel() - dataOkay := map[string][]byte{ - api.TLSSecretCA: []byte("ca"), - api.TLSSecretKey: []byte("key"), - api.TLSSecretCert: []byte("cert"), - } - cases := map[string]pmemDeployment{ "deployment with defaults": { name: "test-deployment", @@ -358,50 +346,6 @@ func TestDeploymentController(t *testing.T) { name: "test-driver-modes", deviceMode: "direct", }, - "with controller, no secret": { - name: "test-controller", - controllerTLSSecret: "controller-secret", - expectFailure: true, - }, - "with controller, wrong secret content": { - name: "test-controller", - controllerTLSSecret: "controller-secret", - objects: []runtime.Object{createSecret("controller-secret", testNamespace, nil)}, - expectFailure: true, - }, - "with controller, secret okay": { - name: "test-controller", - controllerTLSSecret: "controller-secret", - objects: []runtime.Object{createSecret("controller-secret", testNamespace, dataOkay)}, - }, - "controller, no mutate": { - name: "test-controller", - controllerTLSSecret: "controller-secret", - mutatePods: api.MutatePodsNever, - objects: []runtime.Object{createSecret("controller-secret", testNamespace, dataOkay)}, - }, - "controller, try mutate": { - name: "test-controller", - controllerTLSSecret: "controller-secret", - mutatePods: api.MutatePodsTry, - objects: []runtime.Object{createSecret("controller-secret", testNamespace, dataOkay)}, - }, - "controller, always mutate": { - name: "test-controller", - controllerTLSSecret: "controller-secret", - mutatePods: api.MutatePodsAlways, - objects: []runtime.Object{createSecret("controller-secret", testNamespace, dataOkay)}, - }, - "controller, port 31000": { - name: "test-controller", - controllerTLSSecret: "controller-secret", - schedulerNodePort: 31000, - objects: []runtime.Object{createSecret("controller-secret", testNamespace, dataOkay)}, - }, - "openshift": { - name: "test-controller", - controllerTLSSecret: "-openshift-", - }, } for name, d := range cases { diff --git a/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go b/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go index 9897a9d19..ef05e2918 100644 --- a/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go +++ b/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go @@ -102,9 +102,6 @@ func UpdateTests() []UpdateTest { "kubeletDir": func(d *api.PmemCSIDeployment) { d.Spec.KubeletDir = "/foo/bar" }, - "openshift": func(d *api.PmemCSIDeployment) { - d.Spec.ControllerTLSSecret = "-openshift-" - }, } full := api.PmemCSIDeployment{ @@ -201,22 +198,5 @@ func UpdateTests() []UpdateTest { }) } - // Special case: remove -openshift- - openshiftDep := api.PmemCSIDeployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pmem-csi-for-openshift", - }, - Spec: api.DeploymentSpec{ - ControllerTLSSecret: "-openshift-", - }, - } - tests = append(tests, UpdateTest{ - Name: "remove-openshift", - Deployment: openshiftDep, - Mutate: func(d *api.PmemCSIDeployment) { - d.Spec.ControllerTLSSecret = "" - }, - }) - return tests } diff --git a/pkg/scheduler/README.md b/pkg/scheduler/README.md deleted file mode 100644 index 374c759be..000000000 --- a/pkg/scheduler/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Scheduler - -This directory contains the implementation of a [scheduler -extender](https://github.com/kubernetes/design-proposals-archive/blob/main/scheduling/scheduler_extender.md). -It is meant to be linked into the PMEM-CSI master controller because -that is where we know about nodes and can get their current capacity. - - diff --git a/pkg/scheduler/capacity.go b/pkg/scheduler/capacity.go deleted file mode 100644 index fe4efe38b..000000000 --- a/pkg/scheduler/capacity.go +++ /dev/null @@ -1,138 +0,0 @@ -/* -Copyright 2020 Intel Corp. - -SPDX-License-Identifier: Apache-2.0 -*/ - -package scheduler - -import ( - "errors" - "fmt" - "io" - "net/http" - - dto "github.com/prometheus/client_model/go" - "github.com/prometheus/common/expfmt" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/labels" - corelistersv1 "k8s.io/client-go/listers/core/v1" -) - -type capacityFromMetrics struct { - namespace string - driverName string - podLister corelistersv1.PodLister - client http.Client -} - -func CapacityViaMetrics(namespace, driverName string, podLister corelistersv1.PodLister) Capacity { - return &capacityFromMetrics{ - namespace: namespace, - driverName: driverName, - podLister: podLister, - } -} - -// All PMEM-CSI deployments must use these labels for the node driver pods. -func (c *capacityFromMetrics) pmemCSINodeSelector() labels.Selector { - return labels.Set{ - "app.kubernetes.io/part-of": "pmem-csi", - "app.kubernetes.io/component": "node", - "app.kubernetes.io/instance": c.driverName, - }.AsSelector() -} - -// NodeCapacity implements the necessary method for the NodeCapacity interface by -// looking up pods in the namespace which run on the node (usually one) -// and retrieving metrics data from them. The driver name is checked to allow -// more than one driver instance per node (unlikely). -func (c *capacityFromMetrics) NodeCapacity(nodeName string) (int64, error) { - pods, err := c.podLister.List(c.pmemCSINodeSelector()) - if err != nil { - return 0, fmt.Errorf("list PMEM-CSI node pods: %v", err) - } - for _, pod := range pods { - if pod.Spec.NodeName != nodeName || - pod.Namespace != c.namespace { - continue - } - url := metricsURL(pod) - if url == "" { - continue - } - capacity, err := c.retrieveMaxVolumeSize(url) - switch err { - case wrongPod: - continue - case nil: - return capacity, nil - default: - return 0, fmt.Errorf("get metrics from pod %s via %s: %v", pod.Name, url, err) - } - } - - // Node not known or no metrics. - return 0, nil -} - -var wrongPod = errors.New("wrong driver pod") - -func (c *capacityFromMetrics) retrieveMaxVolumeSize(url string) (int64, error) { - // TODO (?): negotiate encoding (https://pkg.go.dev/github.com/prometheus/common/expfmt#Negotiate) - resp, err := c.client.Get(url) - if err != nil { - return 0, err - } - if resp.StatusCode != http.StatusOK { - return 0, fmt.Errorf("bad HTTP response status: %s", resp.Status) - } - decoder := expfmt.NewDecoder(resp.Body, expfmt.ResponseFormat(resp.Header)) - if err != nil { - return 0, fmt.Errorf("read response: %v", err) - } - var metrics dto.MetricFamily - - for { - err := decoder.Decode(&metrics) - if err != nil { - if errors.Is(err, io.EOF) { - // If we get here without finding what we look for, we must be talking - // to the wrong pod and should keep looking. - return 0, wrongPod - } - return 0, fmt.Errorf("decode response: %v", err) - } - if metrics.GetName() == "pmem_amount_max_volume_size" { - for _, metric := range metrics.GetMetric() { - for _, label := range metric.GetLabel() { - if label.GetName() == "driver_name" && - label.GetValue() != c.driverName { - return 0, wrongPod - } - } - // "driver_name" was not present yet in PMEM-CSI 0.8.0, so - // we cannot fail when it is missing. - gauge := metric.GetGauge() - if gauge == nil { - return 0, fmt.Errorf("unexpected metric type for pmem_amount_max_volume_size: %s", metrics.GetType()) - } - return int64(gauge.GetValue()), nil - } - } - } -} - -func metricsURL(pod *corev1.Pod) string { - for _, container := range pod.Spec.Containers { - if container.Name == "pmem-driver" { - for _, containerPort := range container.Ports { - if containerPort.Name == "metrics" { - return fmt.Sprintf("http://%s:%d/metrics", pod.Status.PodIP, containerPort.ContainerPort) - } - } - return "" - } - } - return "" -} diff --git a/pkg/scheduler/capacity_test.go b/pkg/scheduler/capacity_test.go deleted file mode 100644 index b8ada5179..000000000 --- a/pkg/scheduler/capacity_test.go +++ /dev/null @@ -1,209 +0,0 @@ -/* -Copyright 2020 Intel Corp. - -SPDX-License-Identifier: Apache-2.0 -*/ - -package scheduler - -import ( - "net" - "net/http" - "strconv" - "testing" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - corelistersv1 "k8s.io/client-go/listers/core/v1" - "k8s.io/client-go/tools/cache" - - pmdmanager "github.com/intel/pmem-csi/pkg/pmem-device-manager" -) - -type node struct { - name, namespace string - capacity pmdmanager.Capacity - driverName string - noMetrics bool -} - -func (n node) createPMEMPod(port int) *corev1.Pod { - pod := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pmem-csi-node-" + n.name, - Namespace: n.namespace, - Labels: map[string]string{ - "app.kubernetes.io/part-of": "pmem-csi", - "app.kubernetes.io/component": "node", - "app.kubernetes.io/instance": n.driverName, - }, - }, - Spec: corev1.PodSpec{ - NodeName: n.name, - Containers: []corev1.Container{ - { - Name: "pmem-driver", - }, - }, - }, - Status: corev1.PodStatus{ - PodIP: "127.0.0.1", // address we listen on - }, - } - if port != 0 { - pod.Spec.Containers[0].Ports = []corev1.ContainerPort{ - { - Name: "metrics", - ContainerPort: int32(port), - }, - } - } - return &pod -} - -func TestCapacityFromMetrics(t *testing.T) { - cap := pmdmanager.Capacity{ - MaxVolumeSize: 1000, - Available: 2000, - Managed: 3000, - Total: 4000, - } - capSmall := pmdmanager.Capacity{ - MaxVolumeSize: 1, - Available: 2, - Managed: 3, - Total: 4, - } - testcases := map[string]struct { - nodes []node - node string - namespace string - driverName string - expected int64 - expectError bool - }{ - "one node": { - nodes: []node{ - { - name: "foobar", - capacity: cap, - }, - }, - node: "foobar", - expected: 1000, - }, - "no such node": { - nodes: []node{ - { - name: "foo", - capacity: pmdmanager.Capacity{MaxVolumeSize: 1000}, - }, - }, - node: "bar", - }, - "no driver": { - node: "foobar", - }, - "wrong driver": { - nodes: []node{ - { - name: "foobar", - driverName: "AAA", - capacity: cap, - }, - }, - node: "foobar", - driverName: "BBB", - }, - "wrong namespace": { - nodes: []node{ - { - name: "foobar", - namespace: "default", - capacity: cap, - }, - }, - node: "foobar", - namespace: "pmem-csi", - }, - "multiple drivers": { - nodes: []node{ - { - name: "foobar", - driverName: "AAA", - capacity: capSmall, - }, - { - name: "foobar", - driverName: "BBB", - capacity: cap, - }, - }, - node: "foobar", - driverName: "BBB", - expected: 1000, - }, - "metrics handler missing": { - nodes: []node{ - { - name: "foobar", - capacity: cap, - noMetrics: true, - }, - }, - node: "foobar", - expectError: true, - }, - } - - for name, tc := range testcases { - t.Run(name, func(t *testing.T) { - podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) - - // We need one metrics server per node. - for _, node := range tc.nodes { - mux := http.NewServeMux() - registry := prometheus.NewPedanticRegistry() - collector := pmdmanager.CapacityCollector{PmemDeviceCapacity: node.capacity} - collector.MustRegister(registry, node.name, node.driverName) - if !node.noMetrics { - mux.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{})) - } - listen := "127.0.0.1:" - listener, err := net.Listen("tcp", listen) - require.NoError(t, err, "listen") - tcpListener := listener.(*net.TCPListener) - server := http.Server{ - Handler: mux, - } - go func() { - _ = server.Serve(listener) - }() - defer server.Close() - - // Now fake a PMEM-CSI pod on that node. - _, portStr, err := net.SplitHostPort(tcpListener.Addr().String()) - require.NoError(t, err, "split listen address") - port, err := strconv.Atoi(portStr) - require.NoError(t, err, "parse listen port") - pod := node.createPMEMPod(port) - if pod != nil { - _ = podIndexer.Add(pod) - } - } - podLister := corelistersv1.NewPodLister(podIndexer) - c := CapacityViaMetrics(tc.namespace, tc.driverName, podLister) - actual, err := c.NodeCapacity(tc.node) - if tc.expectError { - t.Logf("got error %v", err) - require.Error(t, err, "NodeCapacity should have failed") - } else { - require.NoError(t, err, "NodeCapacity should have succeeded") - require.Equal(t, tc.expected, actual, "capacity") - } - }) - } -} diff --git a/pkg/scheduler/mutate_pod.go b/pkg/scheduler/mutate_pod.go deleted file mode 100644 index b1b2b1546..000000000 --- a/pkg/scheduler/mutate_pod.go +++ /dev/null @@ -1,180 +0,0 @@ -/* -Copyright 2019 Cybozu -Copyright 2020 Intel Corp. - -SPDX-License-Identifier: Apache-2.0 - -Based on https://github.com/cybozu-go/topolvm/blob/7b79ee30e997a165b220d4519c784e50eaec36c8/hook/mutate_pod.go -and information from https://banzaicloud.com/blog/k8s-admission-webhooks/ -*/ - -package scheduler - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - - corev1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" - apierrs "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/klog/v2" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - - pmemlog "github.com/intel/pmem-csi/pkg/logger" -) - -const ( - // resourceSuffix is the part which gets added to the CSI driver name to - // create the extended resource name that will trigger the scheduler extender. - resourceSuffix = "/scheduler" -) - -// Handle implements admission.Handler interface. -func (s scheduler) Handle(ctx context.Context, req admission.Request) admission.Response { - logger := klog.FromContext(ctx).WithName("mutating-pod-webhook") - pod := &corev1.Pod{} - err := s.decoder.Decode(req, pod) - if err != nil { - return admission.Errored(http.StatusBadRequest, err) - } - logger = logger.WithValues("pod", pmemlog.KObj(pod)) - ctx = klog.NewContext(ctx, logger) - if len(pod.Spec.Containers) == 0 { - return admission.Denied("pod has no containers") - } - - // short cut - if len(pod.Spec.Volumes) == 0 { - return admission.Allowed("no volumes") - } - - // Pods instantiated from templates may have empty name/namespace. - // To lookup PVC in the same namespace, we set namespace obtained from req. - if pod.Namespace == "" { - pod.Namespace = req.Namespace - } - - targets, err := s.targetStorageClasses(ctx) - if err != nil { - return admission.Errored(http.StatusInternalServerError, err) - } - logger.V(5).Info("Checking storage classes", "storage-classes", targets) - - filter, err := s.mustFilterPod(ctx, pod, targets) - if err != nil { - return admission.Errored(http.StatusInternalServerError, err) - } - if !filter { - logger.V(5).Info("Pod does not use inline or delayed PMEM volumes.") - return admission.Allowed("no relevant PMEM volumes") - } - - ctnr := &pod.Spec.Containers[0] - quantity := resource.NewQuantity(1, resource.DecimalSI) - resource := corev1.ResourceName(s.driverName + resourceSuffix) - if ctnr.Resources.Requests == nil { - ctnr.Resources.Requests = corev1.ResourceList{} - } - ctnr.Resources.Requests[resource] = *quantity - if ctnr.Resources.Limits == nil { - ctnr.Resources.Limits = corev1.ResourceList{} - } - ctnr.Resources.Limits[resource] = *quantity - - marshaledPod, err := json.Marshal(pod) - if err != nil { - return admission.Errored(http.StatusInternalServerError, err) - } - - logger.V(5).Info("Pod uses PMEM.") - return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod) -} - -func (s scheduler) targetStorageClasses(ctx context.Context) (map[string]bool, error) { - scs, err := s.scLister.List(labels.Everything()) - if err != nil { - return nil, err - } - - targets := make(map[string]bool) - for _, sc := range scs { - targets[sc.Name] = s.mustFilterSC(sc) - } - return targets, nil -} - -// mustFilter returns true iff we must mark a pod using a PVC with this storage class -// as one which needs the scheduler extender, i.e. when the storage class uses PMEM-CSI -// and isn't using immediate binding. -func (s scheduler) mustFilterSC(sc *storagev1.StorageClass) bool { - return sc.Provisioner == s.driverName && - (sc.VolumeBindingMode == nil || - *sc.VolumeBindingMode != storagev1.VolumeBindingImmediate) -} - -func (s scheduler) mustFilterPod(ctx context.Context, pod *corev1.Pod, targets map[string]bool) (bool, error) { - logger := klog.FromContext(ctx) - for _, vol := range pod.Spec.Volumes { - if vol.PersistentVolumeClaim != nil { - pvcName := vol.PersistentVolumeClaim.ClaimName - pvc, err := s.pvcLister.PersistentVolumeClaims(pod.Namespace).Get(pvcName) - if err != nil && apierrs.IsNotFound(err) { - // Bypass the lister, it might not have a recently created PVC yet. - pvc, err = s.clientSet.CoreV1().PersistentVolumeClaims(pod.Namespace).Get(ctx, pvcName, metav1.GetOptions{}) - } - - if err != nil { - if apierrs.IsNotFound(err) { - // A pod is getting created before its PVC. TopoLVM returns an error in this case (https://github.com/cybozu-go/topolvm/blob/7b79ee30e997a165b220d4519c784e50eaec36c8/hook/mutate_pod.go#L129-L136), - // but then pod creation fails while normally it would go through. - // We ignore the PVC instead. - logger.Info("Pod with unknown pvc, ignoring the pvc", "pvc", klog.KRef(pod.Namespace, pvcName)) - continue - } - return false, fmt.Errorf("check PVC %s/%s: %v", pod.Namespace, pvcName, err) - } - - if pvc.Spec.StorageClassName == nil { - // empty class name may appear when DefaultStorageClass admission plugin - // is turned off, or there are no default StorageClass. - // https://kubernetes.io/docs/concepts/storage/persistent-volumes/#class-1 - continue - } - - scName := *pvc.Spec.StorageClassName - filter, known := targets[scName] - if !known { - // Query API server directly. - sc, err := s.clientSet.StorageV1().StorageClasses().Get(ctx, scName, metav1.GetOptions{}) - if err != nil { - if apierrs.IsNotFound(err) { - // As with non-existent PVC, assume that it isn't using PMEM. - continue - } - return false, fmt.Errorf("check storage class %s: %v", scName, err) - } - filter = s.mustFilterSC(sc) - - // Also cache result. - targets[scName] = filter - } - if !filter { - continue - } - - // We don't care here whether the volume is already bound. - // That can be checked more reliably by the scheduler extender. - return true, nil - } else if vol.CSI != nil { - if vol.CSI.Driver == s.driverName { - return true, nil - } - } - } - return false, nil -} diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go deleted file mode 100644 index 1431de3df..000000000 --- a/pkg/scheduler/scheduler.go +++ /dev/null @@ -1,366 +0,0 @@ -/* -Copyright 2020 Intel Corp. - -SPDX-License-Identifier: Apache-2.0 -*/ - -package scheduler - -import ( - "encoding/json" - "errors" - "fmt" - "net/http" - "sort" - "sync" - - "github.com/go-logr/logr" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" - v1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" - "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/kubernetes" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - corelisters "k8s.io/client-go/listers/core/v1" - storagelisters "k8s.io/client-go/listers/storage/v1" - "k8s.io/klog/v2/klogr" - schedulerapi "k8s.io/kube-scheduler/extender/v1" - "sigs.k8s.io/controller-runtime/pkg/webhook" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - - "github.com/intel/pmem-csi/pkg/pmem-csi-driver/parameters" -) - -// Capacity provides information of remaining free PMEM per node. -type Capacity interface { - // NodeCapacity returns the available PMEM for the node. - NodeCapacity(nodeName string) (int64, error) -} - -var ( - inFlightGauge = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "scheduler_in_flight_requests", - Help: "A gauge of HTTP requests currently being served by the PMEM-CSI scheduler.", - }) - - counter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "scheduler_requests_total", - Help: "A counter for HTTP requests to the PMEM-CSI scheduler.", - }, - []string{"code", "method"}, - ) - - // duration is partitioned by the HTTP method and handler. - duration = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "scheduler_request_duration_seconds", - Help: "A histogram of latencies for PMEM-CSI scheduler HTTP requests.", - }, - []string{"handler", "method"}, - ) - - // responseSize has no labels, making it a zero-dimensional - // ObserverVec. - responseSize = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "scheduler_response_size_bytes", - Help: "A histogram of response sizes for PMEM-CSI scheduler requests.", - Buckets: []float64{200, 500, 900, 1500}, - }, - []string{}, - ) -) - -func init() { - prometheus.MustRegister(inFlightGauge, counter, duration, responseSize) -} - -func wrapHTTPHandler(handlerName string, handler http.HandlerFunc) http.HandlerFunc { - return promhttp.InstrumentHandlerDuration( - duration.MustCurryWith(prometheus.Labels{"handler": handlerName}), - promhttp.InstrumentHandlerCounter(counter, - promhttp.InstrumentHandlerResponseSize(responseSize, handler), - ), - ) -} - -type scheduler struct { - driverName string - capacity Capacity - clientSet kubernetes.Interface - pvcLister corelisters.PersistentVolumeClaimLister - scLister storagelisters.StorageClassLister - decoder *admission.Decoder - log logr.Logger - - instrumentedFilter, instrumentedStatus, instrumentedMutate http.HandlerFunc -} - -func NewScheduler( - driverName string, - capacity Capacity, - clientSet kubernetes.Interface, - pvcLister corelisters.PersistentVolumeClaimLister, - scLister storagelisters.StorageClassLister, -) (http.Handler, error) { - s := &scheduler{ - driverName: driverName, - capacity: capacity, - clientSet: clientSet, - pvcLister: pvcLister, - scLister: scLister, - log: klogr.New().WithName("scheduler"), - } - scheme := runtime.NewScheme() - if err := clientgoscheme.AddToScheme(scheme); err != nil { - return nil, fmt.Errorf("initialize client-go scheme: %v", err) - } - decoder, err := admission.NewDecoder(scheme) - if err != nil { - return nil, fmt.Errorf("initialize admission decoder: %v", err) - } - s.decoder = decoder - webhook := webhook.Admission{Handler: s} - if err := webhook.InjectLogger(s.log.WithName("webhook")); err != nil { - return nil, fmt.Errorf("inject logger: %v", err) - } - - s.instrumentedFilter = wrapHTTPHandler("filter", s.filter) - s.instrumentedStatus = wrapHTTPHandler("status", s.status) - s.instrumentedMutate = wrapHTTPHandler("mutate", webhook.ServeHTTP) - - return s, nil -} - -func (s *scheduler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - switch r.Method { - case http.MethodGet, http.MethodPost: - // Both are treated the same below. - default: - http.Error(w, r.Method+" not supported", http.StatusMethodNotAllowed) - return - } - - switch r.URL.Path { - case "/filter": - s.instrumentedFilter(w, r) - // TODO (?): prioritize nodes similar to https://github.com/cybozu-go/topolvm/blob/master/scheduler/prioritize.go - // case "/prioritize": - // s.prioritize(w, r) - case "/status": - s.instrumentedStatus(w, r) - case "/pod/mutate": - s.instrumentedMutate(w, r) - default: - http.Error(w, "not found", http.StatusNotFound) - } -} - -func (s *scheduler) status(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("ok")) -} - -// filter handles the JSON decoding+encoding. -func (s *scheduler) filter(w http.ResponseWriter, r *http.Request) { - // From https://github.com/Huang-Wei/sample-scheduler-extender/blob/047fdd5ae8b1a6d7fdc0e6d20ce4d70a1d6e7178/routers.go#L19-L39 - var args schedulerapi.ExtenderArgs - var result *schedulerapi.ExtenderFilterResult - err := json.NewDecoder(r.Body).Decode(&args) - if err == nil { - result, err = s.doFilter(args) - } - - // Always try to write a resonable response. - if result == nil && err != nil { - result = &schedulerapi.ExtenderFilterResult{ - Error: err.Error(), - } - } - if response, err := json.Marshal(result); err != nil { - s.log.Error(err, "JSON encoding") - w.WriteHeader(http.StatusInternalServerError) - } else { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - _, _ = w.Write(response) - } -} - -// doFilter determines how much PMEM storage the pod wants and filters out nodes with -// insufficient storage. A better solution would be a reservation system, but that's -// complicated to implement and should better be handled generically for volumes -// in Kubernetes. -func (s *scheduler) doFilter(args schedulerapi.ExtenderArgs) (*schedulerapi.ExtenderFilterResult, error) { - var filteredNodes []string - failedNodes := make(schedulerapi.FailedNodesMap) - if args.Pod == nil || - args.Pod.Name == "" || - (args.NodeNames == nil && args.Nodes == nil) { - return nil, errors.New("incomplete parameters") - } - - log := s.log.WithValues("pod", args.Pod.Name) - log.V(5).Info("node filter", "request", args) - required, err := s.requiredStorage(args.Pod) - if err != nil { - return nil, fmt.Errorf("checking for unbound volumes: %v", err) - } - log.V(5).Info("needs PMEM", "bytes", required) - - var mutex sync.Mutex - var waitgroup sync.WaitGroup - var nodeNames []string - if args.NodeNames != nil { - nodeNames = *args.NodeNames - } else { - // Fallback for Extender.NodeCacheCapable == false: - // not recommended, but may still be used by users who followed the - // PMEM-CSI 0.7 setup instructions. - log.Info("NodeCacheCapable is false in Extender configuration, should be set to true.") - nodeNames = listNodeNames(args.Nodes.Items) - } - for _, nodeName := range nodeNames { - if required == 0 { - // Nothing to check. - filteredNodes = append(filteredNodes, nodeName) - continue - } - - // Check in parallel. - nodeName := nodeName - waitgroup.Add(1) - go func() { - err := s.nodeHasEnoughCapacity(required, nodeName) - - mutex.Lock() - defer mutex.Unlock() - defer waitgroup.Done() - switch err { - case nil: - filteredNodes = append(filteredNodes, nodeName) - default: - failedNodes[nodeName] = err.Error() - } - }() - } - waitgroup.Wait() - - response := &schedulerapi.ExtenderFilterResult{ - FailedNodes: failedNodes, - Error: "", - } - if args.NodeNames != nil { - response.NodeNames = &filteredNodes - } else { - // fallback response... - response.Nodes = &v1.NodeList{} - for _, node := range filteredNodes { - response.Nodes.Items = append(response.Nodes.Items, getNode(args.Nodes.Items, node)) - } - } - log.V(5).Info("node filter", "response", response) - return response, nil -} - -// requiredStorage sums up total size of all currently unbound -// persistent volumes and all inline ephemeral volumes. This is a -// rough estimate whether the pod may still fit onto a node. -func (s *scheduler) requiredStorage(pod *v1.Pod) (int64, error) { - var total int64 - - for _, volume := range pod.Spec.Volumes { - if volume.PersistentVolumeClaim != nil { - claimName := volume.PersistentVolumeClaim.ClaimName - pvc, err := s.pvcLister.PersistentVolumeClaims(pod.Namespace).Get(claimName) - if err != nil { - return 0, fmt.Errorf("look up claim: %v", err) - } - - if pvc.Status.Phase == v1.ClaimBound || - pvc.Spec.VolumeName != "" { - // No need to check, the volume already exists. - continue - } - - scName := pvc.Spec.StorageClassName - if scName == nil { - // Shouldn't happen. - continue - } - sc, err := s.scLister.Get(*scName) - if err != nil { - return 0, fmt.Errorf("look up storage class: %v", err) - } - if sc.Provisioner != s.driverName { - // Not us. - continue - } - if sc.VolumeBindingMode != nil && - *sc.VolumeBindingMode == storagev1.VolumeBindingImmediate { - // Picking nodes for normal volumes will be handled by the master controller. - continue - } - - storage := pvc.Spec.Resources.Requests[v1.ResourceStorage] - size := storage.Value() - if size == 0 { - // We don't know exactly how the driver is going to round up. - // Let's use a conservative guess here - 1GiB. - size = 1024 * 1024 * 1024 - } - total += size - } - if volume.CSI != nil { - if volume.CSI.Driver != s.driverName { - // Not us. - continue - } - p, err := parameters.Parse(parameters.EphemeralVolumeOrigin, volume.CSI.VolumeAttributes) - if err != nil { - return 0, fmt.Errorf("ephemeral inline volume %s: %v", volume.Name, err) - } - total += p.GetSize() - } - } - return total, nil -} - -// nodeHasEnoughCapacity determines whether a node has enough storage available. It returns -// an error if not, otherwise nil. -func (s *scheduler) nodeHasEnoughCapacity(required int64, nodeName string) error { - available, err := s.capacity.NodeCapacity(nodeName) - if err != nil { - return fmt.Errorf("retrieve capacity: %v", err) - } - - if available < required { - return fmt.Errorf("only %vB of PMEM available, need %vB", - resource.NewQuantity(available, resource.BinarySI), - resource.NewQuantity(required, resource.BinarySI)) - } - - // Success! - return nil -} - -func listNodeNames(nodes []v1.Node) []string { - var names []string - for _, node := range nodes { - names = append(names, node.Name) - } - sort.Strings(names) - return names -} - -func getNode(nodes []v1.Node, nodeName string) v1.Node { - for _, node := range nodes { - if node.Name == nodeName { - return node - } - } - return v1.Node{} -} diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go deleted file mode 100644 index 3c9aab802..000000000 --- a/pkg/scheduler/scheduler_test.go +++ /dev/null @@ -1,763 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. -Copyright 2020 Intel Corp. - -SPDX-License-Identifier: Apache-2.0 - -Based on https://github.com/kubernetes/kubernetes/blob/5d5b444c4da95eb49618d380c49608e20a08c31c/pkg/controller/volume/scheduling/scheduler_binder_test.go -*/ - -package scheduler - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "net/url" - "sort" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - admissionv1 "k8s.io/api/admission/v1" - v1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/informers" - coreinformers "k8s.io/client-go/informers/core/v1" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/fake" - schedulerapi "k8s.io/kube-scheduler/extender/v1" - "k8s.io/kubernetes/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" -) - -var ( - // PVCs for manual binding - unboundPVC = makeTestPVC("unbound-pvc", "1Gi", "", pvcUnbound, "", "1", &waitClass) - unboundPVC2 = makeTestPVC("unbound-pvc2", "5Gi", "", pvcUnbound, "", "1", &waitClass) - unboundPVCUnknownClass = makeTestPVC("unbound-pvc", "1Gi", "", pvcUnbound, "", "1", &unknownClass) - unboundNoPVC = makeTestPVC("unbound-pvc", "1Gi", "", pvcUnbound, "", "1", &waitClassNoProvisioner) - unboundOtherPVC = makeTestPVC("unbound-pvc", "1Gi", "", pvcUnbound, "", "1", &waitClassOtherProvisioner) - immediateUnboundPVC = makeTestPVC("immediate-unbound-pvc", "1Gi", "", pvcUnbound, "", "1", &immediateClass) - boundPVC = makeTestPVC("bound-pvc", "1Gi", "", pvcBound, "pv-bound", "1", &waitClass) - - // storage class names - waitClass = "waitClass" - immediateClass = "immediateClass" - waitClassNoProvisioner = "waitClassNoProvisioner" - waitClassOtherProvisioner = "waitClassOtherProvisioner" - unknownClass = "unknownClass" -) - -const ( - GiG = 1024 * 1024 * 1024 - - // Provisioner aka driver names. - driverName = "test.pmem-csi.intel.com" - - // Our nodes. - nodeA = "node-A" - nodeB = "node-B" -) - -// Done indirectly by k8s.io/component-base/logs -// func init() { -// klog.InitFlags(nil) -// } - -type inlineVolume struct { - driverName string - size string -} - -// clusterCapacity is a stub implementation of the Capacity interface. -type clusterCapacity map[string]int64 - -func (cc clusterCapacity) NodeCapacity(nodeName string) (int64, error) { - available, ok := cc[nodeName] - if !ok { - return 0, fmt.Errorf("node %s unknown", nodeName) - } - return available, nil -} - -type testEnv struct { - client clientset.Interface - scheduler *scheduler - pvcInformer coreinformers.PersistentVolumeClaimInformer -} - -func newTestEnv(t *testing.T, capacity Capacity, stopCh <-chan struct{}) *testEnv { - client := &fake.Clientset{} - informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) - - pvcInformer := informerFactory.Core().V1().PersistentVolumeClaims() - classInformer := informerFactory.Storage().V1().StorageClasses() - - // Wait for informers cache sync - informerFactory.Start(stopCh) - for v, synced := range informerFactory.WaitForCacheSync(stopCh) { - if !synced { - t.Fatalf("Error syncing informer for %v", v) - } - } - - // Add storageclasses - waitMode := storagev1.VolumeBindingWaitForFirstConsumer - immediateMode := storagev1.VolumeBindingImmediate - classes := []*storagev1.StorageClass{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: waitClass, - }, - VolumeBindingMode: &waitMode, - Provisioner: driverName, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: immediateClass, - }, - VolumeBindingMode: &immediateMode, - Provisioner: driverName, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: waitClassNoProvisioner, - }, - VolumeBindingMode: &waitMode, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: waitClassOtherProvisioner, - }, - VolumeBindingMode: &waitMode, - Provisioner: driverName + ".example", - }, - } - for _, class := range classes { - if err := classInformer.Informer().GetIndexer().Add(class); err != nil { - t.Fatalf("Failed to add storage class to internal cache: %v", err) - } - } - - handler, err := NewScheduler(driverName, - capacity, - client, - pvcInformer.Lister(), - classInformer.Lister(), - ) - if err != nil { - t.Fatalf("Failed to create scheduler: %v", err) - } - - internalScheduler, ok := handler.(*scheduler) - if !ok { - t.Fatalf("Failed to convert %T to *scheduler", handler) - } - - return &testEnv{ - client: client, - scheduler: internalScheduler, - pvcInformer: pvcInformer, - } -} - -func (env *testEnv) initClaims(pvcs []*v1.PersistentVolumeClaim) { - for _, pvc := range pvcs { - _ = env.pvcInformer.Informer().GetIndexer().Add(pvc) - } -} - -const ( - pvcUnbound = iota - pvcPrebound - pvcBound -) - -func makeTestPVC(name, size, node string, pvcBoundState int, pvName, resourceVersion string, className *string) *v1.PersistentVolumeClaim { - fs := v1.PersistentVolumeFilesystem - pvc := &v1.PersistentVolumeClaim{ - TypeMeta: metav1.TypeMeta{ - Kind: "PersistentVolumeClaim", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: "testns", - UID: types.UID("pvc-uid"), - ResourceVersion: resourceVersion, - SelfLink: "/api/v1/namespaces/testns/persistentvolumeclaims/" + name, - }, - Spec: v1.PersistentVolumeClaimSpec{ - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceName(v1.ResourceStorage): resource.MustParse(size), - }, - }, - StorageClassName: className, - VolumeMode: &fs, - }, - } - - switch pvcBoundState { - case pvcBound: - pvc.Status.Phase = v1.ClaimBound - case pvcPrebound: - pvc.Spec.VolumeName = pvName - } - return pvc -} - -func makePod(pvcs []*v1.PersistentVolumeClaim, inline []inlineVolume) *v1.Pod { - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "testns", - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - v1.Container{}, - }, - }, - } - - volumes := []v1.Volume{} - i := 0 - for _, pvc := range pvcs { - pvcVol := v1.Volume{ - Name: fmt.Sprintf("vol%v", i), - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc.Name, - }, - }, - } - volumes = append(volumes, pvcVol) - i++ - } - for _, vol := range inline { - pvcVol := v1.Volume{ - Name: fmt.Sprintf("vol%v", i), - VolumeSource: v1.VolumeSource{ - CSI: &v1.CSIVolumeSource{ - Driver: vol.driverName, - VolumeAttributes: map[string]string{ - "size": vol.size, - }, - }, - }, - } - volumes = append(volumes, pvcVol) - i++ - } - pod.Spec.Volumes = volumes - pod.Spec.NodeName = "node1" - return pod -} - -func makeNodeList(nodeNames []string) *v1.NodeList { - nodes := v1.NodeList{} - for _, nodeName := range nodeNames { - nodes.Items = append(nodes.Items, v1.Node{ObjectMeta: metav1.ObjectMeta{Name: nodeName}}) - } - return &nodes -} - -type response struct { - statusCode int - body []byte -} - -func (r *response) Header() http.Header { - return http.Header{} -} - -func (r *response) WriteHeader(statusCode int) { - r.statusCode = statusCode -} - -func (r *response) Write(buffer []byte) (int, error) { - r.body = append(r.body, buffer...) - return len(buffer), nil -} - -func TestScheduler(t *testing.T) { - t.Parallel() - type scenarioType struct { - // Inputs - pvcs []*v1.PersistentVolumeClaim - inline []inlineVolume - // If nil, makePod with podPVCs - pod *v1.Pod - capacity clusterCapacity - // Nodes to check. - nodes []string - // Whether we pass v1.NodeList (false) or slice of node names (true). - nodeCacheCapable bool - - // Results - expectedError string - expectedNodes []string - expectedFailures schedulerapi.FailedNodesMap - } - scenarios := map[string]scenarioType{ - "no volumes, no nodes": {}, - "no volumes, one node, no capacity": { - nodes: []string{nodeA}, - expectedNodes: []string{nodeA}, - }, - "one volume, one node, no capacity": { - pvcs: []*v1.PersistentVolumeClaim{ - unboundPVC, - }, - nodes: []string{nodeA}, - expectedFailures: map[string]string{ - nodeA: "retrieve capacity: node node-A unknown", - }, - }, - "one volume, one node, enough capacity": { - pvcs: []*v1.PersistentVolumeClaim{ - unboundPVC, - }, - nodes: []string{nodeA}, - capacity: clusterCapacity{ - nodeA: GiG, - }, - expectedNodes: []string{nodeA}, - }, - "one inline volume, one node, insufficient capacity": { - inline: []inlineVolume{ - { - driverName: driverName, - size: resource.NewQuantity(GiG, resource.BinarySI).String(), - }, - }, - nodes: []string{nodeA}, - capacity: clusterCapacity{ - nodeA: 1, - }, - expectedFailures: map[string]string{ - nodeA: "only 1B of PMEM available, need 1GiB", - }, - }, - "one inline volume, one node, invalid size": { - inline: []inlineVolume{ - { - driverName: driverName, - size: "foobar", - }, - }, - nodes: []string{nodeA}, - capacity: clusterCapacity{ - nodeA: 1, - }, - expectedError: "checking for unbound volumes: ephemeral inline volume vol0: parameter \"size\": failed to parse \"foobar\" as int64: quantities must match the regular expression '^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$'", - }, - "one inline volume, one node, enough capacity": { - inline: []inlineVolume{ - { - driverName: driverName, - size: resource.NewQuantity(GiG, resource.DecimalSI).String(), - }, - }, - nodes: []string{nodeA}, - capacity: clusterCapacity{ - nodeA: GiG, - }, - expectedNodes: []string{nodeA}, - }, - "one other inline volume, one node, no capacity": { - inline: []inlineVolume{ - { - driverName: driverName + ".example", - size: resource.NewQuantity(GiG, resource.BinarySI).String(), - }, - }, - nodes: []string{nodeA}, - expectedNodes: []string{nodeA}, - }, - "one bound volume, one node, no capacity": { - pvcs: []*v1.PersistentVolumeClaim{ - boundPVC, - }, - nodes: []string{nodeA}, - expectedNodes: []string{nodeA}, - }, - "one pending volume, one node, no capacity": { - // This might never happen because Kubernetes probably first waits - // for the volume to be bound. - pvcs: []*v1.PersistentVolumeClaim{ - immediateUnboundPVC, - }, - nodes: []string{nodeA}, - expectedNodes: []string{nodeA}, - }, - "one volume with no provisioner, one node, no capacity": { - pvcs: []*v1.PersistentVolumeClaim{ - unboundNoPVC, - }, - nodes: []string{nodeA}, - expectedNodes: []string{nodeA}, - }, - "one volume with other provisioner, one node, no capacity": { - pvcs: []*v1.PersistentVolumeClaim{ - unboundOtherPVC, - }, - nodes: []string{nodeA}, - expectedNodes: []string{nodeA}, - }, - "two volumes, one node, enough capacity for one": { - pvcs: []*v1.PersistentVolumeClaim{ - unboundPVC, - unboundPVC2, - }, - nodes: []string{nodeA}, - capacity: clusterCapacity{ - nodeA: GiG, - }, - expectedFailures: map[string]string{ - nodeA: "only 1GiB of PMEM available, need 6GiB", - }, - }, - "one volume, two nodes, enough capacity": { - pvcs: []*v1.PersistentVolumeClaim{ - unboundPVC, - }, - nodes: []string{nodeA, nodeB}, - capacity: clusterCapacity{ - nodeA: GiG, - nodeB: GiG, - }, - expectedNodes: []string{nodeA, nodeB}, - }, - "nodeCacheCapable": { - pvcs: []*v1.PersistentVolumeClaim{ - unboundPVC, - }, - nodes: []string{nodeA, nodeB}, - capacity: clusterCapacity{ - nodeA: GiG, - nodeB: GiG, - }, - nodeCacheCapable: true, - expectedNodes: []string{nodeA, nodeB}, - }, - "one volume, two nodes, enough capacity on A": { - pvcs: []*v1.PersistentVolumeClaim{ - unboundPVC, - }, - nodes: []string{nodeA, nodeB}, - capacity: clusterCapacity{ - nodeA: GiG, - nodeB: 1, - }, - expectedNodes: []string{nodeA}, - expectedFailures: map[string]string{ - nodeB: "only 1B of PMEM available, need 1GiB", - }, - }, - "one volume, two nodes, insufficient capacity": { - pvcs: []*v1.PersistentVolumeClaim{ - unboundPVC, - }, - nodes: []string{nodeA, nodeB}, - capacity: clusterCapacity{ - nodeA: 100, - nodeB: 10, - }, - expectedFailures: map[string]string{ - nodeA: "only 100B of PMEM available, need 1GiB", - nodeB: "only 10B of PMEM available, need 1GiB", - }, - }, - "unknown pvc": { - pod: makePod([]*v1.PersistentVolumeClaim{unboundPVC}, nil), - expectedError: "checking for unbound volumes: look up claim: persistentvolumeclaim \"" + unboundPVC.Name + "\" not found", - }, - "unknown storage class": { - pvcs: []*v1.PersistentVolumeClaim{ - unboundPVCUnknownClass, - }, - expectedError: "checking for unbound volumes: look up storage class: storageclass.storage.k8s.io \"" + unknownClass + "\" not found", - }, - } - - run := func(t *testing.T, scenario scenarioType) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // Setup - testEnv := newTestEnv(t, scenario.capacity, ctx.Done()) - testEnv.initClaims(scenario.pvcs) - - // Generate input. We keep it very simple. Kubernetes actually sends - // the entire pod and node objects. - pod := scenario.pod - if pod == nil { - pod = makePod(scenario.pvcs, scenario.inline) - } - args := schedulerapi.ExtenderArgs{ - Pod: pod, - } - if scenario.nodeCacheCapable { - args.NodeNames = &scenario.nodes - } else { - args.Nodes = makeNodeList(scenario.nodes) - } - requestBody, err := json.Marshal(args) - require.NoError(t, err, "marshal request") - request := &http.Request{ - URL: &url.URL{Path: "/filter"}, - Body: ioutil.NopCloser(bytes.NewReader(requestBody)), - Method: http.MethodGet, - } - r := &response{} - testEnv.scheduler.ServeHTTP(r, request) - - // Check response. - var result schedulerapi.ExtenderFilterResult - err = json.Unmarshal(r.body, &result) - require.NoError(t, err, "unmarshal response") - assert.Equal(t, scenario.expectedError, result.Error) - var names []string - if scenario.nodeCacheCapable { - if result.NodeNames != nil { - names = *result.NodeNames - } - } else { - if result.Nodes != nil { - names = listNodeNames(result.Nodes.Items) - } - } - sort.Strings(names) - assert.Equal(t, scenario.expectedNodes, names) - failures := scenario.expectedFailures - if failures == nil && scenario.expectedError == "" { - failures = schedulerapi.FailedNodesMap{} - } - assert.Equal(t, failures, result.FailedNodes) - } - - for name, scenario := range scenarios { - scenario := scenario - t.Run(name, func(t *testing.T) { - t.Parallel() - run(t, scenario) - }) - } -} - -func TestMutatePod(t *testing.T) { - t.Parallel() - // denied := admission.Denied("pod has no containers") - noFiltering := admission.Allowed("no relevant PMEM volumes") - noVolumes := admission.Allowed("no volumes") - - type scenarioType struct { - // Inputs - pvcs []*v1.PersistentVolumeClaim - inline []inlineVolume - // If nil, makePod with podPVCs - pod *v1.Pod - - // If a result is specified, that is what should be returned. - // If not, then the pod is expected to get filtered. - expectedResult *admission.Response - } - scenarios := map[string]scenarioType{ - "no volumes": { - expectedResult: &noVolumes, - }, - "one PMEM volume": { - pvcs: []*v1.PersistentVolumeClaim{ - unboundPVC, - }, - }, - "one immediate PMEM volume": { - pvcs: []*v1.PersistentVolumeClaim{ - immediateUnboundPVC, - }, - expectedResult: &noFiltering, - }, - "one other volume": { - pvcs: []*v1.PersistentVolumeClaim{ - unboundOtherPVC, - }, - expectedResult: &noFiltering, - }, - "two volumes": { - pvcs: []*v1.PersistentVolumeClaim{ - unboundOtherPVC, - unboundPVC, - }, - }, - "unknown pvc": { - pod: makePod([]*v1.PersistentVolumeClaim{unboundPVC}, nil), - expectedResult: &noFiltering, - }, - "unknown storage class": { - pvcs: []*v1.PersistentVolumeClaim{ - unboundPVCUnknownClass, - }, - expectedResult: &noFiltering, - }, - "one inline volume": { - inline: []inlineVolume{ - { - driverName: driverName, - size: resource.NewQuantity(GiG, resource.BinarySI).String(), - }, - }, - }, - "one other inline volume": { - inline: []inlineVolume{ - { - driverName: driverName + ".example", - size: resource.NewQuantity(GiG, resource.BinarySI).String(), - }, - }, - expectedResult: &noFiltering, - }, - } - - run := func(t *testing.T, scenario scenarioType) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // Setup - testEnv := newTestEnv(t, nil, ctx.Done()) - testEnv.initClaims(scenario.pvcs) - - // Generate input. We keep it very simple. Kubernetes actually sends - // the entire pod and node objects. - pod := scenario.pod - if pod == nil { - pod = makePod(scenario.pvcs, scenario.inline) - } - obj, err := json.Marshal(pod) - require.NoError(t, err, "encode pod") - req := admission.Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - Namespace: "default", - Object: runtime.RawExtension{ - Raw: obj, - }, - }, - } - - // Now mutate. - response := testEnv.scheduler.Handle(context.Background(), req) - - // Check response. - if scenario.expectedResult != nil { - assert.Equal(t, *scenario.expectedResult, response, "webhook result") - } else { - assert.True(t, response.Allowed, "allowed") - // That the patches do indeed add the extended - // resource is covered by the E2E test. - assert.NotEmpty(t, response.Patches, "JSON patch") - } - } - - for name, scenario := range scenarios { - scenario := scenario - t.Run(name, func(t *testing.T) { - t.Parallel() - run(t, scenario) - }) - } -} - -func TestInputValidation(t *testing.T) { - t.Parallel() - type scenarioType struct { - path string - method string - body string - expectedStatus int - expectedResult string - } - scenarios := map[string]scenarioType{ - "status ok": { - expectedStatus: http.StatusOK, - }, - "invalid path": { - path: "/no-such-path", - expectedStatus: http.StatusNotFound, - }, - "invalid method": { - method: http.MethodPatch, - expectedStatus: http.StatusMethodNotAllowed, - }, - "filter ok": { - path: "/filter", - body: `{"pod": { "metadata":{"name":"test-pod"}}, "nodes": {}}`, - expectedStatus: http.StatusOK, - }, - "missing pod": { - path: "/filter", - body: `{"nodes": {}}`, - expectedStatus: http.StatusOK, - expectedResult: "incomplete parameters", - }, - "missing nodes": { - path: "/filter", - body: `{"pod": { "metadata":{"name":"test-pod"}}}`, - expectedStatus: http.StatusOK, - expectedResult: "incomplete parameters", - }, - } - - run := func(t *testing.T, scenario scenarioType) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // Setup - testEnv := newTestEnv(t, clusterCapacity{}, ctx.Done()) - - // Prepare request. - requestBody := []byte(scenario.body) - request := &http.Request{ - URL: &url.URL{Path: "/status"}, - Body: ioutil.NopCloser(bytes.NewReader(requestBody)), - Method: http.MethodGet, - Header: http.Header{}, - } - request.Header.Add("Content-Type", "application/json") - if scenario.method != "" { - request.Method = scenario.method - } - if scenario.path != "" { - request.URL = &url.URL{Path: scenario.path} - } - - // Now handle it. - r := &response{} - testEnv.scheduler.ServeHTTP(r, request) - - // Check response. - require.Equal(t, scenario.expectedStatus, r.statusCode) - switch scenario.path { - case "/filter": - var result schedulerapi.ExtenderFilterResult - err := json.Unmarshal(r.body, &result) - require.NoError(t, err, "unmarshal filter response") - assert.Equal(t, scenario.expectedResult, result.Error) - } - } - - for name, scenario := range scenarios { - scenario := scenario - t.Run(name, func(t *testing.T) { - t.Parallel() - run(t, scenario) - }) - } -} diff --git a/runtime-deps.csv b/runtime-deps.csv index 67c411be2..786f8067d 100644 --- a/runtime-deps.csv +++ b/runtime-deps.csv @@ -62,7 +62,6 @@ k8s.io/component-base,Apache-2.0 k8s.io/klog/v2,Apache-2.0 k8s.io/kube-openapi/pkg,Apache-2.0 k8s.io/kube-openapi/pkg/validation/spec,Apache-2.0 -k8s.io/kube-scheduler/extender/v1,Apache-2.0 k8s.io/kubectl/pkg/scheme,Apache-2.0 k8s.io/utils,Apache-2.0 k8s.io/utils/internal/third_party/forked/golang/net,BSD-3-Clause diff --git a/test/e2e/deploy/deploy.go b/test/e2e/deploy/deploy.go index 7de78f25e..c9cad0e2d 100644 --- a/test/e2e/deploy/deploy.go +++ b/test/e2e/deploy/deploy.go @@ -16,7 +16,6 @@ import ( "os/exec" "reflect" "regexp" - "strconv" "strings" "time" @@ -42,7 +41,6 @@ import ( pmemlog "github.com/intel/pmem-csi/pkg/logger" "github.com/intel/pmem-csi/pkg/version" "github.com/intel/pmem-csi/test/e2e/pod" - testconfig "github.com/intel/pmem-csi/test/test-config" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" @@ -1357,27 +1355,6 @@ func (d *Deployment) GetDriverDeployment(c *Cluster) api.PmemCSIDeployment { }, }, } - - if d.HasController && !c.StorageCapacitySupported() { - dep.Spec.MutatePods = api.MutatePodsAlways - if c.isOpenShift { - // Use certificates prepared by OpenShift. - dep.Spec.ControllerTLSSecret = api.ControllerTLSSecretOpenshift - } else { - // Use a secret that must have been prepared beforehand. - dep.Spec.ControllerTLSSecret = strings.ReplaceAll(d.DriverName, ".", "-") + "-controller-secret" - } - // The scheduler must have been configured manually. We just - // create the corresponding service in the namespace where the - // driver is going to run. - portStr := testconfig.GetOrFail("TEST_SCHEDULER_EXTENDER_NODE_PORT") - port, err := strconv.ParseInt(portStr, 10, 32) - if err != nil { - panic(fmt.Errorf("not an int32: TEST_SCHEDULER_EXTENDER_NODE_PORT=%q: %v", portStr, err)) - } - dep.Spec.SchedulerNodePort = int32(port) - } - return dep } diff --git a/test/e2e/operator/deployment_api.go b/test/e2e/operator/deployment_api.go index 16676a6b1..65fa50ed5 100644 --- a/test/e2e/operator/deployment_api.go +++ b/test/e2e/operator/deployment_api.go @@ -23,7 +23,6 @@ import ( "github.com/intel/pmem-csi/test/e2e/operator/validate" "github.com/intel/pmem-csi/test/e2e/pod" - admissionregistrationv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -301,7 +300,6 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { deployment.Spec.Image = "" deployment.Spec.PMEMPercentage = 50 deployment.Spec.LogFormat = format - deployment.Spec.ControllerTLSSecret = "pmem-csi-intel-com-controller-secret" deployment = deploy.CreateDeploymentCR(f, deployment) defer deploy.DeleteDeploymentCR(f, deployment.Name) @@ -464,36 +462,28 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { It("shall recover from conflicts", func() { deployment := getDeployment("test-recover-from-conflicts") - deployment.Spec.ControllerTLSSecret = "pmem-csi-intel-com-controller-secret" - se := &corev1.Service{ + csiDriver := &storagev1.CSIDriver{ ObjectMeta: metav1.ObjectMeta{ - Name: deployment.GetHyphenedName() + "-scheduler", + Name: deployment.GetHyphenedName(), Namespace: d.Namespace, }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{"app": "foobar"}, - Type: corev1.ServiceTypeClusterIP, - Ports: []corev1.ServicePort{ - {Port: 433}, - }, - }, } - deleteService := func() { + deleteCSIDriver := func() { Eventually(func() error { - err := f.ClientSet.CoreV1().Services(d.Namespace).Delete(context.Background(), se.Name, metav1.DeleteOptions{}) - deploy.LogError(err, "Delete service error: %v, will retry...", err) + err := f.ClientSet.StorageV1().CSIDrivers().Delete(context.Background(), csiDriver.Name, metav1.DeleteOptions{}) + deploy.LogError(err, "Delete CSIDriver error: %v, will retry...", err) if errors.IsNotFound(err) { return nil } return err - }, "3m", "1s").ShouldNot(HaveOccurred(), "delete service %s", se.Name) + }, "3m", "1s").ShouldNot(HaveOccurred(), "delete CSIDriver %s", csiDriver.Name) } Eventually(func() error { - _, err := f.ClientSet.CoreV1().Services(d.Namespace).Create(context.Background(), se, metav1.CreateOptions{}) - deploy.LogError(err, "create service error: %v, will retry...", err) + _, err := f.ClientSet.StorageV1().CSIDrivers().Create(context.Background(), csiDriver, metav1.CreateOptions{}) + deploy.LogError(err, "create CSIDriver error: %v, will retry...", err) return err - }, "3m", "1s").ShouldNot(HaveOccurred(), "create service %s", se.Name) - defer deleteService() + }, "3m", "1s").ShouldNot(HaveOccurred(), "create CSIDriver %s", csiDriver.Name) + defer deleteCSIDriver() deployment = deploy.CreateDeploymentCR(f, deployment) defer deploy.DeleteDeploymentCR(f, deployment.Name) @@ -506,8 +496,8 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { }, "3m", "1s").Should(BeTrue(), "deployment should fail %q", deployment.Name) validateEvents(&deployment, []string{api.EventReasonNew, api.EventReasonFailed}) - // Deleting the existing service should make the deployment succeed. - deleteService() + // Deleting the existing CSIDriver should make the deployment succeed. + deleteCSIDriver() Eventually(func() bool { out := deploy.GetDeploymentCR(f, deployment.Name) return out.Status.Phase == api.DeploymentPhaseRunning @@ -517,59 +507,6 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { err := validate.DriverDeployment(ctx, client, k8sver, d.Namespace, deployment) framework.ExpectNoError(err, "validate driver after resolved conflicts") }) - - It("shall recover from missing secret", func() { - deployment := getDeployment("test-recover-from-missing-secret") - sec := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-controller-secret", - Namespace: d.Namespace, - }, - Type: corev1.SecretTypeTLS, - Data: map[string][]byte{ - "ca.crt": []byte("fake ca"), - "tls.key": []byte("fake key"), - "tls.crt": []byte("fake crt"), - }, - } - deployment.Spec.ControllerTLSSecret = sec.Name - deployment = deploy.CreateDeploymentCR(f, deployment) - defer deploy.DeleteDeploymentCR(f, deployment.Name) - - // The deployment should fail because the required secret is missing. - Eventually(func() bool { - out := deploy.GetDeploymentCR(f, deployment.Name) - return out.Status.Phase == api.DeploymentPhaseFailed - }, "3m", "1s").Should(BeTrue(), "deployment should fail %q", deployment.Name) - validateEvents(&deployment, []string{api.EventReasonNew, api.EventReasonFailed}) - - // Creating the secret should make the deployment succeed. - deleteSecret := func() { - Eventually(func() error { - err := f.ClientSet.CoreV1().Secrets(d.Namespace).Delete(context.Background(), sec.Name, metav1.DeleteOptions{}) - deploy.LogError(err, "Delete secret error: %v, will retry...", err) - if errors.IsNotFound(err) { - return nil - } - return err - }, "3m", "1s").ShouldNot(HaveOccurred(), "delete secret %s", sec.Name) - } - Eventually(func() error { - _, err := f.ClientSet.CoreV1().Secrets(d.Namespace).Create(context.Background(), sec, metav1.CreateOptions{}) - deploy.LogError(err, "create secret error: %v, will retry...", err) - return err - }, "3m", "1s").ShouldNot(HaveOccurred(), "create secret %s", sec.Name) - defer deleteSecret() - // Now as the missing secrets are there in place, the deployment should - // move ahead and get succeed - Eventually(func() bool { - deployment := deploy.GetDeploymentCR(f, deployment.Name) - return deployment.Status.Phase == api.DeploymentPhaseRunning - }, "3m", "1s").Should(BeTrue(), "deployment should not fail %q", deployment.Name) - validateEvents(&deployment, []string{api.EventReasonNew, api.EventReasonRunning}) - err := validate.DriverDeployment(context.Background(), client, k8sver, d.Namespace, deployment) - Expect(err).ShouldNot(HaveOccurred(), "validate driver after secret creation") - }) }) Context("switch device mode", func() { @@ -805,11 +742,6 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { ObjectMeta: metav1.ObjectMeta{Name: dep.WebhooksServiceAccountName(), Namespace: d.Namespace}, } }, - "scheduler service": func(dep *api.PmemCSIDeployment) runtime.Object { - return &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{Name: dep.SchedulerServiceName(), Namespace: d.Namespace}, - } - }, "webhooks role": func(dep *api.PmemCSIDeployment) runtime.Object { return &rbacv1.Role{ ObjectMeta: metav1.ObjectMeta{Name: dep.WebhooksRoleName(), Namespace: d.Namespace}, @@ -830,11 +762,6 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { ObjectMeta: metav1.ObjectMeta{Name: dep.WebhooksClusterRoleBindingName()}, } }, - "mutating webhook config": func(dep *api.PmemCSIDeployment) runtime.Object { - return &admissionregistrationv1.MutatingWebhookConfiguration{ - ObjectMeta: metav1.ObjectMeta{Name: dep.MutatingWebhookName()}, - } - }, "provisioner role": func(dep *api.PmemCSIDeployment) runtime.Object { return &rbacv1.Role{ ObjectMeta: metav1.ObjectMeta{Name: dep.ProvisionerRoleName(), Namespace: d.Namespace}, @@ -887,8 +814,6 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { It(name, func() { // Create a deployment with controller and webhook config. dep := getDeployment("recover-" + strings.ReplaceAll(name, " ", "-")) - dep.Spec.ControllerTLSSecret = "pmem-csi-intel-com-controller-secret" - dep.Spec.MutatePods = api.MutatePodsAlways deployment := deploy.CreateDeploymentCR(f, dep) defer deploy.DeleteDeploymentCR(f, dep.Name) validateDriver(deployment, nil) @@ -939,7 +864,6 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { name, mutate := name, mutate It(name, func() { dep := getDeployment("recover-" + strings.ReplaceAll(name, " ", "-")) - dep.Spec.ControllerTLSSecret = "pmem-csi-intel-com-controller-secret" deployment := deploy.CreateDeploymentCR(f, dep) defer deploy.DeleteDeploymentCR(f, dep.Name) validateDriver(deployment, nil, "validate before conflicts") diff --git a/test/e2e/operator/validate/validate.go b/test/e2e/operator/validate/validate.go index 39c544e43..58ee2fc11 100644 --- a/test/e2e/operator/validate/validate.go +++ b/test/e2e/operator/validate/validate.go @@ -24,12 +24,9 @@ import ( "github.com/intel/pmem-csi/pkg/pmem-csi-operator/metrics" "github.com/intel/pmem-csi/pkg/version" "github.com/intel/pmem-csi/test/e2e/deploy" - apierrs "k8s.io/apimachinery/pkg/api/errors" cm "github.com/prometheus/client_model/go" "gopkg.in/yaml.v2" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -225,31 +222,7 @@ func DriverDeployment(ctx context.Context, c client.Client, k8sver version.Versi return err } - // Load secret if it exists. If it doesn't, we validate without it. - var controllerCABundle []byte - if deployment.Spec.ControllerTLSSecret != "" { - secret := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Secret", - }, - } - objKey := client.ObjectKey{ - Namespace: namespace, - Name: deployment.Spec.ControllerTLSSecret, - } - if err := c.Get(ctx, objKey, secret); err != nil { - if !apierrs.IsNotFound(err) { - return err - } - } else { - if ca, ok := secret.Data[api.TLSSecretCA]; ok { - controllerCABundle = ca - } - } - } - - expectedObjects, err := deployments.LoadAndCustomizeObjects(k8sver, deployment.Spec.DeviceMode, namespace, deployment, controllerCABundle) + expectedObjects, err := deployments.LoadAndCustomizeObjects(k8sver, deployment.Spec.DeviceMode, namespace, deployment) if err != nil { return fmt.Errorf("customize expected objects: %v", err) } diff --git a/test/e2e/storage/pmem_csi.go b/test/e2e/storage/pmem_csi.go index 0ecedd079..9deb65f9f 100644 --- a/test/e2e/storage/pmem_csi.go +++ b/test/e2e/storage/pmem_csi.go @@ -26,7 +26,6 @@ import ( "github.com/intel/pmem-csi/test/e2e/deploy" "github.com/intel/pmem-csi/test/e2e/driver" "github.com/intel/pmem-csi/test/e2e/operator/validate" - "github.com/intel/pmem-csi/test/e2e/storage/scheduler" "github.com/intel/pmem-csi/test/e2e/versionskew" . "github.com/onsi/ginkgo/v2" @@ -42,11 +41,6 @@ var _ = deploy.DescribeForAll("Deployment", func(d *deploy.Deployment) { versionskew.InitSkewTestSuite, } - if d.HasController { - // Scheduler tests depend on the webhooks in the controller. - csiTestSuites = append(csiTestSuites, scheduler.InitSchedulerTestSuite) - } - storageframework.DefineTestSuites(csiTestDriver, csiTestSuites) }) diff --git a/test/e2e/storage/scheduler/scheduler.go b/test/e2e/storage/scheduler/scheduler.go deleted file mode 100644 index 4ea559938..000000000 --- a/test/e2e/storage/scheduler/scheduler.go +++ /dev/null @@ -1,199 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package scheduler - -import ( - "fmt" - "os" - - v1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kubernetes/test/e2e/framework" - e2epod "k8s.io/kubernetes/test/e2e/framework/pod" - e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" - "k8s.io/kubernetes/test/e2e/framework/volume" - storageframework "k8s.io/kubernetes/test/e2e/storage/framework" - - "github.com/intel/pmem-csi/test/e2e/deploy" - e2edriver "github.com/intel/pmem-csi/test/e2e/driver" - "github.com/intel/pmem-csi/test/e2e/ephemeral" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -type schedulerTestSuite struct { - tsInfo storageframework.TestSuiteInfo -} - -var _ storageframework.TestSuite = &schedulerTestSuite{} - -// InitSchedulerTestSuite returns a test suite which verifies that the scheduler extender and -// webhook work. -func InitSchedulerTestSuite() storageframework.TestSuite { - // We test with an ephemeral inline volume and a PVC with late - // binding. - lateBinding := storageframework.DefaultFsDynamicPV - lateBinding.Name = "Dynamic PV with late binding" - lateBinding.BindingMode = storagev1.VolumeBindingWaitForFirstConsumer - - suite := &schedulerTestSuite{ - tsInfo: storageframework.TestSuiteInfo{ - Name: "scheduler", - TestPatterns: []storageframework.TestPattern{ - lateBinding, - }, - }, - } - if ephemeral.Supported { - suite.tsInfo.TestPatterns = append(suite.tsInfo.TestPatterns, - storageframework.DefaultFsCSIEphemeralVolume, - ) - } - return suite -} - -func (p *schedulerTestSuite) GetTestSuiteInfo() storageframework.TestSuiteInfo { - return p.tsInfo -} - -func (p *schedulerTestSuite) SkipUnsupportedTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) { -} - -type local struct { - config *storageframework.PerTestConfig - testCleanup func() - - resource *storageframework.VolumeResource -} - -func (p *schedulerTestSuite) DefineTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) { - var l local - - f := framework.NewDefaultFramework("scheduler") - - init := func() { - l = local{} - - c, err := deploy.NewCluster(f.ClientSet, f.DynamicClient, f.ClientConfig()) - framework.ExpectNoError(err, "create cluster object") - if c.StorageCapacitySupported() { - e2eskipper.Skipf("storage capacity tracking is enabled, not using scheduler extensions") - } - - // Now do the more expensive test initialization. - l.config, l.testCleanup = driver.PrepareTest(f) - l.resource = storageframework.CreateVolumeResource(driver, l.config, pattern, volume.SizeRange{}) - } - - cleanup := func() { - if l.resource != nil { - _ = l.resource.CleanupResource() - l.resource = nil - } - - if l.testCleanup != nil { - l.testCleanup() - l.testCleanup = nil - } - } - - It("should call PMEM-CSI controller", func() { - init() - defer cleanup() - - driverName := driver.(e2edriver.CSIDriver).GetCSIDriverName(l.config) - - l.testSchedulerInPod(f, driverName, l.resource.Pattern.VolType, l.resource.VolSource, l.config) - }) -} - -func (l local) testSchedulerInPod( - f *framework.Framework, - driverName string, - volumeType storageframework.TestVolType, - source *v1.VolumeSource, - config *storageframework.PerTestConfig) { - - const ( - volPath = "/vol1" - volName = "vol1" - containerName = "scheduler-container" - ) - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "dax-volume-test", - Namespace: f.Namespace.Name, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: containerName, - Image: os.Getenv("PMEM_CSI_IMAGE"), - Command: []string{"sleep", "1000000"}, - VolumeMounts: []v1.VolumeMount{ - v1.VolumeMount{ - Name: volName, - MountPath: "/mnt", - }, - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: volName, - VolumeSource: *source, - }, - }, - RestartPolicy: v1.RestartPolicyNever, - }, - } - e2epod.SetNodeSelection(&pod.Spec, config.ClientNodeSelection) - - By(fmt.Sprintf("Creating pod %s", pod.Name)) - ns := f.Namespace.Name - podClient := f.PodClientNS(ns) - createdPod := podClient.Create(pod) - defer func() { - By("delete the pod") - podClient.DeleteSync(createdPod.Name, metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout) - }() - - resourceName := v1.ResourceName(driverName + "/scheduler") - - Expect(createdPod.Spec.Containers[0].Resources).NotTo(BeNil(), "pod resources") - Expect(createdPod.Spec.Containers[0].Resources.Requests).NotTo(BeNil(), "pod resource requests") - _, ok := createdPod.Spec.Containers[0].Resources.Requests[resourceName] - Expect(ok).To(BeTrue(), "PMEM-CSI extended resource request") - Expect(createdPod.Spec.Containers[0].Resources.Limits).NotTo(BeNil(), "pod resource requests") - _, ok = createdPod.Spec.Containers[0].Resources.Requests[resourceName] - Expect(ok).To(BeTrue(), "PMEM-CSI extended resource limit") - - podErr := e2epod.WaitForPodRunningInNamespace(f.ClientSet, createdPod) - framework.ExpectNoError(podErr, "running pod") - - // If we get here, we know that the scheduler extender - // worked. If it wasn't active, kube-scheduler would have - // tried to handle pmem-csi.intel.com/scheduler itself, which - // can't work because there is no node provising that - // resource. - - By(fmt.Sprintf("Deleting pod %s", pod.Name)) - err := e2epod.DeletePodWithWait(f.ClientSet, pod) - framework.ExpectNoError(err, "while deleting pod") -} diff --git a/test/e2e/tls/tls.go b/test/e2e/tls/tls.go index 9b9fad836..3c8effb50 100644 --- a/test/e2e/tls/tls.go +++ b/test/e2e/tls/tls.go @@ -209,7 +209,7 @@ func checkTLS(f *framework.Framework, server string) { // in our Dockerfile) // - the generated keys and thus the deployment method (the // current list is for "make start" and keys created with - // test/setup-ca.sh, which in turn uses cfssl as installed + // test/setup-ca.sh from PMEM-CSI 1.0.x, which in turn used cfssl as installed // by test/test.make, at least in the CI) // // This list may have to be adapted when changing either of these. diff --git a/test/setup-ca-kubernetes.sh b/test/setup-ca-kubernetes.sh deleted file mode 100755 index c77b21450..000000000 --- a/test/setup-ca-kubernetes.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/sh -e - -# This script generates certificates using setup-ca.sh and converts them into -# the Kubernetes secrets that the PMEM-CSI deployments rely upon for -# securing communication between PMEM-CSI components. Existing secrets -# are updated with new certificates when running it again. - -# The script needs a functional kubectl that uses the target cluster. -: ${KUBECTL:=kubectl} - -# The directory containing setup-ca*.sh. -: ${TEST_DIRECTORY:=$(dirname $(readlink -f $0))} -. ${TEST_CONFIG:-${TEST_DIRECTORY}/test-config.sh} - -tmpdir=`mktemp -d` -trap 'rm -r $tmpdir' EXIT - -# Generate certificates. They are not going to be needed again and will -# be deleted together with the temp directory. Only the root CA is -# stored in a permanent location. -WORKDIR="$tmpdir" CA="$TEST_CA" NS="${TEST_DRIVER_NAMESPACE}" PREFIX="${TEST_DRIVER_PREFIX}" "$TEST_DIRECTORY/setup-ca.sh" - -# This reads a file and encodes it for use in a secret. -read_key () { - base64 -w 0 "$1" -} - -# Read certificate files and turn them into Kubernetes secrets. -# -# -caFile (controller and all nodes) -CA=$(read_key "${TEST_CA}.pem") -# -certFile (controller) -CONTROLLER_CERT=$(read_key "$tmpdir/pmem-controller.pem") -# -keyFile (controller) -CONTROLLER_KEY=$(read_key "$tmpdir/pmem-controller-key.pem") - -${KUBECTL} get ns ${TEST_DRIVER_NAMESPACE} 2>/dev/null >/dev/null || ${KUBECTL} create ns ${TEST_DRIVER_NAMESPACE} - -${KUBECTL} apply -f - <&1 >/dev/null && command -v cfssljson 2>&1 >/dev/null) || cfssl_found=0 -if [ $cfssl_found -eq 0 ]; then - echo "cfssl tools not found, Please install cfssl and cfssljson." - exit 1 -fi - -CADIR=$(dirname ${CA}) -mkdir -p "${CADIR}" -CA_CRT=$(realpath ${CA}.pem) -CA_KEY=$(realpath ${CA}-key.pem) -if ! [ -f ${CA_CRT} -a -f ${CA_KEY} ]; then - echo "Generating CA certificate in $CADIR ..." - (cd "$CADIR" && - <= 1.21. - DEPLOY+=(scheduler webhook) - if ${KUBECTL} get crd | grep -q schedulers.config.openshift.io; then - # Only the service gets deployed anew, with TEST_SCHEDULER_EXTENDER_NODE_PORT as - # well-known, fixed node port. Config map and scheduler config must be - # set up manually once before running tests. - OPENSHIFT_SCHEDULER=true - - # Do some sanity checkig... - policy=$(${KUBECTL} get scheduler/cluster -o jsonpath={.spec.policy.name}) - if [ ! "$policy" ]; then - echo >&2 "The scheduler config in scheduler/cluster must be set up manually. Currently there is no policy." - fi - if ! ${KUBECTL} get -n openshift-config configmap/$policy >/dev/null; then - echo >&2 "The scheduler policy configmap must be set up manually." - fi - fi - ;; -esac +# Set up the TEST_DRIVER_NAMESPACE. +if ! ${KUBECTL} get "ns/${TEST_DRIVER_NAMESPACE}" 2>/dev/null >/dev/null; then + ${KUBECTL} create ns "${TEST_DRIVER_NAMESPACE}" +fi for deploy in ${DEPLOY[@]}; do # Deployment files can come from: @@ -117,32 +96,8 @@ EOF ${TEST_DEVICEMODE}${deployment_suffix}) ${SSH} "cat >>'$tmpdir/my-deployment/kustomization.yaml'" <'$tmpdir/my-deployment/scheduler-patch.yaml'" <'$tmpdir/my-deployment/scheduler-patch.yaml'" <>'$tmpdir/my-deployment/kustomization.yaml'" <'$tmpdir/my-deployment/scheduler-patch.yaml'" <>'$tmpdir/my-deployment/scheduler-patch.yaml'" <>'$tmpdir/my-deployment/kustomization.yaml'" <'$tmpdir/my-deployment/webhook-patch.yaml'" <>'$tmpdir/my-deployment/kustomization.yaml'" <'$tmpdir/my-deployment/webhook-service-patch.yaml'" <'$tmpdir/my-deployment/webhook-patch.yaml'" <>'$tmpdir/my-deployment/kustomization.yaml'" </var/lib/scheduler/scheduler-config.yaml' </var/lib/scheduler/scheduler-policy.cfg' </var/lib/scheduler/scheduler-config.yaml' </dev/null || die "failed to ca.pem to $vm_name = $ip" scp $SSH_ARGS ${TEST_DIRECTORY}/${install_k8s_script} ${CLOUD_USER}@${ip}:. >/dev/null || die "failed to copy install scripts to $vm_name = $ip" ssh $SSH_ARGS ${CLOUD_USER}@${ip} "env $ENV_VARS ./$install_k8s_script" >(log_lines "$vm_name" "$log_name") & pids+=" $!" diff --git a/test/start-operator.sh b/test/start-operator.sh index 22e6c5453..98c77a45b 100755 --- a/test/start-operator.sh +++ b/test/start-operator.sh @@ -171,12 +171,6 @@ case $deploy_method in exit 1 ;; esac -if [ "$upgrade" == "" ] ; then - # Set up TLS secrets in the TEST_OPERATOR_NAMESPACE, with the two different prefixes. - PATH="${REPO_ROOT}/_work/bin:$PATH" TEST_DRIVER_NAMESPACE="${TEST_OPERATOR_NAMESPACE}" TEST_DRIVER_PREFIX=second-pmem-csi-intel-com ${TEST_DIRECTORY}/setup-ca-kubernetes.sh - PATH="${REPO_ROOT}/_work/bin:$PATH" TEST_DRIVER_NAMESPACE="${TEST_OPERATOR_NAMESPACE}" ${TEST_DIRECTORY}/setup-ca-kubernetes.sh -fi - cat <= 1.11.0-rc0 for support of PMEM volumes. : ${TEST_KATA_CONTAINERS_VERSION:=} -# Kubernetes node port number -# (https://kubernetes.io/docs/concepts/services-networking/service/#nodeport) -# that is going to be used by kube-scheduler to reach the scheduler -# extender service (see test/setup-kubernetes.sh) -: ${TEST_SCHEDULER_EXTENDER_NODE_PORT:=32000} - # The OpenShift baremetal installer binary. : ${TEST_OPENSHIFT_INSTALLER:=openshift-baremetal-install} diff --git a/test/test.make b/test/test.make index 82f67e872..10ce98d21 100644 --- a/test/test.make +++ b/test/test.make @@ -190,7 +190,9 @@ RUN_E2E = KUBECONFIG=`pwd`/_work/$(CLUSTER)/kube.config \ -ginkgo.timeout=$(TEST_E2E_TIMEOUT) \ $(TEST_E2E_ARGS) \ -report-dir=$(TEST_E2E_REPORT_DIR) -test_e2e: start $(RUN_TEST_DEPS) operator-generate-bundle + +# Downgrade tests to v1.0.x depends on cfssl because the v1.0.x install scripts need it. +test_e2e: start $(RUN_TEST_DEPS) operator-generate-bundle _work/.setupcfssl-stamp _work/.operator-sdk-stamp $(RUN_E2E) run_dm_tests: TEST_BINARY_NAME=pmem-dm-tests @@ -204,12 +206,6 @@ run_dm_tests: _work/bin/govm start_test_vm scp $$SSH_ARGS `pwd`/_work/$(TEST_BINARY_NAME) $$SSH_USER@$$NODE_IP:. ; \ $$SSH sudo ./$(TEST_BINARY_NAME) -ginkgo.v -_work/%/.ca-stamp: test/setup-ca.sh _work/.setupcfssl-stamp - rm -rf $(@D) - WORKDIR='$(@D)' PATH='$(PWD)/_work/bin/:$(PATH)' NS=default $< - touch $@ - - _work/.setupcfssl-stamp: CFSSL_VERSION=1.5.0 _work/.setupcfssl-stamp: rm -rf _work/bin From c19b0f1beb3db38f49a850a5b100faa3ece54566 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 8 Dec 2022 08:31:26 +0100 Subject: [PATCH 4/6] test: disable testing against OLM Installing it was a constant source of test failures in the CI. --- test/e2e/deploy/deploy.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/e2e/deploy/deploy.go b/test/e2e/deploy/deploy.go index c9cad0e2d..32ddb6624 100644 --- a/test/e2e/deploy/deploy.go +++ b/test/e2e/deploy/deploy.go @@ -1017,9 +1017,12 @@ func findOperatorOnce(c *Cluster) (*Deployment, error) { // errors ("operatorhubio-catalog-lbdrz 0/1 CrashLoopBackOff") that did not // occur locally or when running the tests on a "fresh" CI cluster. var allDeployments = []string{ - "olm", // operator installed by OLM - "olm-lvm-production", - "olm-direct-production", + // Testing with OLM is disabled. Installing it was a frequent source of + // random failures, it's simply not reliable enough. + // + // "olm", // operator installed by OLM + // "olm-lvm-production", + // "olm-direct-production", "lvm-production", "direct-production", From bbfb7f28364d1da027410adfa8f179a39356f092 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 13 Dec 2022 09:06:09 +0100 Subject: [PATCH 5/6] test: skip image validation if govm is missing Not sure what changed in Jenkins, but "govm" is not found on all worker VMs anymore. --- pkg/imagefile/test/imagefiletest.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/imagefile/test/imagefiletest.go b/pkg/imagefile/test/imagefiletest.go index fd4b34c69..097f29e04 100644 --- a/pkg/imagefile/test/imagefiletest.go +++ b/pkg/imagefile/test/imagefiletest.go @@ -105,6 +105,10 @@ func testImageFile(t TInterface, fs imagefile.FsType, size imagefile.Bytes, expe t.Log("for testing the image under QEMU, download files for a cluster and set REPO_ROOT and CLUSTER") return } + if _, err := exec.LookPath("govm"); err != nil { + t.Log("for testing the image under QEMU, install govm") + return + } env := []string{ "EXISTING_VM_FILE=" + file.Name(), os.ExpandEnv("PATH=${REPO_ROOT}/_work/bin:${PATH}"), From 003fada88ecd0baac95e4d49aa9951016e556814 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 13 Dec 2022 11:06:26 +0100 Subject: [PATCH 6/6] test: enable -ginkgo.v when using dlv The output then is more suitable for interactive usage. --- test/test.make | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.make b/test/test.make index 10ce98d21..4e32e04b5 100644 --- a/test/test.make +++ b/test/test.make @@ -157,7 +157,7 @@ space:= $(empty) $(empty) GO_TEST_E2E = $(GO) test -count=1 -timeout 0 -v ./test/e2e -args ifneq ($(WITH_DLV),) -GO_TEST_E2E = dlv test ./test/e2e -- +GO_TEST_E2E = dlv test ./test/e2e -- -ginkgo.v endif # E2E testing relies on a running QEMU test cluster. It therefore starts it,