From 91601edb3d21665329db8cad2f8fc9cb785f2755 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 1 Jul 2021 13:45:08 +0200 Subject: [PATCH] operator: OpenShift support On OpenShift we can use automatically generated certificates and have to use http for the scheduler extender. The scheduler extender and API webhooks then need services with different settings, therefore we now configure them separately. While at it, the formatting of the operator API documentation gets enhanced (remove unnecessary indention, odd placement of note). On 1.20, the operator testing still used the scheduler extensions whereas the YAML files didn't. Now both stop using them, for the sake of consistency. --- ...pmem-csi.intel.com_pmemcsideployments.yaml | 4 +- deploy/kubernetes-1.19/direct/pmem-csi.yaml | 2 +- .../direct/testing/pmem-csi.yaml | 2 +- deploy/kubernetes-1.19/lvm/pmem-csi.yaml | 2 +- .../kubernetes-1.19/lvm/testing/pmem-csi.yaml | 2 +- .../pmem-csi-direct-testing.yaml | 2 +- deploy/kubernetes-1.19/pmem-csi-direct.yaml | 2 +- .../kubernetes-1.19/pmem-csi-lvm-testing.yaml | 2 +- deploy/kubernetes-1.19/pmem-csi-lvm.yaml | 2 +- deploy/kubernetes-1.20/direct/pmem-csi.yaml | 2 +- .../direct/testing/pmem-csi.yaml | 2 +- deploy/kubernetes-1.20/lvm/pmem-csi.yaml | 2 +- .../kubernetes-1.20/lvm/testing/pmem-csi.yaml | 2 +- .../pmem-csi-direct-testing.yaml | 2 +- deploy/kubernetes-1.20/pmem-csi-direct.yaml | 2 +- .../kubernetes-1.20/pmem-csi-lvm-testing.yaml | 2 +- deploy/kubernetes-1.20/pmem-csi-lvm.yaml | 2 +- deploy/kubernetes-1.21/direct/pmem-csi.yaml | 2 +- .../direct/testing/pmem-csi.yaml | 2 +- deploy/kubernetes-1.21/lvm/pmem-csi.yaml | 2 +- .../kubernetes-1.21/lvm/testing/pmem-csi.yaml | 2 +- .../pmem-csi-direct-testing.yaml | 2 +- deploy/kubernetes-1.21/pmem-csi-direct.yaml | 2 +- .../kubernetes-1.21/pmem-csi-lvm-testing.yaml | 2 +- deploy/kubernetes-1.21/pmem-csi-lvm.yaml | 2 +- deploy/kustomize/driver/pmem-csi.yaml | 2 +- deploy/kustomize/webhook/kustomization.yaml | 1 + deploy/kustomize/webhook/webhook-service.yaml | 12 ++ deploy/kustomize/webhook/webhook.yaml | 2 +- deploy/yamls.go | 1 + docs/install.md | 199 ++++++++++++++++-- go.mod | 2 +- pkg/apis/pmemcsi/v1beta1/deployment_types.go | 28 ++- pkg/deployments/load.go | 70 ++++-- pkg/k8sutil/client.go | 20 ++ .../deployment/controller_driver.go | 123 +++++++++-- .../deployment/deployment_controller.go | 8 +- .../deployment/deployment_controller_test.go | 4 + .../deployment/testcases/testcases.go | 20 ++ runtime-deps.csv | 1 + test/e2e/deploy/cluster.go | 21 ++ test/e2e/deploy/deploy.go | 31 ++- test/e2e/operator/driver.go | 10 +- test/e2e/operator/validate/validate.go | 1 + test/e2e/storage/scheduler/scheduler.go | 4 +- test/setup-ca.sh | 4 + test/setup-deployment.sh | 61 +++++- 47 files changed, 566 insertions(+), 111 deletions(-) create mode 100644 deploy/kustomize/webhook/webhook-service.yaml diff --git a/deploy/crd/pmem-csi.intel.com_pmemcsideployments.yaml b/deploy/crd/pmem-csi.intel.com_pmemcsideployments.yaml index 7078fb1daf..e777f1466a 100644 --- a/deploy/crd/pmem-csi.intel.com_pmemcsideployments.yaml +++ b/deploy/crd/pmem-csi.intel.com_pmemcsideployments.yaml @@ -86,7 +86,9 @@ spec: description: ControllerTLSSecret is the name of a secret which contains ca.crt, tls.crt and tls.key data for the scheduler extender and pod mutation webhook. A controller is started if (and only if) this - secret is specified. + secret is specified. The special string "-openshift-" enables the + usage of https://docs.openshift.com/container-platform/4.6/security/certificates/service-serving-certificate.html + to create certificates. type: string deviceMode: description: DeviceMode to use to manage PMEM devices. diff --git a/deploy/kubernetes-1.19/direct/pmem-csi.yaml b/deploy/kubernetes-1.19/direct/pmem-csi.yaml index 0dd2ca248e..594d3ea725 100644 --- a/deploy/kubernetes-1.19/direct/pmem-csi.yaml +++ b/deploy/kubernetes-1.19/direct/pmem-csi.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kubernetes-1.19/direct/testing/pmem-csi.yaml b/deploy/kubernetes-1.19/direct/testing/pmem-csi.yaml index fa9f008835..356821f71b 100644 --- a/deploy/kubernetes-1.19/direct/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.19/direct/testing/pmem-csi.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kubernetes-1.19/lvm/pmem-csi.yaml b/deploy/kubernetes-1.19/lvm/pmem-csi.yaml index 91c3d68fc7..56ccb5c289 100644 --- a/deploy/kubernetes-1.19/lvm/pmem-csi.yaml +++ b/deploy/kubernetes-1.19/lvm/pmem-csi.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kubernetes-1.19/lvm/testing/pmem-csi.yaml b/deploy/kubernetes-1.19/lvm/testing/pmem-csi.yaml index 89fe9ed1a8..50359719a3 100644 --- a/deploy/kubernetes-1.19/lvm/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.19/lvm/testing/pmem-csi.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kubernetes-1.19/pmem-csi-direct-testing.yaml b/deploy/kubernetes-1.19/pmem-csi-direct-testing.yaml index fa9f008835..356821f71b 100644 --- a/deploy/kubernetes-1.19/pmem-csi-direct-testing.yaml +++ b/deploy/kubernetes-1.19/pmem-csi-direct-testing.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kubernetes-1.19/pmem-csi-direct.yaml b/deploy/kubernetes-1.19/pmem-csi-direct.yaml index 0dd2ca248e..594d3ea725 100644 --- a/deploy/kubernetes-1.19/pmem-csi-direct.yaml +++ b/deploy/kubernetes-1.19/pmem-csi-direct.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kubernetes-1.19/pmem-csi-lvm-testing.yaml b/deploy/kubernetes-1.19/pmem-csi-lvm-testing.yaml index 89fe9ed1a8..50359719a3 100644 --- a/deploy/kubernetes-1.19/pmem-csi-lvm-testing.yaml +++ b/deploy/kubernetes-1.19/pmem-csi-lvm-testing.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kubernetes-1.19/pmem-csi-lvm.yaml b/deploy/kubernetes-1.19/pmem-csi-lvm.yaml index 91c3d68fc7..56ccb5c289 100644 --- a/deploy/kubernetes-1.19/pmem-csi-lvm.yaml +++ b/deploy/kubernetes-1.19/pmem-csi-lvm.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kubernetes-1.20/direct/pmem-csi.yaml b/deploy/kubernetes-1.20/direct/pmem-csi.yaml index 0dd2ca248e..594d3ea725 100644 --- a/deploy/kubernetes-1.20/direct/pmem-csi.yaml +++ b/deploy/kubernetes-1.20/direct/pmem-csi.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kubernetes-1.20/direct/testing/pmem-csi.yaml b/deploy/kubernetes-1.20/direct/testing/pmem-csi.yaml index fa9f008835..356821f71b 100644 --- a/deploy/kubernetes-1.20/direct/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.20/direct/testing/pmem-csi.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kubernetes-1.20/lvm/pmem-csi.yaml b/deploy/kubernetes-1.20/lvm/pmem-csi.yaml index 91c3d68fc7..56ccb5c289 100644 --- a/deploy/kubernetes-1.20/lvm/pmem-csi.yaml +++ b/deploy/kubernetes-1.20/lvm/pmem-csi.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kubernetes-1.20/lvm/testing/pmem-csi.yaml b/deploy/kubernetes-1.20/lvm/testing/pmem-csi.yaml index 89fe9ed1a8..50359719a3 100644 --- a/deploy/kubernetes-1.20/lvm/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.20/lvm/testing/pmem-csi.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kubernetes-1.20/pmem-csi-direct-testing.yaml b/deploy/kubernetes-1.20/pmem-csi-direct-testing.yaml index fa9f008835..356821f71b 100644 --- a/deploy/kubernetes-1.20/pmem-csi-direct-testing.yaml +++ b/deploy/kubernetes-1.20/pmem-csi-direct-testing.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kubernetes-1.20/pmem-csi-direct.yaml b/deploy/kubernetes-1.20/pmem-csi-direct.yaml index 0dd2ca248e..594d3ea725 100644 --- a/deploy/kubernetes-1.20/pmem-csi-direct.yaml +++ b/deploy/kubernetes-1.20/pmem-csi-direct.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kubernetes-1.20/pmem-csi-lvm-testing.yaml b/deploy/kubernetes-1.20/pmem-csi-lvm-testing.yaml index 89fe9ed1a8..50359719a3 100644 --- a/deploy/kubernetes-1.20/pmem-csi-lvm-testing.yaml +++ b/deploy/kubernetes-1.20/pmem-csi-lvm-testing.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kubernetes-1.20/pmem-csi-lvm.yaml b/deploy/kubernetes-1.20/pmem-csi-lvm.yaml index 91c3d68fc7..56ccb5c289 100644 --- a/deploy/kubernetes-1.20/pmem-csi-lvm.yaml +++ b/deploy/kubernetes-1.20/pmem-csi-lvm.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kubernetes-1.21/direct/pmem-csi.yaml b/deploy/kubernetes-1.21/direct/pmem-csi.yaml index 4e2ef10fd3..9a7e718345 100644 --- a/deploy/kubernetes-1.21/direct/pmem-csi.yaml +++ b/deploy/kubernetes-1.21/direct/pmem-csi.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kubernetes-1.21/direct/testing/pmem-csi.yaml b/deploy/kubernetes-1.21/direct/testing/pmem-csi.yaml index 709656bfc2..1384b6633f 100644 --- a/deploy/kubernetes-1.21/direct/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.21/direct/testing/pmem-csi.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kubernetes-1.21/lvm/pmem-csi.yaml b/deploy/kubernetes-1.21/lvm/pmem-csi.yaml index cf86896824..687d215f17 100644 --- a/deploy/kubernetes-1.21/lvm/pmem-csi.yaml +++ b/deploy/kubernetes-1.21/lvm/pmem-csi.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kubernetes-1.21/lvm/testing/pmem-csi.yaml b/deploy/kubernetes-1.21/lvm/testing/pmem-csi.yaml index d3cedf4074..2aecf3d926 100644 --- a/deploy/kubernetes-1.21/lvm/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.21/lvm/testing/pmem-csi.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kubernetes-1.21/pmem-csi-direct-testing.yaml b/deploy/kubernetes-1.21/pmem-csi-direct-testing.yaml index 709656bfc2..1384b6633f 100644 --- a/deploy/kubernetes-1.21/pmem-csi-direct-testing.yaml +++ b/deploy/kubernetes-1.21/pmem-csi-direct-testing.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kubernetes-1.21/pmem-csi-direct.yaml b/deploy/kubernetes-1.21/pmem-csi-direct.yaml index 4e2ef10fd3..9a7e718345 100644 --- a/deploy/kubernetes-1.21/pmem-csi-direct.yaml +++ b/deploy/kubernetes-1.21/pmem-csi-direct.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kubernetes-1.21/pmem-csi-lvm-testing.yaml b/deploy/kubernetes-1.21/pmem-csi-lvm-testing.yaml index d3cedf4074..2aecf3d926 100644 --- a/deploy/kubernetes-1.21/pmem-csi-lvm-testing.yaml +++ b/deploy/kubernetes-1.21/pmem-csi-lvm-testing.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kubernetes-1.21/pmem-csi-lvm.yaml b/deploy/kubernetes-1.21/pmem-csi-lvm.yaml index cf86896824..687d215f17 100644 --- a/deploy/kubernetes-1.21/pmem-csi-lvm.yaml +++ b/deploy/kubernetes-1.21/pmem-csi-lvm.yaml @@ -364,7 +364,7 @@ spec: - -mode=webhooks - -drivername=$(PMEM_CSI_DRIVER_NAME) - -nodeSelector={"storage":"pmem"} - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kustomize/driver/pmem-csi.yaml b/deploy/kustomize/driver/pmem-csi.yaml index 86c3f54360..4bd2dd491a 100644 --- a/deploy/kustomize/driver/pmem-csi.yaml +++ b/deploy/kustomize/driver/pmem-csi.yaml @@ -167,7 +167,7 @@ spec: - -nodeSelector={"storage":"pmem"} # ca.crt is present in pmem-csi-intel-com-controller-secret but not required for anything at # the moment. - - -caFile=/certs/ca.crt + - -caFile= - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -schedulerListen=:8000 diff --git a/deploy/kustomize/webhook/kustomization.yaml b/deploy/kustomize/webhook/kustomization.yaml index d8317be2a0..103ffe6705 100644 --- a/deploy/kustomize/webhook/kustomization.yaml +++ b/deploy/kustomize/webhook/kustomization.yaml @@ -1,2 +1,3 @@ resources: - webhook.yaml + - webhook-service.yaml diff --git a/deploy/kustomize/webhook/webhook-service.yaml b/deploy/kustomize/webhook/webhook-service.yaml new file mode 100644 index 0000000000..9d3a2e497a --- /dev/null +++ b/deploy/kustomize/webhook/webhook-service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: pmem-csi-intel-com-webhook + namespace: pmem-csi +spec: + selector: + app.kubernetes.io/name: pmem-csi-controller + app.kubernetes.io/instance: pmem-csi.intel.com + ports: + - targetPort: 8000 + port: 443 diff --git a/deploy/kustomize/webhook/webhook.yaml b/deploy/kustomize/webhook/webhook.yaml index ae61ac8d37..ab224da110 100644 --- a/deploy/kustomize/webhook/webhook.yaml +++ b/deploy/kustomize/webhook/webhook.yaml @@ -24,7 +24,7 @@ webhooks: admissionReviewVersions: ["v1"] clientConfig: service: - name: pmem-csi-intel-com-scheduler + name: pmem-csi-intel-com-webhook namespace: pmem-csi path: /pod/mutate caBundle: diff --git a/deploy/yamls.go b/deploy/yamls.go index cec2d6641d..ff95f47b4e 100644 --- a/deploy/yamls.go +++ b/deploy/yamls.go @@ -19,6 +19,7 @@ import ( //go:embed kubernetes-*/*/pmem-csi.yaml //go:embed kustomize/webhook/webhook.yaml //go:embed kustomize/scheduler/scheduler-service.yaml +//go:embed kustomize/webhook/webhook-service.yaml var assets embed.FS // YamlFile contains all objects of a certain deployment. diff --git a/docs/install.md b/docs/install.md index a6d8d4136a..6c17b46a32 100644 --- a/docs/install.md +++ b/docs/install.md @@ -319,7 +319,11 @@ for a complete list of supported properties. Here is a minimal example driver deployment created with a custom resource: **NOTE**: `nodeSelector` must match the node label that was set in the -[installation and setup](#installation-and-setup) section. +[installation and setup](#installation-and-setup) section. The PMEM-CSI +[scheduler extender](design.md#scheduler-extender) and +[webhook](design.md#pod-admission-webhook) are not enabled in this basic +installation. See [below](#enable-scheduler-extensions) for +instructions about that. ``` ShellSession $ kubectl create -f - <]:` parameter. The listen address is optional and can be left out. The port is where a -HTTPS server will run. +HTTPS server will run. The YAML files already enable this. The +operator has the `controllerTLSSecret` and `mutatePods` properties in +the [`DeploymentSpec`](#deploymentspec). The controller needs TLS certificates which must be created in advance. The YAML files expects them in a secret called @@ -1217,6 +1234,152 @@ EOF $ kubectl create --kustomize my-webhook ``` +#### OpenShift scheduler configuration + +**NOTE**: The scheduler extensions are only needed on OpenShift 4.6 +and 4.7. On OpenShift 4.8, [storage capacity +tracking](#storage-capacity-tracking) can and should be used instead. + +The operator should be used on OpenShift. When creating the +deployment, set `controllerTLSSecret` to the special string +`-openshift-`: + +``` ShellSession +$ kubectl create -f - <` state directory on each node. +Although the operator allows running multiple PMEM-CSI driver deployments, one +has to take extreme care of such deployments by ensuring that not more than +one driver ends up running on the same node(s). Nodes on which a PMEM-CSI +driver could run can be configured by using the `nodeSelector` property of +the `DeploymentSpec`. + **NOTE**: Starting from release v0.9.0 reconciling of the `Deployment` CRD in `pmem-csi.intel.com/v1alpha1` API group is not supported by the PMEM-CSI operator anymore. Such resources in the cluster must be migrated manually to new the `PmemCSIDeployment` API. -The current API for `PmemCSIDeployment` resources is: -### PmemCSIDeployment +The current API for `PmemCSIDeployment` resources is: |Field | Type | Description | |---|---|---| @@ -1425,7 +1593,7 @@ The current API for `PmemCSIDeployment` resources is: | metadata | [ObjectMeta](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata) | Object metadata, name used for CSI driver and as prefix for sub-objects | | spec | [DeploymentSpec](#deployment-spec) | Specification of the desired behavior of the deployment | -#### DeploymentSpec +### DeploymentSpec Below specification fields are valid in all API versions unless noted otherwise in the description. @@ -1442,7 +1610,7 @@ of the API specification. | logLevel | integer | PMEM-CSI driver logging level | 3 | | logFormat | text | log output format | "text" or "json" 3 | | deviceMode | string | Device management mode to use. Supports one of `lvm` or `direct` | `lvm` -| controllerTLSSecret | string | Name of an existing secret in the driver's namespace which contains ca.crt, tls.crt and tls.key data for the scheduler extender and pod mutation webhook. A controller is started if (and only if) this secret is specified. | empty +| controllerTLSSecret | string | Name of an existing secret in the driver's namespace which contains ca.crt, tls.crt and tls.key data for the scheduler extender and pod mutation webhook. A controller is started if (and only if) this secret is specified.
Alternatively, the special string `-openshift-` can be used on OpenShift to let OpenShift create the necessary secrets. | empty | mutatePods | Always/Try/Never | Defines how a mutating pod webhook is configured if a controller is started. The field is ignored if the controller is not enabled. "Never" disables pod mutation. "Try" configured it so that pod creation is allowed to proceed even when the webhook fails. "Always" requires that the webhook gets invoked successfully before creating a pod. | Try | schedulerNodePort | If non-zero, the scheduler service is created as a NodeService with that fixed port number. Otherwise that service is created as a cluster service. The number must be from the range reserved by Kubernetes for node ports. This is useful if the kube-scheduler cannot reach the scheduler extender via a cluster service. | 0 | controllerResources | [ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#resourcerequirements-v1-core) | Describes the compute resource requirements for controller pod.
4_Deprecated and only available in `v1alpha1`._ | @@ -1487,7 +1655,7 @@ propagated to the deployed driver, not all changes are safe. In particular, changing the `deviceMode` will not work when there are active volumes. -#### DeploymentStatus +### DeploymentStatus A PMEM-CSI Deployment's `status` field is a `DeploymentStatus` object, which carries the detailed state of the driver deployment. It is comprised of [deployment @@ -1505,7 +1673,7 @@ The possible `phase` values and their meaning are as below: 1 This check has not been implemented yet. Instead, the deployment goes straight to `Running` after creating sub-resources. -#### Deployment Conditions +### Deployment Conditions PMEM-CSI `DeploymentStatus` has an array of `conditions` through which the PMEM-CSI Deployment has or has not passed. Below are the possible condition @@ -1517,7 +1685,7 @@ types and their meanings: | CertsVerified | Verified that the provided certificates are valid. | | DriverDeployed | All the componentes required for the PMEM-CSI deployment have been deployed. | -#### Driver component status +### Driver component status PMEM-CSI `DeploymentStatus` has an array of `components` of type `DriverStatus` in which the operator records the brief driver components status. This is @@ -1531,22 +1699,13 @@ Below are the fields and their meanings of `DriverStatus`: | reason | A brief message that explains why the component is in this state. | | lastUpdateTime | Time at which the status updated. | -#### Deployment Events +### Deployment Events The PMEM-CSI operator posts events on the progress of a `PmemCSIDeployment`. If the deployment is in the `Failed` state, then one can look into the event(s) using `kubectl describe` on that deployment for the detailed failure reason. - -> **Note on multiple deployments** -> -> Though the operator allows running multiple PMEM-CSI driver deployments, one -> has to take extreme care of such deployments by ensuring that not more than -> one driver ends up running on the same node(s). Nodes on which a PMEM-CSI -> driver could run can be configured by using `nodeSelector` property of -> [`DeploymentSpec`](#pmem-csi-deployment-crd). - -#### Operator metrics data +### Operator metrics data PMEM-CSI operator exposes below metrics data about active PmemCSIDeployment custom resources and it's sub-object in addition to the diff --git a/go.mod b/go.mod index 9ac90c16ce..16f75eed69 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( gopkg.in/freddierice/go-losetup.v1 v1.0.0-20170407175016-fc9adea44124 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.21.1 - k8s.io/apiextensions-apiserver v0.21.1 + k8s.io/apiextensions-apiserver v0.21.2 k8s.io/apimachinery v0.21.1 k8s.io/client-go v12.0.0+incompatible k8s.io/component-base v0.21.1 diff --git a/pkg/apis/pmemcsi/v1beta1/deployment_types.go b/pkg/apis/pmemcsi/v1beta1/deployment_types.go index 680a8ad8be..3db344a943 100644 --- a/pkg/apis/pmemcsi/v1beta1/deployment_types.go +++ b/pkg/apis/pmemcsi/v1beta1/deployment_types.go @@ -74,6 +74,14 @@ const ( MutatePodsNever MutatePods = "Never" ) +const ( + // ControllerTLSSecretOpenshift is a special string which + // enables the usage of + // https://docs.openshift.com/container-platform/4.6/security/certificates/service-serving-certificate.html + // to create certificates. + ControllerTLSSecretOpenshift = "-openshift-" +) + // +k8s:deepcopy-gen=true // DeploymentSpec defines the desired state of Deployment type DeploymentSpec struct { @@ -97,7 +105,9 @@ type DeploymentSpec struct { ControllerDriverResources *corev1.ResourceRequirements `json:"controllerDriverResources,omitempty"` // ControllerTLSSecret is the name of a secret which contains ca.crt, tls.crt and tls.key data // for the scheduler extender and pod mutation webhook. A controller is started if (and only if) - // this secret is specified. + // this secret is specified. The special string "-openshift-" enables the usage of + // https://docs.openshift.com/container-platform/4.6/security/certificates/service-serving-certificate.html + // to create certificates. ControllerTLSSecret string `json:"controllerTLSSecret,omitempty"` // MutatePod defines how a mutating pod webhook is configured if a controller // is started. The field is ignored if the controller is not enabled. @@ -468,6 +478,12 @@ func (d *PmemCSIDeployment) GetHyphenedName() string { return strings.ReplaceAll(d.GetName(), ".", "-") } +// ControllerTLSSecretOpenshiftName returns the name of the secret that +// we want OpenShift to create for the controller service. +func (d *PmemCSIDeployment) ControllerTLSSecretOpenshiftName() string { + return d.GetHyphenedName() + "-openshift-controller-tls" +} + // RegistrySecretName returns the name of the registry // Secret object used by the deployment func (d *PmemCSIDeployment) RegistrySecretName() string { @@ -492,12 +508,18 @@ func (d *PmemCSIDeployment) MetricsServiceName() string { return d.GetHyphenedName() + "-metrics" } -// SchedulerServiceName returns the name of the controller's scheduler -// Service object +// SchedulerServiceName returns the name of the controller's +// Service object for the scheduler extender. func (d *PmemCSIDeployment) SchedulerServiceName() string { return d.GetHyphenedName() + "-scheduler" } +// SchedulerServiceName returns the name of the controller's +// Service object for the webhooks. +func (d *PmemCSIDeployment) WebhooksServiceName() string { + return d.GetHyphenedName() + "-webhook" +} + // WebhooksServiceAccountName returns the name of the service account // used by the StatefulSet with the webhooks. func (d *PmemCSIDeployment) WebhooksServiceAccountName() string { diff --git a/pkg/deployments/load.go b/pkg/deployments/load.go index fc3b5be344..58ca64ecd0 100644 --- a/pkg/deployments/load.go +++ b/pkg/deployments/load.go @@ -92,13 +92,7 @@ func LoadAndCustomizeObjects(kubernetes version.Version, deviceMode api.DeviceMo } enabled := func(obj *unstructured.Unstructured) bool { - // The controller is always enabled, but the mutating webhook depends on the spec. - switch obj.GetKind() + "/" + obj.GetName() { - case "MutatingWebhookConfiguration/" + deployment.MutatingWebhookName(): - return deployment.Spec.ControllerTLSSecret != "" && deployment.Spec.MutatePods != api.MutatePodsNever - default: - return true - } + return true } patchUnstructured := func(obj *unstructured.Unstructured) { @@ -166,14 +160,32 @@ func LoadAndCustomizeObjects(kubernetes version.Version, deviceMode api.DeviceMo clientConfig["caBundle"] = base64.StdEncoding.EncodeToString(controllerCABundle) } + if deployment.Spec.ControllerTLSSecret == api.ControllerTLSSecretOpenshift { + meta := obj.Object["metadata"].(map[string]interface{}) + meta["annotations"] = map[string]string{ + "service.beta.openshift.io/inject-cabundle": "true", + } + } case "Service": switch obj.GetName() { case deployment.SchedulerServiceName(): + spec := obj.Object["spec"].(map[string]interface{}) + ports := spec["ports"].([]interface{}) + port0 := ports[0].(map[string]interface{}) if deployment.Spec.SchedulerNodePort != 0 { - spec := obj.Object["spec"].(map[string]interface{}) spec["type"] = "NodePort" - ports := spec["ports"].([]interface{}) - ports[0].(map[string]interface{})["nodePort"] = deployment.Spec.SchedulerNodePort + port0["nodePort"] = deployment.Spec.SchedulerNodePort + } + if deployment.Spec.ControllerTLSSecret == api.ControllerTLSSecretOpenshift { + port0["targetPort"] = 8001 + port0["port"] = 80 + } + case deployment.WebhooksServiceName(): + if deployment.Spec.ControllerTLSSecret == api.ControllerTLSSecretOpenshift { + meta := obj.Object["metadata"].(map[string]interface{}) + meta["annotations"] = map[string]string{ + "service.beta.openshift.io/serving-cert-secret-name": deployment.ControllerTLSSecretOpenshiftName(), + } } } } @@ -190,12 +202,17 @@ func LoadAndCustomizeObjects(kubernetes version.Version, deviceMode api.DeviceMo } objects = append(objects, scheduler...) - if deployment.Spec.MutatePods != api.MutatePodsNever { + if deployment.Spec.ControllerTLSSecret != "" && deployment.Spec.MutatePods != api.MutatePodsNever { webhook, err := loadYAML("kustomize/webhook/webhook.yaml", patchYAML, enabled, patchUnstructured) if err != nil { return nil, err } objects = append(objects, webhook...) + service, err := loadYAML("kustomize/webhook/webhook-service.yaml", patchYAML, enabled, patchUnstructured) + if err != nil { + return nil, err + } + objects = append(objects, service...) } return objects, nil @@ -207,9 +224,9 @@ func patchPodTemplate(obj *unstructured.Unstructured, deployment api.PmemCSIDepl spec := template["spec"].(map[string]interface{}) metadata := template["metadata"].(map[string]interface{}) - // isController := strings.Contains(obj.Object["metadata"].(map[string]interface{})["name"].(string), "controller") isController := strings.Contains(obj.GetName(), "controller") stripTLS := isController && deployment.Spec.ControllerTLSSecret == "" + openshiftTLS := isController && deployment.Spec.ControllerTLSSecret == api.ControllerTLSSecretOpenshift if deployment.Spec.Labels != nil { labels := metadata["labels"] @@ -260,11 +277,11 @@ func patchPodTemplate(obj *unstructured.Unstructured, deployment api.PmemCSIDepl container["volumeMounts"] = nil var command []interface{} for _, arg := range container["command"].([]interface{}) { - switch arg.(string) { - case "-caFile=/certs/ca.crt", - "-certFile=/certs/tls.crt", - "-keyFile=/certs/tls.key", - "-schedulerListen=:8000": + switch strings.Split(arg.(string), "=")[0] { + case "-caFile", + "-certFile", + "-keyFile", + "-schedulerListen": // remove these parameters default: command = append(command, arg) @@ -273,6 +290,19 @@ func patchPodTemplate(obj *unstructured.Unstructured, deployment api.PmemCSIDepl container["command"] = command } + if openshiftTLS && container["name"].(string) == "pmem-driver" { + var command []interface{} + for _, arg := range container["command"].([]interface{}) { + switch arg.(string) { + case "-schedulerListen=:8000": + command = append(command, arg, "-insecureSchedulerListen=:8001") + default: + command = append(command, arg) + } + } + container["command"] = command + } + // Override driver name in env var. env := container["env"] if env != nil { @@ -313,7 +343,11 @@ func patchPodTemplate(obj *unstructured.Unstructured, deployment api.PmemCSIDepl volume := volume.(map[string]interface{}) volumeName := volume["name"].(string) if volumeName == "webhook-cert" { - volume["secret"].(map[string]interface{})["secretName"] = deployment.Spec.ControllerTLSSecret + name := deployment.Spec.ControllerTLSSecret + if name == api.ControllerTLSSecretOpenshift { + name = deployment.ControllerTLSSecretOpenshiftName() + } + volume["secret"].(map[string]interface{})["secretName"] = name } } } diff --git a/pkg/k8sutil/client.go b/pkg/k8sutil/client.go index 691d33afd7..d95878f572 100644 --- a/pkg/k8sutil/client.go +++ b/pkg/k8sutil/client.go @@ -7,12 +7,16 @@ SPDX-License-Identifier: Apache-2.0 package k8sutil import ( + "context" "fmt" "os" "regexp" "strconv" "github.com/intel/pmem-csi/pkg/version" + apiclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/discovery" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -67,3 +71,19 @@ func GetKubernetesVersion(cfg *rest.Config) (*version.Version, error) { v := version.NewVersion(uint(major), uint(minor)) return &v, nil } + +// IsOpenShift determines whether the cluster is based on OpenShift. +func IsOpenShift(cfg *rest.Config) (bool, error) { + client, err := apiclient.NewForConfig(cfg) + if err != nil { + return false, err + } + // For our purposed we run on OpenShift if the scheduler operator is installed. + if _, err := client.ApiextensionsV1().CustomResourceDefinitions().Get(context.Background(), "schedulers.config.openshift.io", metav1.GetOptions{}); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, fmt.Errorf("check for OpenShift CRD: %v", err) + } + return true, nil +} diff --git a/pkg/pmem-csi-operator/controller/deployment/controller_driver.go b/pkg/pmem-csi-operator/controller/deployment/controller_driver.go index 63fe183e0b..81a33b5bd0 100644 --- a/pkg/pmem-csi-operator/controller/deployment/controller_driver.go +++ b/pkg/pmem-csi-operator/controller/deployment/controller_driver.go @@ -10,6 +10,7 @@ import ( "context" "fmt" "reflect" + "strings" api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" pmemlog "github.com/intel/pmem-csi/pkg/logger" @@ -38,6 +39,7 @@ const ( nodeMetricsPort = 10010 provisionerMetricsPort = 10011 schedulerPort = 8000 + insecureSchedulerPort = 8001 ) func typeMeta(gv schema.GroupVersion, kind string) metav1.TypeMeta { @@ -96,6 +98,15 @@ func cloneObject(from client.Object) (client.Object, error) { } } +func isNamespaced(kind string) bool { + switch kind { + case "ClusterRole", "ClusterRoleBinding", "CSIDriver", "MutatingWebhookConfiguration": + return false + default: + return true + } +} + // CurrentObjects returns the active sub-object types used by the operator // for a driver deployment. func CurrentObjects() []client.Object { @@ -482,6 +493,20 @@ var subObjectHandlers = map[string]redeployObject{ return nil }, }, + "webhooks service": { + objType: reflect.TypeOf(&corev1.Service{}), + enabled: mutatingWebhookEnabled, + object: func(d *pmemCSIDeployment) client.Object { + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: d.getObjectMeta(d.WebhooksServiceName(), false), + } + }, + modify: func(d *pmemCSIDeployment, o client.Object) error { + d.getWebhooksService(o.(*corev1.Service)) + return nil + }, + }, "mutating webhook configuration": { objType: reflect.TypeOf(&admissionregistrationv1.MutatingWebhookConfiguration{}), enabled: mutatingWebhookEnabled, @@ -711,8 +736,11 @@ func (d *pmemCSIDeployment) deleteObsoleteObjects(ctx context.Context, r *Reconc } for _, list := range AllObjectLists() { - opts := &client.ListOptions{ - Namespace: d.namespace, + opts := &client.ListOptions{} + // The namespace is ignored by the real API server, but the fake client + // fails to return cluster-scoped objects when the namespace is set. + if isNamespaced(strings.TrimSuffix(list.GroupVersionKind().Kind, "List")) { + opts.Namespace = d.namespace } l.V(5).Info("fetching objects", "gkv", list.GetObjectKind(), "options", opts.Namespace) @@ -852,6 +880,21 @@ func (d *pmemCSIDeployment) getWebhooksClusterRoleBinding(crb *rbacv1.ClusterRol } } +func (d *pmemCSIDeployment) getWebhooksService(service *corev1.Service) { + d.getService(service, corev1.ServiceTypeClusterIP, 443) + service.Spec.Ports[0].TargetPort.IntVal = schedulerPort + if d.Spec.ControllerTLSSecret == api.ControllerTLSSecretOpenshift { + if service.Annotations == nil { + service.Annotations = map[string]string{} + } + service.Annotations["service.beta.openshift.io/serving-cert-secret-name"] = d.ControllerTLSSecretOpenshiftName() + } else { + if service.Annotations != nil { + delete(service.Annotations, "service.beta.openshift.io/serving-cert-secret-name") + } + } +} + func (d *pmemCSIDeployment) getMutatingWebhookConfig(hook *admissionregistrationv1.MutatingWebhookConfiguration) { servicePort := int32(443) // default webhook service port selector := &metav1.LabelSelector{ @@ -869,26 +912,42 @@ func (d *pmemCSIDeployment) getMutatingWebhookConfig(hook *admissionregistration } path := "/pod/mutate" none := admissionregistrationv1.SideEffectClassNone - if len(d.controllerCABundle) == 0 { - panic("controller CA bundle empty, should have been loaded") + controllerCABundle := d.controllerCABundle + // Preserve defaults when updating. + var scope *admissionregistrationv1.ScopeType + var timeoutSeconds *int32 + var matchPolicy *admissionregistrationv1.MatchPolicyType + var reinvocationPolicy *admissionregistrationv1.ReinvocationPolicyType + if len(hook.Webhooks) > 0 { + if d.Spec.ControllerTLSSecret == api.ControllerTLSSecretOpenshift { + // Below we overwrite the entire hook.Webhooks. Before we do that, we must + // retrieve the CABundle that was generated for us by OpenShift. + controllerCABundle = hook.Webhooks[0].ClientConfig.CABundle + } + scope = hook.Webhooks[0].Rules[0].Scope + timeoutSeconds = hook.Webhooks[0].TimeoutSeconds + matchPolicy = hook.Webhooks[0].MatchPolicy + reinvocationPolicy = hook.Webhooks[0].ReinvocationPolicy } hook.Webhooks = []admissionregistrationv1.MutatingWebhook{ { // Name must be "fully-qualified" (i.e. with domain) but not unique, so // here "pmem-csi.intel.com" is not the default driver name. // https://pkg.go.dev/k8s.io/api/admissionregistration/v1#MutatingWebhook - Name: "pod-hook.pmem-csi.intel.com", - NamespaceSelector: selector, - ObjectSelector: selector, - FailurePolicy: &failurePolicy, + Name: "pod-hook.pmem-csi.intel.com", + NamespaceSelector: selector, + ObjectSelector: selector, + FailurePolicy: &failurePolicy, + MatchPolicy: matchPolicy, + ReinvocationPolicy: reinvocationPolicy, ClientConfig: admissionregistrationv1.WebhookClientConfig{ Service: &admissionregistrationv1.ServiceReference{ - Name: d.SchedulerServiceName(), + Name: d.WebhooksServiceName(), Namespace: d.namespace, Path: &path, Port: &servicePort, }, - CABundle: d.controllerCABundle, // loaded earlier in reconcile() + // CABundle set below. }, Rules: []admissionregistrationv1.RuleWithOperations{ { @@ -897,18 +956,44 @@ func (d *pmemCSIDeployment) getMutatingWebhookConfig(hook *admissionregistration APIGroups: []string{""}, APIVersions: []string{"v1"}, Resources: []string{"pods"}, + Scope: scope, }, }, }, SideEffects: &none, AdmissionReviewVersions: []string{"v1"}, + TimeoutSeconds: timeoutSeconds, }, } + + switch { + case d.Spec.ControllerTLSSecret == api.ControllerTLSSecretOpenshift: + if hook.Annotations == nil { + hook.Annotations = map[string]string{} + } + hook.Annotations["service.beta.openshift.io/inject-cabundle"] = "true" + case len(controllerCABundle) == 0: + panic("controller CA bundle empty, should have been loaded") + default: + if hook.Annotations != nil { + delete(hook.Annotations, "service.beta.openshift.io/inject-cabundle") + } + } + // Set or preserve the CABundle. + hook.Webhooks[0].ClientConfig.CABundle = controllerCABundle } func (d *pmemCSIDeployment) getSchedulerService(service *corev1.Service) { - d.getService(service, corev1.ServiceTypeClusterIP, 443) - service.Spec.Ports[0].TargetPort.IntVal = schedulerPort + targetPort := schedulerPort + port := 443 + if d.Spec.ControllerTLSSecret == api.ControllerTLSSecretOpenshift { + targetPort = insecureSchedulerPort + port = 80 + } + d.getService(service, corev1.ServiceTypeClusterIP, int32(port)) + service.Spec.Ports[0].TargetPort = intstr.IntOrString{ + IntVal: int32(targetPort), + } service.Spec.Ports[0].NodePort = d.Spec.SchedulerNodePort if d.Spec.SchedulerNodePort != 0 { service.Spec.Type = corev1.ServiceTypeNodePort @@ -1116,11 +1201,15 @@ func (d *pmemCSIDeployment) getControllerStatefulSet(ss *appsv1.StatefulSet) { ss.Spec.Template.Spec.Volumes = []corev1.Volume{} if d.Spec.ControllerTLSSecret != "" { mode := corev1.SecretVolumeSourceDefaultMode + name := d.Spec.ControllerTLSSecret + if name == api.ControllerTLSSecretOpenshift { + name = d.ControllerTLSSecretOpenshiftName() + } ss.Spec.Template.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{ Name: "webhook-cert", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: d.Spec.ControllerTLSSecret, + SecretName: name, DefaultMode: &mode, }, }, @@ -1270,13 +1359,17 @@ func (d *pmemCSIDeployment) getControllerCommand() []string { if d.Spec.ControllerTLSSecret != "" { args = append(args, - "-caFile=/certs/ca.crt", + "-caFile=", "-certFile=/certs/tls.crt", "-keyFile=/certs/tls.key", fmt.Sprintf("-schedulerListen=:%d", schedulerPort), ) + if d.Spec.ControllerTLSSecret == api.ControllerTLSSecretOpenshift { + args = append(args, + fmt.Sprintf("-insecureSchedulerListen=:%d", insecureSchedulerPort), + ) + } } - args = append(args, fmt.Sprintf("-metricsListen=:%d", controllerMetricsPort)) return args diff --git a/pkg/pmem-csi-operator/controller/deployment/deployment_controller.go b/pkg/pmem-csi-operator/controller/deployment/deployment_controller.go index febc926667..02d9b2f404 100644 --- a/pkg/pmem-csi-operator/controller/deployment/deployment_controller.go +++ b/pkg/pmem-csi-operator/controller/deployment/deployment_controller.go @@ -448,7 +448,13 @@ func (r *ReconcileDeployment) newDeployment(ctx context.Context, deployment *api k8sVersion: r.k8sVersion, } - if d.Spec.ControllerTLSSecret != "" { + switch d.Spec.ControllerTLSSecret { + case "": + // Nothing to do. + case api.ControllerTLSSecretOpenshift: + // Nothing to load, we just add annotations. + default: + // Load the specified secret. secret := &corev1.Secret{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", diff --git a/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go b/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go index 9e5df60f95..b107a1a94b 100644 --- a/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go +++ b/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go @@ -435,6 +435,10 @@ func TestDeploymentController(t *testing.T) { schedulerNodePort: 31000, objects: []runtime.Object{createSecret("controller-secret", testNamespace, dataOkay)}, }, + "openshift": { + name: "test-controller", + controllerTLSSecret: "-openshift-", + }, } for name, d := range cases { diff --git a/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go b/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go index 5e29efeb20..ea422c665c 100644 --- a/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go +++ b/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go @@ -99,6 +99,9 @@ func UpdateTests() []UpdateTest { "kubeletDir": func(d *api.PmemCSIDeployment) { d.Spec.KubeletDir = "/foo/bar" }, + "openshift": func(d *api.PmemCSIDeployment) { + d.Spec.ControllerTLSSecret = "-openshift-" + }, } full := api.PmemCSIDeployment{ @@ -195,5 +198,22 @@ func UpdateTests() []UpdateTest { }) } + // Special case: remove -openshift- + openshiftDep := api.PmemCSIDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pmem-csi-for-openshift", + }, + Spec: api.DeploymentSpec{ + ControllerTLSSecret: "-openshift-", + }, + } + tests = append(tests, UpdateTest{ + Name: "remove-openshift", + Deployment: openshiftDep, + Mutate: func(d *api.PmemCSIDeployment) { + d.Spec.ControllerTLSSecret = "" + }, + }) + return tests } diff --git a/runtime-deps.csv b/runtime-deps.csv index 7144eaba82..ad0bb25d5e 100644 --- a/runtime-deps.csv +++ b/runtime-deps.csv @@ -47,6 +47,7 @@ gopkg.in/inf.v0,BSD-3-Clause gopkg.in/yaml.v2,Apache-2.0 gopkg.in/yaml.v3,MIT k8s.io/api,Apache-2.0 +k8s.io/apiextensions-apiserver/pkg,Apache-2.0 k8s.io/apimachinery,Apache-2.0 k8s.io/client-go,Apache-2.0 k8s.io/component-base,Apache-2.0 diff --git a/test/e2e/deploy/cluster.go b/test/e2e/deploy/cluster.go index ec72f9ae11..6b648d0019 100644 --- a/test/e2e/deploy/cluster.go +++ b/test/e2e/deploy/cluster.go @@ -21,6 +21,8 @@ import ( "k8s.io/client-go/rest" e2essh "k8s.io/kubernetes/test/e2e/framework/ssh" + "github.com/intel/pmem-csi/pkg/k8sutil" + "github.com/intel/pmem-csi/pkg/version" . "github.com/onsi/gomega" ) @@ -29,6 +31,9 @@ type Cluster struct { cs kubernetes.Interface dc dynamic.Interface cfg *rest.Config + + version *version.Version + isOpenShift bool } func NewCluster(cs kubernetes.Interface, dc dynamic.Interface, cfg *rest.Config) (*Cluster, error) { @@ -49,6 +54,16 @@ func NewCluster(cs kubernetes.Interface, dc dynamic.Interface, cfg *rest.Config) host := strings.Split(sshHost, ":")[0] // Instead of duplicating the NodeSSHHosts logic we simply strip the ssh port. cluster.nodeIPs = append(cluster.nodeIPs, host) } + version, err := k8sutil.GetKubernetesVersion(cfg) + if err != nil { + return nil, err + } + cluster.version = version + isOpenShift, err := k8sutil.IsOpenShift(cfg) + if err != nil { + return nil, err + } + cluster.isOpenShift = isOpenShift return cluster, nil } @@ -142,3 +157,9 @@ func (c *Cluster) WaitForDaemonSet(setName, namespace string) *appsv1.DaemonSet func (c *Cluster) GetStatefulSet(ctx context.Context, setName, namespace string) (*appsv1.StatefulSet, error) { return c.cs.AppsV1().StatefulSets(namespace).Get(ctx, setName, metav1.GetOptions{}) } + +// StorageCapacitySupported checks that the v1beta1 CSIStorageCapacity API is supported. +// It only checks the Kubernetes version. +func (c *Cluster) StorageCapacitySupported() bool { + return c.version.Compare(1, 21) >= 0 +} diff --git a/test/e2e/deploy/deploy.go b/test/e2e/deploy/deploy.go index 91167a19bf..82d81511ef 100644 --- a/test/e2e/deploy/deploy.go +++ b/test/e2e/deploy/deploy.go @@ -31,14 +31,12 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/rest" "k8s.io/klog/v2/klogr" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/skipper" api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" pmemexec "github.com/intel/pmem-csi/pkg/exec" - "github.com/intel/pmem-csi/pkg/k8sutil" pmemlog "github.com/intel/pmem-csi/pkg/logger" "github.com/intel/pmem-csi/test/e2e/pod" testconfig "github.com/intel/pmem-csi/test/test-config" @@ -227,7 +225,7 @@ func WaitForPMEMDriver(c *Cluster, d *Deployment) (metricsURL string) { if actualNodes != c.NumNodes()-1 { return fmt.Errorf("only %d of %d nodes have registered", actualNodes, c.NumNodes()-1) } - case d.Version == "0.9" || !d.StorageCapacitySupported(c.cfg): + case d.Version == "0.9" || !c.StorageCapacitySupported(): // It is possible that we just // (re)configured the mutating pod // webhook. We must be sure that @@ -768,14 +766,6 @@ func (d Deployment) Label() string { return d.Name() } -// StorageCapacitySupported checks that the v1beta1 CSIStorageCapacity API is supported. -// It only checks the Kubernetes version. -func (d *Deployment) StorageCapacitySupported(cfg *rest.Config) bool { - ver, err := k8sutil.GetKubernetesVersion(cfg) - framework.ExpectNoError(err, "check Kubernetes version") - return ver.Compare(1, 21) >= 0 -} - // FindDeployment checks whether there is a PMEM-CSI driver and/or // operator deployment in the cluster. A deployment is found via its // deployment resp. statefulset object, which must have a @@ -1164,7 +1154,7 @@ func EnsureDeploymentNow(f *framework.Framework, deployment *Deployment) { if deployment.HasDriver { if deployment.HasOperator { // Deploy driver through operator. - dep := deployment.GetDriverDeployment() + dep := deployment.GetDriverDeployment(c) EnsureDeploymentCR(f, dep) } else { // Deploy with script. @@ -1209,7 +1199,7 @@ func StopOperator(d *Deployment) error { // GetDriverDeployment returns the spec for the driver deployment that is used // for deployments like operator-lvm-production. -func (d *Deployment) GetDriverDeployment() api.PmemCSIDeployment { +func (d *Deployment) GetDriverDeployment(c *Cluster) api.PmemCSIDeployment { dep := api.PmemCSIDeployment{ // TypeMeta is needed because // DefaultUnstructuredConverter does not add it for us. Is there a better way? @@ -1239,11 +1229,18 @@ func (d *Deployment) GetDriverDeployment() api.PmemCSIDeployment { }, } - if d.HasController { - // The controller is enabled, using a secret that must have - // been prepared beforehand. - dep.Spec.ControllerTLSSecret = strings.ReplaceAll(d.DriverName, ".", "-") + "-controller-secret" + if d.HasController && !c.StorageCapacitySupported() { dep.Spec.MutatePods = api.MutatePodsAlways + if c.isOpenShift { + // Use certificates prepared by OpenShift. + dep.Spec.ControllerTLSSecret = api.ControllerTLSSecretOpenshift + } else { + // Use a secret that must have been prepared beforehand. + dep.Spec.ControllerTLSSecret = strings.ReplaceAll(d.DriverName, ".", "-") + "-controller-secret" + } + // The scheduler must have been configured manually. We just + // create the corresponding service in the namespace where the + // driver is going to run. portStr := testconfig.GetOrFail("TEST_SCHEDULER_EXTENDER_NODE_PORT") port, err := strconv.ParseInt(portStr, 10, 32) if err != nil { diff --git a/test/e2e/operator/driver.go b/test/e2e/operator/driver.go index 8811706e31..0d369526a9 100644 --- a/test/e2e/operator/driver.go +++ b/test/e2e/operator/driver.go @@ -36,10 +36,13 @@ var _ = deploy.DescribeForSome("driver", func(d *deploy.Deployment) bool { f.SkipNamespaceCreation = true It("runs", func() { + c, err := deploy.NewCluster(f.ClientSet, f.DynamicClient, f.ClientConfig()) + framework.ExpectNoError(err, "create cluster object") + // Delete existing driver and start a new driver // so that this test results does not effected by the other // tests run using the same driver deployment. - deployment := d.GetDriverDeployment() + deployment := d.GetDriverDeployment(c) deploy.DeleteDeploymentCR(f, deployment.Name) deploy.CreateDeploymentCR(f, deployment) @@ -58,9 +61,6 @@ var _ = deploy.DescribeForSome("driver", func(d *deploy.Deployment) bool { k8sver, err := k8sutil.GetKubernetesVersion(f.ClientConfig()) framework.ExpectNoError(err, "get Kubernetes version") - c, err := deploy.NewCluster(f.ClientSet, f.DynamicClient, f.ClientConfig()) - framework.ExpectNoError(err, "new cluster") - metricsURL, err := deploy.GetOperatorMetricsURL(ctx, c, d) Expect(err).ShouldNot(HaveOccurred(), "get operator metrics URL") @@ -69,7 +69,7 @@ var _ = deploy.DescribeForSome("driver", func(d *deploy.Deployment) bool { }) // Just very minimal testing at the moment. - csiTestDriver := driver.New(d.Name(), d.GetDriverDeployment().Name, []string{""} /* only the default fs type */, nil) + csiTestDriver := driver.New(d.Name(), d.DriverName, []string{""} /* only the default fs type */, nil) var csiTestSuites = []func() storageframework.TestSuite{ dax.InitDaxTestSuite, } diff --git a/test/e2e/operator/validate/validate.go b/test/e2e/operator/validate/validate.go index 2bed5bff13..5f62287f0d 100644 --- a/test/e2e/operator/validate/validate.go +++ b/test/e2e/operator/validate/validate.go @@ -390,6 +390,7 @@ CSIDriver: MutatingWebhookConfiguration: webhooks: clientConfig: + caBundle: ignore # Can change, in particular when generated by OpenShift. service: port: 443 admissionReviewVersions: diff --git a/test/e2e/storage/scheduler/scheduler.go b/test/e2e/storage/scheduler/scheduler.go index 54ce307567..975cde47be 100644 --- a/test/e2e/storage/scheduler/scheduler.go +++ b/test/e2e/storage/scheduler/scheduler.go @@ -92,9 +92,7 @@ func (p *schedulerTestSuite) DefineTests(driver storageframework.TestDriver, pat c, err := deploy.NewCluster(f.ClientSet, f.DynamicClient, f.ClientConfig()) framework.ExpectNoError(err, "create cluster object") - d, err := deploy.FindDeployment(c) - framework.ExpectNoError(err, "find PMEM-CSI deployment") - if d.StorageCapacitySupported(f.ClientConfig()) { + if c.StorageCapacitySupported() { e2eskipper.Skipf("storage capacity tracking is enabled, not using scheduler extensions") } diff --git a/test/setup-ca.sh b/test/setup-ca.sh index a3bcbe37b6..102d62278b 100755 --- a/test/setup-ca.sh +++ b/test/setup-ca.sh @@ -51,6 +51,10 @@ for name in ${CNS}; do # the version starting with 0.9.0 for the sake of consistency with # the pmem-csi.intel.com driver name. echo '"127.0.0.1",' + # mutating pod webhook + echo '"pmem-csi-webhook", "pmem-csi-webhook.'$NS'", "pmem-csi-webhook.'$NS'.svc",' + echo '"'$PREFIX'-webhook", "'$PREFIX'-webhook.'$NS'", "'$PREFIX'-webhook.'$NS'.svc",' + # scheduler extender echo '"pmem-csi-scheduler", "pmem-csi-scheduler.'$NS'", "pmem-csi-scheduler.'$NS'.svc",' echo '"'$PREFIX'-scheduler", "'$PREFIX'-scheduler.'$NS'", "'$PREFIX'-scheduler.'$NS'.svc",' # And for metrics server. diff --git a/test/setup-deployment.sh b/test/setup-deployment.sh index d3fd0c8a3f..cc27bd6bfb 100755 --- a/test/setup-deployment.sh +++ b/test/setup-deployment.sh @@ -40,10 +40,26 @@ echo "INFO: deploying from ${DEPLOYMENT_DIRECTORY}/${TEST_DEVICEMODE}${deploymen # Set up TLS secrets in the TEST_DRIVER_NAMESPACE. PATH="${REPO_DIRECTORY}/_work/bin:$PATH" KUBECTL="${KUBECTL}" ${TEST_DIRECTORY}/setup-ca-kubernetes.sh +OPENSHIFT_SCHEDULER=false case "$KUBERNETES_VERSION" in 1.19|1.20) # Enable scheduler extensions. Not needed on >= 1.21. DEPLOY+=(scheduler webhook) + if ${KUBECTL} get crd | grep -q schedulers.config.openshift.io; then + # Only the service gets deployed anew, with TEST_SCHEDULER_EXTENDER_NODE_PORT as + # well-known, fixed node port. Config map and scheduler config must be + # set up manually once before running tests. + OPENSHIFT_SCHEDULER=true + + # Do some sanity checkig... + policy=$(${KUBECTL} get scheduler/cluster -o jsonpath={.spec.policy.name}) + if [ ! "$policy" ]; then + echo >&2 "The scheduler config in scheduler/cluster must be set up manually. Currently there is no policy." + fi + if ! ${KUBECTL} get -n openshift-config configmap/$policy >/dev/null; then + echo >&2 "The scheduler policy configmap must be set up manually." + fi + fi ;; esac @@ -116,6 +132,17 @@ EOF path: /spec/template/spec/containers/0/command/- value: -nodeSelector={$(echo ${TEST_PMEM_NODE_LABEL} | sed -e 's/\([^=]*\)=\(.*\)/"\1":"\2"/')} EOF + if $OPENSHIFT_SCHEDULER; then + ${SSH} "cat >'$tmpdir/my-deployment/scheduler-patch.yaml'" <>'$tmpdir/my-deployment/scheduler-patch.yaml'" <>'$tmpdir/my-deployment/kustomization.yaml'" <'$tmpdir/my-deployment/webhook-patch.yaml'" <'$tmpdir/my-deployment/webhook-patch.yaml'" <>'$tmpdir/my-deployment/kustomization.yaml'" <'$tmpdir/my-deployment/webhook-service-patch.yaml'" <'$tmpdir/my-deployment/webhook-patch.yaml'" <