Secure metrics endpoint with cntrlr-runtime metrics authz mechanics (#3660)

anik120 · joelanford · web-flow · commit 589a5d0cc11e · 2025-10-10T12:04:33.000Z
Signed-off-by: Anik Bhattacharjee &lt;anbhatta@redhat.com&gt;
Co-authored-by: Joe Lanford &lt;joe.lanford@gmail.com&gt;
diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml
@@ -86,7 +86,7 @@ jobs:
           for i in $(seq 1 ${E2E_NODES}); do
             KIND_CLUSTER_NAME="kind-olmv0-${i}" \
             KIND_CREATE_OPTS="--kubeconfig=${E2E_KUBECONFIG_ROOT}/kubeconfig-${i}" \
-            HELM_INSTALL_OPTS="--kubeconfig ${E2E_KUBECONFIG_ROOT}/kubeconfig-${i}" \
+            HELM_INSTALL_OPTS="--kubeconfig ${E2E_KUBECONFIG_ROOT}/kubeconfig-${i} --set certManager.enabled=false" \
             make kind-create deploy;
           done
 
@@ -173,7 +173,7 @@ jobs:
           for i in $(seq 1 ${E2E_NODES}); do
             KIND_CLUSTER_NAME="kind-olmv0-${i}" \
             KIND_CREATE_OPTS="--kubeconfig=${E2E_KUBECONFIG_ROOT}/kubeconfig-${i}" \
-            HELM_INSTALL_OPTS="--kubeconfig ${E2E_KUBECONFIG_ROOT}/kubeconfig-${i}" \
+            HELM_INSTALL_OPTS="--kubeconfig ${E2E_KUBECONFIG_ROOT}/kubeconfig-${i} --set certManager.enabled=false" \
             make kind-create deploy;
           done
 
diff --git a/Makefile b/Makefile
@@ -48,6 +48,12 @@ GINKGO := $(TOOL_EXEC) github.com/onsi/ginkgo/v2/ginkgo
 
 # Target environment and Dependencies #
 
+# Cert-manager version - update this for new releases
+CERT_MANAGER_VERSION ?= v1.18.2
+
+# Cert-manager deployment timeout
+CERT_MANAGER_TIMEOUT ?= 120s
+
 # Minor Kubernetes version to build against derived from the client-go dependency version
 KUBE_MINOR ?= $(shell go list -m k8s.io/client-go | cut -d" " -f2 | sed 's/^v0\.\([[:digit:]]\{1,\}\)\.[[:digit:]]\{1,\}$$/1.\1/')
 
@@ -157,7 +163,29 @@ local-build: IMAGE_TAG = local
 local-build: image
 
 .PHONY: run-local
-run-local: local-build kind-create deploy
+run-local: local-build kind-create cert-manager-install deploy
+
+.PHONY: cert-manager-install
+cert-manager-install: #HELP Install cert-manager $(CERT_MANAGER_VERSION)
+	@echo "Installing cert-manager $(CERT_MANAGER_VERSION)"
+	kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.yaml
+	@echo "Waiting for cert-manager to be ready..."
+	kubectl wait --for=condition=Available --namespace=cert-manager deployment/cert-manager --timeout=$(CERT_MANAGER_TIMEOUT)
+	kubectl wait --for=condition=Available --namespace=cert-manager deployment/cert-manager-cainjector --timeout=$(CERT_MANAGER_TIMEOUT)
+	kubectl wait --for=condition=Available --namespace=cert-manager deployment/cert-manager-webhook --timeout=$(CERT_MANAGER_TIMEOUT)
+	@echo "Waiting for cert-manager webhook to be ready..."
+	kubectl wait --for=condition=Ready --namespace=cert-manager pod -l app=webhook --timeout=$(CERT_MANAGER_TIMEOUT)
+	@echo "Waiting for cert-manager CRDs to be available..."
+	kubectl wait --for condition=established --timeout=$(CERT_MANAGER_TIMEOUT) crd/certificates.cert-manager.io
+	kubectl wait --for condition=established --timeout=$(CERT_MANAGER_TIMEOUT) crd/issuers.cert-manager.io
+	@echo "cert-manager $(CERT_MANAGER_VERSION) installed successfully"
+
+.PHONY: cert-manager-uninstall
+cert-manager-uninstall: #HELP Uninstall cert-manager
+	@echo "Uninstalling cert-manager..."
+	kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.yaml --ignore-not-found=true
+	@echo "cert-manager uninstalled"
+
 
 .PHONY: clean
 clean: #HELP Clean up build artifacts
@@ -231,6 +259,7 @@ deploy: $(KIND) $(HELM) #HELP Deploy OLM to kind cluster $KIND_CLUSTER_NAME (def
 	$(KIND) load docker-image $(OLM_IMAGE) --name $(KIND_CLUSTER_NAME); \
 	$(HELM) upgrade --install olm deploy/chart \
 		--set debug=true \
+		--set certManager.enabled=true \
 		--set olm.image.ref=$(OLM_IMAGE) \
 		--set olm.image.pullPolicy=IfNotPresent \
 		--set catalog.image.ref=$(OLM_IMAGE) \
@@ -254,6 +283,9 @@ undeploy: $(KIND) $(HELM) #HELP Uninstall OLM from kind cluster $KIND_CLUSTER_NA
 	$(HELM) uninstall olm
 	kubectl delete -f deploy/chart/crds
 
+	# Uninstall cert-manager
+	$(MAKE) cert-manager-uninstall
+
 #SECTION e2e
 
 # E2E test configuration
@@ -269,7 +301,24 @@ e2e: #HELP Run e2e tests against a cluster running OLM (params: $E2E_TEST_NS (op
 	$(GO_TEST_ENV) $(GINKGO) -timeout $(E2E_TIMEOUT) $(GINKGO_OPTS) $(E2E_GINKGO_OPTS) ./test/e2e -- -namespace=$(E2E_TEST_NS) -olmNamespace=$(E2E_INSTALL_NS) -catalogNamespace=$(E2E_CATALOG_NS) $(E2E_OPTS)
 
 .PHONY: e2e-local
-e2e-local: e2e-build kind-create deploy e2e
+e2e-local: e2e-build kind-create e2e-local-deploy e2e
+
+.PHONY: e2e-local-deploy
+e2e-local-deploy: $(KIND) $(HELM) #HELP Deploy OLM for e2e testing (without cert-manager)
+	$(KIND) load docker-image $(OLM_IMAGE) --name $(KIND_CLUSTER_NAME); \
+	$(HELM) upgrade --install olm deploy/chart \
+		--set debug=true \
+		--set certManager.enabled=false \
+		--set olm.image.ref=$(OLM_IMAGE) \
+		--set olm.image.pullPolicy=IfNotPresent \
+		--set catalog.image.ref=$(OLM_IMAGE) \
+		--set catalog.image.pullPolicy=IfNotPresent \
+		--set catalog.commandArgs=--configmapServerImage=$(CONFIGMAP_SERVER_IMAGE) \
+		--set catalog.opmImageArgs=--opmImage=$(OPERATOR_REGISTRY_IMAGE) \
+		--set package.image.ref=$(OLM_IMAGE) \
+		--set package.image.pullPolicy=IfNotPresent \
+		$(HELM_INSTALL_OPTS) \
+		--wait;
 
 #SECTION Code Generation
 
diff --git a/cmd/catalog/main.go b/cmd/catalog/main.go
@@ -57,9 +57,16 @@ func (o *options) run(ctx context.Context, logger *logrus.Logger) error {
 		o.catalogNamespace = catalogNamespaceEnvVarValue
 	}
 
+	// create a config client for operator status
+	config, err := clientcmd.BuildConfigFromFlags("", o.kubeconfig)
+	if err != nil {
+		return fmt.Errorf("error configuring client: %s", err.Error())
+	}
+
 	listenAndServe, err := server.GetListenAndServeFunc(
 		server.WithLogger(logger),
 		server.WithTLS(&o.tlsCertPath, &o.tlsKeyPath, &o.clientCAPath),
+		server.WithKubeConfig(config),
 		server.WithDebug(o.debug),
 	)
 	if err != nil {
@@ -72,11 +79,6 @@ func (o *options) run(ctx context.Context, logger *logrus.Logger) error {
 		}
 	}()
 
-	// create a config client for operator status
-	config, err := clientcmd.BuildConfigFromFlags("", o.kubeconfig)
-	if err != nil {
-		return fmt.Errorf("error configuring client: %s", err.Error())
-	}
 	configClient, err := configv1client.NewForConfig(config)
 	if err != nil {
 		return fmt.Errorf("error configuring client: %s", err.Error())
diff --git a/cmd/olm/main.go b/cmd/olm/main.go
@@ -123,7 +123,18 @@ func main() {
 	}
 	logger.Infof("log level %s", logger.Level)
 
-	listenAndServe, err := server.GetListenAndServeFunc(server.WithLogger(logger), server.WithTLS(tlsCertPath, tlsKeyPath, clientCAPath), server.WithDebug(*debug))
+	mgr, err := Manager(ctx, *debug)
+	if err != nil {
+		logger.WithError(err).Fatal("error configuring controller manager")
+	}
+	config := mgr.GetConfig()
+
+	listenAndServe, err := server.GetListenAndServeFunc(
+		server.WithLogger(logger),
+		server.WithTLS(tlsCertPath, tlsKeyPath, clientCAPath),
+		server.WithKubeConfig(config),
+		server.WithDebug(*debug),
+	)
 	if err != nil {
 		logger.Fatalf("Error setting up health/metric/pprof service: %v", err)
 	}
@@ -134,12 +145,6 @@ func main() {
 		}
 	}()
 
-	mgr, err := Manager(ctx, *debug)
-	if err != nil {
-		logger.WithError(err).Fatal("error configuring controller manager")
-	}
-	config := mgr.GetConfig()
-
 	// create a config that validates we're creating objects with labels
 	validatingConfig := validatingroundtripper.Wrap(config, mgr.GetScheme())
 
diff --git a/deploy/chart/templates/0000_50_olm_04-cert-manager.yaml b/deploy/chart/templates/0000_50_olm_04-cert-manager.yaml
@@ -0,0 +1,46 @@
+{{- if .Values.certManager.enabled }}
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: {{ .Values.certManager.issuer.name }}
+  namespace: {{ .Values.namespace }}
+spec:
+  {{- if .Values.certManager.issuer.selfSigned }}
+  selfSigned: {}
+  {{- else if .Values.certManager.issuer.ca }}
+  ca:
+    secretName: {{ .Values.certManager.issuer.ca.secretName }}
+  {{- end }}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ .Values.certManager.certificate.name }}
+  namespace: {{ .Values.namespace }}
+spec:
+  secretName: {{ .Values.certManager.certificate.secretName }}
+  isCA: false
+  usages:
+    - server auth
+    - client auth
+  dnsNames:
+    - localhost
+    - catalog-operator.{{ .Values.namespace }}.svc
+    - catalog-operator.{{ .Values.namespace }}.svc.cluster.local
+    - olm-operator.{{ .Values.namespace }}.svc
+    - olm-operator.{{ .Values.namespace }}.svc.cluster.local
+    {{- range .Values.certManager.certificate.extraDnsNames }}
+    - {{ . }}
+    {{- end }}
+  ipAddresses:
+    - 127.0.0.1
+    {{- range .Values.certManager.certificate.extraIpAddresses }}
+    - {{ . }}
+    {{- end }}
+  issuerRef:
+    name: {{ .Values.certManager.issuer.name }}
+    kind: Issuer
+    group: cert-manager.io
+{{- end }}
+
diff --git a/deploy/chart/templates/0000_50_olm_07-olm-operator.deployment.yaml b/deploy/chart/templates/0000_50_olm_07-olm-operator.deployment.yaml
@@ -22,16 +22,14 @@ spec:
         seccompProfile:
           type: RuntimeDefault
       serviceAccountName: olm-operator-serviceaccount
-      volumes: 
-      {{- if .Values.olm.tlsSecret }}
+      volumes:
+      {{- if .Values.certManager.enabled }}
       - name: srv-cert
         secret:
-          secretName: {{ .Values.olm.tlsSecret }}
-      {{- end }}
-      {{- if .Values.olm.clientCASecret }}
+          secretName: {{ .Values.certManager.certificate.secretName }}
       - name: profile-collector-cert
         secret:
-          secretName: {{ .Values.olm.clientCASecret }}
+          secretName: {{ .Values.certManager.certificate.secretName }}
       {{- end }}
       - name: tmpfs
         emptyDir: {}
@@ -43,12 +41,10 @@ spec:
             capabilities:
               drop: [ "ALL" ]
           volumeMounts:
-          {{- if .Values.olm.tlsSecret }}
+          {{- if .Values.certManager.enabled }}
           - name: srv-cert
             mountPath: "/srv-cert"
             readOnly: true
-          {{- end }}
-          {{- if .Values.olm.clientCASecret }}
           - name: profile-collector-cert
             mountPath: "/profile-collector-cert"
             readOnly: true
@@ -78,31 +74,29 @@ spec:
           - --writePackageServerStatusName
           - {{ .Values.writePackageServerStatusName }}
           {{- end }}
-         {{- if .Values.olm.tlsSecret }}
+         {{- if .Values.certManager.enabled }}
           - --tls-cert
           - /srv-cert/tls.crt
           - --tls-key
           - /srv-cert/tls.key
-          {{- end }}
-          {{- if .Values.olm.clientCASecret }}
           - --client-ca
           - /profile-collector-cert/tls.crt
           {{- end }}
           image: {{ .Values.olm.image.ref }}
           imagePullPolicy: {{ .Values.olm.image.pullPolicy }}
           ports:
-            - containerPort: {{ .Values.olm.service.internalPort }}
+            - containerPort: {{ if .Values.certManager.enabled }}{{ .Values.olm.service.internalPortHttps }}{{ else }}{{ .Values.olm.service.internalPort }}{{ end }}
               name: metrics
           livenessProbe:
             httpGet:
               path: /healthz
-              port: {{ .Values.olm.service.internalPort }}
-              scheme: {{ if .Values.olm.tlsSecret }}HTTPS{{ else }}HTTP{{end}}
+              port: {{ if .Values.certManager.enabled }}{{ .Values.olm.service.internalPortHttps }}{{ else }}{{ .Values.olm.service.internalPort }}{{ end }}
+              scheme: {{ if .Values.certManager.enabled }}HTTPS{{ else }}HTTP{{ end }}
           readinessProbe:
             httpGet:
               path: /healthz
-              port: {{ .Values.olm.service.internalPort }}
-              scheme: {{ if .Values.olm.tlsSecret }}HTTPS{{ else }}HTTP{{end}}
+              port: {{ if .Values.certManager.enabled }}{{ .Values.olm.service.internalPortHttps }}{{ else }}{{ .Values.olm.service.internalPort }}{{ end }}
+              scheme: {{ if .Values.certManager.enabled }}HTTPS{{ else }}HTTP{{ end }}
           terminationMessagePolicy: FallbackToLogsOnError
           env:
           - name: OPERATOR_NAMESPACE
diff --git a/deploy/chart/templates/0000_50_olm_08-catalog-operator.deployment.yaml b/deploy/chart/templates/0000_50_olm_08-catalog-operator.deployment.yaml
@@ -23,15 +23,13 @@ spec:
           type: RuntimeDefault
       serviceAccountName: olm-operator-serviceaccount
       volumes:
-      {{- if .Values.catalog.tlsSecret }}
+      {{- if .Values.certManager.enabled }}
       - name: srv-cert
         secret:
-          secretName: {{ .Values.catalog.tlsSecret }}
-      {{- end }}
-      {{- if .Values.catalog.clientCASecret }}
+          secretName: {{ .Values.certManager.certificate.secretName }}
       - name: profile-collector-cert
         secret:
-          secretName: {{ .Values.catalog.clientCASecret }}
+          secretName: {{ .Values.certManager.certificate.secretName }}
       {{- end }}
       - name: tmpfs
         emptyDir: {}
@@ -43,12 +41,10 @@ spec:
             capabilities:
               drop: [ "ALL" ]
           volumeMounts:
-          {{- if .Values.catalog.tlsSecret }}
+          {{- if .Values.certManager.enabled }}
           - name: srv-cert
             mountPath: "/srv-cert"
             readOnly: true
-          {{- end }}
-          {{- if .Values.catalog.clientCASecret }}
           - name: profile-collector-cert
             mountPath: "/profile-collector-cert"
             readOnly: true
@@ -75,13 +71,11 @@ spec:
           - --writeStatusName
           - {{ .Values.writeStatusNameCatalog }}
           {{- end }}
-          {{- if .Values.catalog.tlsSecret }}
+          {{- if .Values.certManager.enabled }}
           - --tls-cert
           - /srv-cert/tls.crt
           - --tls-key
           - /srv-cert/tls.key
-          {{- end }}
-          {{- if .Values.catalog.clientCASecret }}
           - --client-ca
           - /profile-collector-cert/tls.crt
           {{- end }}
@@ -98,18 +92,18 @@ spec:
           {{- end }}
           imagePullPolicy: {{ .Values.catalog.image.pullPolicy }}
           ports:
-            - containerPort: {{ .Values.olm.service.internalPort }}
+            - containerPort: {{ if .Values.certManager.enabled }}{{ .Values.catalog.service.internalPortHttps }}{{ else }}{{ .Values.catalog.service.internalPort }}{{ end }}
               name: metrics
           livenessProbe:
             httpGet:
               path: /healthz
-              port: {{ .Values.catalog.service.internalPort }}
-              scheme: {{ if .Values.catalog.tlsSecret }}HTTPS{{ else }}HTTP{{end}}
+              port: {{ if .Values.certManager.enabled }}{{ .Values.catalog.service.internalPortHttps }}{{ else }}{{ .Values.catalog.service.internalPort }}{{ end }}
+              scheme: {{ if .Values.certManager.enabled }}HTTPS{{ else }}HTTP{{ end }}
           readinessProbe:
             httpGet:
               path: /healthz
-              port: {{ .Values.catalog.service.internalPort }}
-              scheme: {{ if .Values.catalog.tlsSecret }}HTTPS{{ else }}HTTP{{end}}
+              port: {{ if .Values.certManager.enabled }}{{ .Values.catalog.service.internalPortHttps }}{{ else }}{{ .Values.catalog.service.internalPort }}{{ end }}
+              scheme: {{ if .Values.certManager.enabled }}HTTPS{{ else }}HTTP{{ end }}
           terminationMessagePolicy: FallbackToLogsOnError
           {{- if .Values.catalog.resources }}
           resources:
diff --git a/deploy/chart/templates/_helpers.tpl b/deploy/chart/templates/_helpers.tpl
@@ -13,4 +13,4 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- define "fullname" -}}
 {{- $name := default .Chart.Name .Values.nameOverride -}}
 {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
+{{- end -}}
diff --git a/deploy/chart/values.yaml b/deploy/chart/values.yaml
diff --git a/pkg/lib/server/server.go b/pkg/lib/server/server.go
diff --git a/vendor/modules.txt b/vendor/modules.txt
diff --git a/vendor/sigs.k8s.io/controller-runtime/pkg/metrics/filters/filters.go b/vendor/sigs.k8s.io/controller-runtime/pkg/metrics/filters/filters.go
