feat(helm): add a ServiceMonitor for the controller metrics endpoint

helm/kagent/templates/_helpers.tpl

@@ -170,6 +170,15 @@ documented contract (see go/core/pkg/app/app.go).
 {{- if and .Values.controller.metrics.enabled $port (ne $port "0") -}}1{{- end -}}
 {{- end -}}
 
+{{/*
+Name of the controller metrics Service port, derived from the scheme the
+controller serves. Shared by the metrics Service and the ServiceMonitor
+endpoint so the two can never drift apart.
+*/}}
+{{- define "kagent.controller.metricsPortName" -}}
+{{- ternary "https" "http-metrics" .Values.controller.metrics.secureServing -}}
+{{- end -}}
+
 {{/*
 PostgreSQL service name for the bundled postgres instance
 */}}

helm/kagent/templates/controller-metrics-service.yaml

@@ -9,7 +9,7 @@ metadata:
 spec:
   type: {{ .Values.controller.metrics.service.type }}
   ports:
-    - name: {{ ternary "https" "http-metrics" .Values.controller.metrics.secureServing }}
+    - name: {{ include "kagent.controller.metricsPortName" . }}
       port: {{ .Values.controller.metrics.service.port }}
       targetPort: {{ include "kagent.controller.metricsPort" . | int }}
       protocol: TCP

helm/kagent/templates/controller-servicemonitor.yaml

@@ -0,0 +1,53 @@
+{{- if and (include "kagent.controller.metricsEnabled" .) .Values.controller.metrics.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "kagent.fullname" . }}-controller-metrics
+  namespace: {{ default (include "kagent.namespace" .) .Values.controller.metrics.serviceMonitor.namespace }}
+  labels:
+    {{- include "kagent.controller.labels" . | nindent 4 }}
+    {{- with .Values.controller.metrics.serviceMonitor.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- with .Values.controller.metrics.serviceMonitor.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "kagent.controller.selectorLabels" . | nindent 6 }}
+  namespaceSelector:
+    matchNames:
+      - {{ include "kagent.namespace" . }}
+  endpoints:
+    - port: {{ include "kagent.controller.metricsPortName" . }}
+      path: /metrics
+      {{- with .Values.controller.metrics.serviceMonitor.interval }}
+      interval: {{ . }}
+      {{- end }}
+      {{- with .Values.controller.metrics.serviceMonitor.scrapeTimeout }}
+      scrapeTimeout: {{ . }}
+      {{- end }}
+      honorLabels: {{ .Values.controller.metrics.serviceMonitor.honorLabels }}
+      {{- if .Values.controller.metrics.secureServing }}
+      scheme: https
+      {{- with .Values.controller.metrics.serviceMonitor.bearerTokenFile }}
+      bearerTokenFile: {{ . | quote }}
+      {{- end }}
+      tlsConfig:
+        {{- with .Values.controller.metrics.serviceMonitor.tlsConfig }}
+        {{- toYaml . | nindent 8 }}
+        {{- else }}
+        insecureSkipVerify: true
+        {{- end }}
+      {{- end }}
+      {{- with .Values.controller.metrics.serviceMonitor.relabelings }}
+      relabelings:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.controller.metrics.serviceMonitor.metricRelabelings }}
+      metricRelabelings:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

helm/kagent/tests/controller-servicemonitor_test.yaml

@@ -0,0 +1,224 @@
+suite: test controller service monitor
+templates:
+  - controller-servicemonitor.yaml
+tests:
+  - it: should not render by default
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: should not render when only metrics are enabled
+    set:
+      controller.metrics.enabled: true
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: should not render when only the service monitor is enabled
+    set:
+      controller.metrics.serviceMonitor.enabled: true
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: should not render when bindAddress disables metrics
+    set:
+      controller.metrics.enabled: true
+      controller.metrics.serviceMonitor.enabled: true
+      controller.metrics.bindAddress: "0"
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: should render when metrics and the service monitor are enabled
+    set:
+      controller.metrics.enabled: true
+      controller.metrics.serviceMonitor.enabled: true
+    asserts:
+      - isKind:
+          of: ServiceMonitor
+      - equal:
+          path: metadata.name
+          value: RELEASE-NAME-controller-metrics
+      - equal:
+          path: metadata.namespace
+          value: NAMESPACE
+      - hasDocuments:
+          count: 1
+
+  - it: should select the controller metrics service
+    set:
+      controller.metrics.enabled: true
+      controller.metrics.serviceMonitor.enabled: true
+    asserts:
+      - equal:
+          path: spec.selector.matchLabels["app.kubernetes.io/name"]
+          value: kagent
+      - equal:
+          path: spec.selector.matchLabels["app.kubernetes.io/instance"]
+          value: RELEASE-NAME
+      - equal:
+          path: spec.selector.matchLabels["app.kubernetes.io/component"]
+          value: controller
+      - equal:
+          path: spec.namespaceSelector.matchNames[0]
+          value: NAMESPACE
+
+  - it: should scrape the https port when secure serving is enabled
+    set:
+      controller.metrics.enabled: true
+      controller.metrics.serviceMonitor.enabled: true
+    asserts:
+      - equal:
+          path: spec.endpoints[0].port
+          value: https
+      - equal:
+          path: spec.endpoints[0].path
+          value: /metrics
+      - equal:
+          path: spec.endpoints[0].scheme
+          value: https
+      - equal:
+          path: spec.endpoints[0].bearerTokenFile
+          value: /var/run/secrets/kubernetes.io/serviceaccount/token
+      - equal:
+          path: spec.endpoints[0].tlsConfig.insecureSkipVerify
+          value: true
+
+  - it: should scrape the plaintext port when secure serving is disabled
+    set:
+      controller.metrics.enabled: true
+      controller.metrics.serviceMonitor.enabled: true
+      controller.metrics.secureServing: false
+    asserts:
+      - equal:
+          path: spec.endpoints[0].port
+          value: http-metrics
+      - notExists:
+          path: spec.endpoints[0].scheme
+      - notExists:
+          path: spec.endpoints[0].bearerTokenFile
+      - notExists:
+          path: spec.endpoints[0].tlsConfig
+
+  - it: should omit the bearer token file when cleared
+    set:
+      controller.metrics.enabled: true
+      controller.metrics.serviceMonitor.enabled: true
+      controller.metrics.serviceMonitor.bearerTokenFile: ""
+    asserts:
+      - notExists:
+          path: spec.endpoints[0].bearerTokenFile
+      - equal:
+          path: spec.endpoints[0].scheme
+          value: https
+
+  - it: should override the tls config
+    set:
+      controller.metrics.enabled: true
+      controller.metrics.serviceMonitor.enabled: true
+      controller.metrics.serviceMonitor.tlsConfig:
+        insecureSkipVerify: false
+        serverName: kagent-controller-metrics.kagent.svc
+    asserts:
+      - equal:
+          path: spec.endpoints[0].tlsConfig.insecureSkipVerify
+          value: false
+      - equal:
+          path: spec.endpoints[0].tlsConfig.serverName
+          value: kagent-controller-metrics.kagent.svc
+
+  - it: should omit optional scrape settings by default
+    set:
+      controller.metrics.enabled: true
+      controller.metrics.serviceMonitor.enabled: true
+    asserts:
+      - notExists:
+          path: spec.endpoints[0].interval
+      - notExists:
+          path: spec.endpoints[0].scrapeTimeout
+      - notExists:
+          path: spec.endpoints[0].relabelings
+      - notExists:
+          path: spec.endpoints[0].metricRelabelings
+      - equal:
+          path: spec.endpoints[0].honorLabels
+          value: false
+
+  - it: should apply scrape settings from values
+    set:
+      controller.metrics.enabled: true
+      controller.metrics.serviceMonitor.enabled: true
+      controller.metrics.serviceMonitor.interval: 30s
+      controller.metrics.serviceMonitor.scrapeTimeout: 10s
+      controller.metrics.serviceMonitor.honorLabels: true
+      controller.metrics.serviceMonitor.relabelings:
+        - action: replace
+          targetLabel: cluster
+          replacement: prod
+      controller.metrics.serviceMonitor.metricRelabelings:
+        - action: drop
+          sourceLabels:
+            - __name__
+          regex: go_.*
+    asserts:
+      - equal:
+          path: spec.endpoints[0].interval
+          value: 30s
+      - equal:
+          path: spec.endpoints[0].scrapeTimeout
+          value: 10s
+      - equal:
+          path: spec.endpoints[0].honorLabels
+          value: true
+      - equal:
+          path: spec.endpoints[0].relabelings[0].targetLabel
+          value: cluster
+      - equal:
+          path: spec.endpoints[0].metricRelabelings[0].regex
+          value: go_.*
+
+  - it: should merge extra labels and annotations
+    set:
+      controller.metrics.enabled: true
+      controller.metrics.serviceMonitor.enabled: true
+      controller.metrics.serviceMonitor.labels:
+        release: kube-prometheus-stack
+      controller.metrics.serviceMonitor.annotations:
+        example.com/owner: platform
+    asserts:
+      - equal:
+          path: metadata.labels["app.kubernetes.io/component"]
+          value: controller
+      - equal:
+          path: metadata.labels.release
+          value: kube-prometheus-stack
+      - equal:
+          path: metadata.annotations["example.com/owner"]
+          value: platform
+
+  - it: should render in the release namespace override
+    set:
+      controller.metrics.enabled: true
+      controller.metrics.serviceMonitor.enabled: true
+      namespaceOverride: custom-namespace
+    asserts:
+      - equal:
+          path: metadata.namespace
+          value: custom-namespace
+      - equal:
+          path: spec.namespaceSelector.matchNames[0]
+          value: custom-namespace
+
+  - it: should render in a dedicated monitoring namespace
+    set:
+      controller.metrics.enabled: true
+      controller.metrics.serviceMonitor.enabled: true
+      controller.metrics.serviceMonitor.namespace: monitoring
+    asserts:
+      - equal:
+          path: metadata.namespace
+          value: monitoring
+      - equal:
+          path: spec.namespaceSelector.matchNames[0]
+          value: NAMESPACE

helm/kagent/values.yaml

@@ -309,6 +309,42 @@ controller:
     service:
       type: ClusterIP
       port: 8443
+    # -- Prometheus Operator `ServiceMonitor` for the metrics `Service`.
+    # Requires `controller.metrics.enabled` and the
+    # `monitoring.coreos.com/v1` CRDs. Rendering the `ServiceMonitor` does
+    # not by itself authorize the scrape: with `secureServing` enabled,
+    # `<fullname>-metrics-reader` still has to be bound to the Prometheus
+    # ServiceAccount.
+    # @default -- disabled
+    serviceMonitor:
+      enabled: false
+      # -- Namespace to create the `ServiceMonitor` in. The scrape target
+      # stays the release namespace either way.
+      # @default -- the release namespace
+      namespace: ""
+      # -- Extra labels for the `ServiceMonitor` (merged with the chart
+      # labels). Set whatever label your Prometheus
+      # `serviceMonitorSelector` matches on.
+      labels: {}
+      # -- Annotations for the `ServiceMonitor`.
+      annotations: {}
+      # -- Scrape interval. Prometheus' global default when empty.
+      interval: ""
+      # -- Scrape timeout. Prometheus' global default when empty.
+      scrapeTimeout: ""
+      # -- Keep the scraped labels when they collide with server-side ones.
+      honorLabels: false
+      # -- `relabelings` applied to the scrape targets.
+      relabelings: []
+      # -- `metricRelabelings` applied to the scraped samples.
+      metricRelabelings: []
+      # -- Token presented to the authenticated metrics endpoint. Only used
+      # when `secureServing` is enabled; set to `""` to omit it.
+      bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+      # -- `tlsConfig` for the scrape. Only used when `secureServing` is
+      # enabled, where the controller serves a self-signed certificate.
+      # @default -- insecureSkipVerify: true
+      tlsConfig: {}
 
   # Extra controller env (mapped to flags via SUBSTRATE_* env names).
   env: []