Add securedMetricsPort

charts/piraeus/crds/piraeus.linbit.com_linstorcontrollers.yaml

@@ -2633,6 +2633,11 @@ spec:
                       to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
                     type: object
                 type: object
+              securedMetricsPort:
+                description: Port for scraping metrics protected by RBAC proxy
+                format: int32
+                nullable: true
+                type: integer
               serviceAccountName:
                 description: Name of the service account that runs leader elections
                   for linstor

doc/security.md

@@ -158,6 +158,15 @@ The metrics collection can be secured using [kube-rbac-proxy](https://github.com
 * Take a look at this [example chart configuration](../examples/rbac-proxy-values.yaml) and [additional resources](../examples/rbac-proxy-resources.yaml)
 which must be configured in the cluster to make rbac-proxy working in your configuration.
 
+## Fixing target down problem in Prometheus when LINSTOR controller is deployed in HA mode and communications for collecting metrics are secured.
+
+If a LINSTOR controller has more than one replica, only one target will be up in Prometheus. This is because of the leader election mechanism for LINSTOR controllers. Once a leader is elected, other pods of LINSTOR controllers won't provide any metrics. To resolve this issue, follow the steps below:
+
+1. Replace the PodMonitor with ServiceMonitor.
+2. Set the `securedMetricsPort` in the LinstorController custom resource.
+
+You can refer to this [example service monitor](../examples/service-monitor-in-ha-mode.yaml) for guidance.
+
 ## Automatically set the passphrase for LINSTOR
 
 LINSTOR may need to store sensitive information in its database, for example for encrypted volumes using the LUKS layer,

examples/service-monitor-in-ha-mode.yaml

@@ -0,0 +1,26 @@
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: linstor-controller
+  namespace: monitoring
+  labels:
+    prometheus: main
+spec:
+  endpoints:
+  - port: metrics
+    scheme: https
+    path: /metrics
+    bearerTokenSecret:
+      name: "prometheus-token"
+      key: "token"
+    tlsConfig:
+      insecureSkipVerify: true
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: linstor
+      app.kubernetes.io/managed-by: piraeus-operator
+      app.kubernetes.io/name: piraeus-controller
+  namespaceSelector:
+    matchNames:
+    - piraeus

pkg/apis/piraeus/v1/linstorcontroller_types.go

@@ -73,6 +73,11 @@ type LinstorControllerSpec struct {
 	// +optional
 	LinstorHttpsControllerSecret string `json:"linstorHttpsControllerSecret"`
 
+	// Port for scraping metrics protected by RBAC proxy
+	// +optional
+	// +nullable
+	SecuredMetricsPort *int32 `json:"securedMetricsPort"`
+
 	// Resource requirements for the LINSTOR controller pod
 	// +optional
 	// +nullable

pkg/controller/linstorcontroller/linstorcontroller_controller.go

@@ -22,6 +22,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"sort"
 	"strings"
 	"time"
 
@@ -674,6 +675,15 @@ func newDeploymentForResource(controllerResource *piraeusv1.LinstorController) *
 		{Name: controllerResource.Name, Port: int32(port)},
 	}
 
+	if controllerResource.Spec.SecuredMetricsPort != nil {
+		metricsPort := corev1.EndpointPort{
+			Name: "metrics", Port: *controllerResource.Spec.SecuredMetricsPort}
+		servicePorts = append(servicePorts, metricsPort)
+		sort.SliceStable(servicePorts, func(i, j int) bool {
+			return servicePorts[i].Port < servicePorts[j].Port
+		})
+	}
+
 	servicePortsJSON, err := json.Marshal(servicePorts)
 	if err != nil {
 		panic(err)
@@ -962,18 +972,33 @@ func newServiceForResource(controllerResource *piraeusv1.LinstorController) *cor
 		port = lc.DefaultHTTPSPort
 	}
 
+	ports := []corev1.ServicePort{
+		{
+			Name:       controllerResource.Name,
+			Port:       int32(port),
+			Protocol:   "TCP",
+			TargetPort: intstr.FromInt(port),
+		},
+	}
+
+	if controllerResource.Spec.SecuredMetricsPort != nil {
+		metricsPort := corev1.ServicePort{
+			Name:       "metrics",
+			Port:       *controllerResource.Spec.SecuredMetricsPort,
+			Protocol:   "TCP",
+			TargetPort: intstr.FromInt(int(*controllerResource.Spec.SecuredMetricsPort)),
+		}
+		ports = append(ports, metricsPort)
+		sort.SliceStable(ports, func(i, j int) bool {
+			return ports[i].Port < ports[j].Port
+		})
+	}
+
 	return &corev1.Service{
 		ObjectMeta: getObjectMeta(controllerResource, "%s"),
 		Spec: corev1.ServiceSpec{
 			ClusterIP: "",
-			Ports: []corev1.ServicePort{
-				{
-					Name:       controllerResource.Name,
-					Port:       int32(port),
-					Protocol:   "TCP",
-					TargetPort: intstr.FromInt(port),
-				},
-			},
+			Ports: ports,
 			Type: corev1.ServiceTypeClusterIP,
 		},
 	}