Add liveness probe to OCS Operator

- Register /healthz endpoint in main.go
- Add liveness, readiness, and startup probes to manager Deployment

Signed-off-by: Oded Viner <[email protected]>

Author: OdedViner

config/manager/manager.yaml

@@ -30,6 +30,9 @@ spec:
         image: ocs-dev/ocs-operator:latest
         imagePullPolicy: Always
         name: ocs-operator
+        ports:
+          - name: healthz
+            containerPort: 8081
         env:
           - name: WATCH_NAMESPACE
             valueFrom:
@@ -38,13 +41,30 @@ spec:
         readinessProbe:
           httpGet:
             path: /readyz
-            port: 8081
+            port: healthz
           initialDelaySeconds: 5
           periodSeconds: 10
+          timeoutSeconds: 5
+          failureThreshold: 3
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: healthz
+          initialDelaySeconds: 15
+          periodSeconds: 10
+          timeoutSeconds: 5
+          failureThreshold: 3
+        startupProbe:
+          httpGet:
+            path: /healthz
+            port: healthz
+          periodSeconds: 10
+          timeoutSeconds: 5
+          failureThreshold: 30
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:
-              drop:
+            drop:
               - all
           readOnlyRootFilesystem: true
         terminationMessagePolicy: FallbackToLogsOnError

deploy/csv-templates/ocs-operator.csv.yaml.in

@@ -624,20 +624,40 @@ spec:
                       fieldPath: metadata.annotations['olm.targetNamespaces']
                 image: quay.io/ocs-dev/ocs-operator:latest
                 imagePullPolicy: Always
+                livenessProbe:
+                  failureThreshold: 3
+                  httpGet:
+                    path: /healthz
+                    port: healthz
+                  initialDelaySeconds: 15
+                  periodSeconds: 10
+                  timeoutSeconds: 5
                 name: ocs-operator
+                ports:
+                - containerPort: 8081
+                  name: healthz
                 readinessProbe:
+                  failureThreshold: 3
                   httpGet:
                     path: /readyz
-                    port: 8081
+                    port: healthz
                   initialDelaySeconds: 5
                   periodSeconds: 10
+                  timeoutSeconds: 5
                 resources: {}
                 securityContext:
                   allowPrivilegeEscalation: false
                   capabilities:
                     drop:
                     - all
                   readOnlyRootFilesystem: true
+                startupProbe:
+                  failureThreshold: 30
+                  httpGet:
+                    path: /healthz
+                    port: healthz
+                  periodSeconds: 10
+                  timeoutSeconds: 5
                 terminationMessagePolicy: FallbackToLogsOnError
                 volumeMounts:
                 - mountPath: /etc/private-key

deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml

@@ -663,13 +663,26 @@ spec:
                       fieldPath: metadata.namespace
                 image: quay.io/ocs-dev/ocs-operator:latest
                 imagePullPolicy: Always
+                livenessProbe:
+                  failureThreshold: 3
+                  httpGet:
+                    path: /healthz
+                    port: healthz
+                  initialDelaySeconds: 15
+                  periodSeconds: 10
+                  timeoutSeconds: 5
                 name: ocs-operator
+                ports:
+                - containerPort: 8081
+                  name: healthz
                 readinessProbe:
+                  failureThreshold: 3
                   httpGet:
                     path: /readyz
-                    port: 8081
+                    port: healthz
                   initialDelaySeconds: 5
                   periodSeconds: 10
+                  timeoutSeconds: 5
                 resources:
                   limits:
                     cpu: 250m
@@ -683,6 +696,13 @@ spec:
                     drop:
                     - all
                   readOnlyRootFilesystem: true
+                startupProbe:
+                  failureThreshold: 30
+                  httpGet:
+                    path: /healthz
+                    port: healthz
+                  periodSeconds: 10
+                  timeoutSeconds: 5
                 terminationMessagePolicy: FallbackToLogsOnError
                 volumeMounts:
                 - mountPath: /etc/private-key

main.go

@@ -61,6 +61,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	apiclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	metrics "sigs.k8s.io/controller-runtime/pkg/metrics/server"
@@ -305,6 +306,12 @@ func main() {
 		os.Exit(1)
 	}
 
+	// Liveness (/healthz)
+	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
+		setupLog.Error(err, "unable to add a health check")
+		os.Exit(1)
+	}
+
 	// Add readiness probe
 	if err := mgr.AddReadyzCheck("readyz", storagecluster.ReadinessChecker); err != nil {
 		setupLog.Error(err, "unable add a readiness check")