deploy: revise probe settings

Containers remained in the "not ready" state for a noticable amount of time, probably because the startup probe failed and then only checked again after 10 seconds. These delays caused test failures and slowed down testing. With the previous values, `make test_e2e TEST_E2E_FOCUS=operator.*API.*switch.device.mode` ran for 1370 seconds and had one failure. With the new values, it finished successfully in 617 seconds.
intel · Jun 15, 2021 · 765b13f · 765b13f
1 parent f2e1671
commit 765b13f
Show file tree

Hide file tree

Showing 27 changed files with 419 additions and 193 deletions.
diff --git a/deploy/kubernetes-1.19/direct/pmem-csi.yaml b/deploy/kubernetes-1.19/direct/pmem-csi.yaml
@@ -381,7 +381,7 @@ spec:
         image: intel/pmem-csi-driver:canary
         imagePullPolicy: IfNotPresent
         livenessProbe:
-          failureThreshold: 5
+          failureThreshold: 6
           httpGet:
             path: /metrics
             port: metrics
@@ -399,6 +399,15 @@ spec:
             memory: 128Mi
         securityContext:
           readOnlyRootFilesystem: true
+        startupProbe:
+          failureThreshold: 60
+          httpGet:
+            path: /metrics
+            port: metrics
+            scheme: HTTP
+          periodSeconds: 1
+          successThreshold: 1
+          timeoutSeconds: 5
         terminationMessagePath: /dev/termination-log
         volumeMounts:
         - mountPath: /certs
@@ -473,7 +482,7 @@ spec:
         image: intel/pmem-csi-driver:canary
         imagePullPolicy: IfNotPresent
         livenessProbe:
-          failureThreshold: 5
+          failureThreshold: 6
           httpGet:
             path: /metrics
             port: metrics
@@ -493,12 +502,12 @@ spec:
           privileged: true
           runAsUser: 0
         startupProbe:
-          failureThreshold: 30
+          failureThreshold: 300
           httpGet:
             path: /metrics
             port: metrics
             scheme: HTTP
-          periodSeconds: 10
+          periodSeconds: 1
           successThreshold: 1
           timeoutSeconds: 5
         terminationMessagePath: /tmp/termination-log
@@ -562,7 +571,7 @@ spec:
         image: k8s.gcr.io/sig-storage/csi-provisioner:v2.2.2
         imagePullPolicy: IfNotPresent
         livenessProbe:
-          failureThreshold: 5
+          failureThreshold: 6
           httpGet:
             path: /metrics
             port: metrics
@@ -581,12 +590,12 @@ spec:
         securityContext:
           readOnlyRootFilesystem: true
         startupProbe:
-          failureThreshold: 30
+          failureThreshold: 300
           httpGet:
             path: /metrics
             port: metrics
             scheme: HTTP
-          periodSeconds: 10
+          periodSeconds: 1
           successThreshold: 1
           timeoutSeconds: 5
         volumeMounts:

diff --git a/deploy/kubernetes-1.19/direct/testing/pmem-csi.yaml b/deploy/kubernetes-1.19/direct/testing/pmem-csi.yaml
@@ -383,7 +383,7 @@ spec:
         image: intel/pmem-csi-driver-test:canary
         imagePullPolicy: IfNotPresent
         livenessProbe:
-          failureThreshold: 5
+          failureThreshold: 6
           httpGet:
             path: /metrics
             port: metrics
@@ -401,6 +401,15 @@ spec:
             memory: 128Mi
         securityContext:
           readOnlyRootFilesystem: true
+        startupProbe:
+          failureThreshold: 60
+          httpGet:
+            path: /metrics
+            port: metrics
+            scheme: HTTP
+          periodSeconds: 1
+          successThreshold: 1
+          timeoutSeconds: 5
         terminationMessagePath: /dev/termination-log
         volumeMounts:
         - mountPath: /certs
@@ -497,7 +506,7 @@ spec:
         image: intel/pmem-csi-driver-test:canary
         imagePullPolicy: IfNotPresent
         livenessProbe:
-          failureThreshold: 5
+          failureThreshold: 6
           httpGet:
             path: /metrics
             port: metrics
@@ -517,12 +526,12 @@ spec:
           privileged: true
           runAsUser: 0
         startupProbe:
-          failureThreshold: 30
+          failureThreshold: 300
           httpGet:
             path: /metrics
             port: metrics
             scheme: HTTP
-          periodSeconds: 10
+          periodSeconds: 1
           successThreshold: 1
           timeoutSeconds: 5
         terminationMessagePath: /tmp/termination-log
@@ -590,7 +599,7 @@ spec:
         image: k8s.gcr.io/sig-storage/csi-provisioner:v2.2.2
         imagePullPolicy: IfNotPresent
         livenessProbe:
-          failureThreshold: 5
+          failureThreshold: 6
           httpGet:
             path: /metrics
             port: metrics
@@ -609,12 +618,12 @@ spec:
         securityContext:
           readOnlyRootFilesystem: true
         startupProbe:
-          failureThreshold: 30
+          failureThreshold: 300
           httpGet:
             path: /metrics
             port: metrics
             scheme: HTTP
-          periodSeconds: 10
+          periodSeconds: 1
           successThreshold: 1
           timeoutSeconds: 5
         volumeMounts:

diff --git a/deploy/kubernetes-1.19/lvm/pmem-csi.yaml b/deploy/kubernetes-1.19/lvm/pmem-csi.yaml
@@ -381,7 +381,7 @@ spec:
         image: intel/pmem-csi-driver:canary
         imagePullPolicy: IfNotPresent
         livenessProbe:
-          failureThreshold: 5
+          failureThreshold: 6
           httpGet:
             path: /metrics
             port: metrics
@@ -399,6 +399,15 @@ spec:
             memory: 128Mi
         securityContext:
           readOnlyRootFilesystem: true
+        startupProbe:
+          failureThreshold: 60
+          httpGet:
+            path: /metrics
+            port: metrics
+            scheme: HTTP
+          periodSeconds: 1
+          successThreshold: 1
+          timeoutSeconds: 5
         terminationMessagePath: /dev/termination-log
         volumeMounts:
         - mountPath: /certs
@@ -473,7 +482,7 @@ spec:
         image: intel/pmem-csi-driver:canary
         imagePullPolicy: IfNotPresent
         livenessProbe:
-          failureThreshold: 5
+          failureThreshold: 6
           httpGet:
             path: /metrics
             port: metrics
@@ -493,12 +502,12 @@ spec:
           privileged: true
           runAsUser: 0
         startupProbe:
-          failureThreshold: 30
+          failureThreshold: 300
           httpGet:
             path: /metrics
             port: metrics
             scheme: HTTP
-          periodSeconds: 10
+          periodSeconds: 1
           successThreshold: 1
           timeoutSeconds: 5
         terminationMessagePath: /tmp/termination-log
@@ -562,7 +571,7 @@ spec:
         image: k8s.gcr.io/sig-storage/csi-provisioner:v2.2.2
         imagePullPolicy: IfNotPresent
         livenessProbe:
-          failureThreshold: 5
+          failureThreshold: 6
           httpGet:
             path: /metrics
             port: metrics
@@ -581,12 +590,12 @@ spec:
         securityContext:
           readOnlyRootFilesystem: true
         startupProbe:
-          failureThreshold: 30
+          failureThreshold: 300
           httpGet:
             path: /metrics
             port: metrics
             scheme: HTTP
-          periodSeconds: 10
+          periodSeconds: 1
           successThreshold: 1
           timeoutSeconds: 5
         volumeMounts:

diff --git a/deploy/kubernetes-1.19/lvm/testing/pmem-csi.yaml b/deploy/kubernetes-1.19/lvm/testing/pmem-csi.yaml
@@ -383,7 +383,7 @@ spec:
         image: intel/pmem-csi-driver-test:canary
         imagePullPolicy: IfNotPresent
         livenessProbe:
-          failureThreshold: 5
+          failureThreshold: 6
           httpGet:
             path: /metrics
             port: metrics
@@ -401,6 +401,15 @@ spec:
             memory: 128Mi
         securityContext:
           readOnlyRootFilesystem: true
+        startupProbe:
+          failureThreshold: 60
+          httpGet:
+            path: /metrics
+            port: metrics
+            scheme: HTTP
+          periodSeconds: 1
+          successThreshold: 1
+          timeoutSeconds: 5
         terminationMessagePath: /dev/termination-log
         volumeMounts:
         - mountPath: /certs
@@ -497,7 +506,7 @@ spec:
         image: intel/pmem-csi-driver-test:canary
         imagePullPolicy: IfNotPresent
         livenessProbe:
-          failureThreshold: 5
+          failureThreshold: 6
           httpGet:
             path: /metrics
             port: metrics
@@ -517,12 +526,12 @@ spec:
           privileged: true
           runAsUser: 0
         startupProbe:
-          failureThreshold: 30
+          failureThreshold: 300
           httpGet:
             path: /metrics
             port: metrics
             scheme: HTTP
-          periodSeconds: 10
+          periodSeconds: 1
           successThreshold: 1
           timeoutSeconds: 5
         terminationMessagePath: /tmp/termination-log
@@ -590,7 +599,7 @@ spec:
         image: k8s.gcr.io/sig-storage/csi-provisioner:v2.2.2
         imagePullPolicy: IfNotPresent
         livenessProbe:
-          failureThreshold: 5
+          failureThreshold: 6
           httpGet:
             path: /metrics
             port: metrics
@@ -609,12 +618,12 @@ spec:
         securityContext:
           readOnlyRootFilesystem: true
         startupProbe:
-          failureThreshold: 30
+          failureThreshold: 300
           httpGet:
             path: /metrics
             port: metrics
             scheme: HTTP
-          periodSeconds: 10
+          periodSeconds: 1
           successThreshold: 1
           timeoutSeconds: 5
         volumeMounts:

diff --git a/deploy/kubernetes-1.19/pmem-csi-direct-testing.yaml b/deploy/kubernetes-1.19/pmem-csi-direct-testing.yaml
@@ -383,7 +383,7 @@ spec:
         image: intel/pmem-csi-driver-test:canary
         imagePullPolicy: IfNotPresent
         livenessProbe:
-          failureThreshold: 5
+          failureThreshold: 6
           httpGet:
             path: /metrics
             port: metrics
@@ -401,6 +401,15 @@ spec:
             memory: 128Mi
         securityContext:
           readOnlyRootFilesystem: true
+        startupProbe:
+          failureThreshold: 60
+          httpGet:
+            path: /metrics
+            port: metrics
+            scheme: HTTP
+          periodSeconds: 1
+          successThreshold: 1
+          timeoutSeconds: 5
         terminationMessagePath: /dev/termination-log
         volumeMounts:
         - mountPath: /certs
@@ -497,7 +506,7 @@ spec:
         image: intel/pmem-csi-driver-test:canary
         imagePullPolicy: IfNotPresent
         livenessProbe:
-          failureThreshold: 5
+          failureThreshold: 6
           httpGet:
             path: /metrics
             port: metrics
@@ -517,12 +526,12 @@ spec:
           privileged: true
           runAsUser: 0
         startupProbe:
-          failureThreshold: 30
+          failureThreshold: 300
           httpGet:
             path: /metrics
             port: metrics
             scheme: HTTP
-          periodSeconds: 10
+          periodSeconds: 1
           successThreshold: 1
           timeoutSeconds: 5
         terminationMessagePath: /tmp/termination-log
@@ -590,7 +599,7 @@ spec:
         image: k8s.gcr.io/sig-storage/csi-provisioner:v2.2.2
         imagePullPolicy: IfNotPresent
         livenessProbe:
-          failureThreshold: 5
+          failureThreshold: 6
           httpGet:
             path: /metrics
             port: metrics
@@ -609,12 +618,12 @@ spec:
         securityContext:
           readOnlyRootFilesystem: true
         startupProbe:
-          failureThreshold: 30
+          failureThreshold: 300
           httpGet:
             path: /metrics
             port: metrics
             scheme: HTTP
-          periodSeconds: 10
+          periodSeconds: 1
           successThreshold: 1
           timeoutSeconds: 5
         volumeMounts: