cicd actions - action fails on deploy failure (#736)

Initial Changes (for build failures):
* Update cd-dev.yaml
* Update App.tsx
* Update cd-prod.yaml
* Update cd-stage.yaml
* Revert App.tsx
* update cd-dev/prod/stage to check k8 rollout status
* break helm thru liveness probe, timeout reduced to 300s for testing
* revert values.yaml changes, timeout reset to 1200s

Testing Liveliness Probe (for runtime failures): no progress
* testing liveness probe, should pass
* add runtime errors, backend testing
* change backend
* frontend break
* try this for backend
* frontend fix
* try: backend should break, frontend should render, but not have anything
* weiofjwefi
* ?????????????
* try
* weofij
* weffff
* wefoij
* fix frontend, backend hope break
* fix backend, frontend changes
* hardcoding a failing livenessProbe

Readiness Probe (working):
* healthz on backend
* use readiness
* set timeout to 3 minutes
* remove comments

---------

Co-authored-by: maxmwang <[email protected]>

.github/workflows/cd-dev.yaml

@@ -51,8 +51,13 @@ jobs:
           username: root
           key: ${{ secrets.SSH_KEY }}
           script: |
+            set -e  # Exit immediately if a command fails
             cd ./infra
+
+            # Uninstall the old helm chart if it exists
             helm uninstall bt-dev-app-${{ needs.compute-sha.outputs.sha_short }} || true
+
+            # Install new chart
             helm install bt-dev-app-${{ needs.compute-sha.outputs.sha_short }} ./app --namespace=bt \
               --set env=dev \
               --set ttl=${{ inputs.ttl }} \
@@ -62,3 +67,7 @@ jobs:
               --set mongoUri=mongodb://bt-dev-mongo-mongodb.bt.svc.cluster.local:27017/bt \
               --set redisUri=redis://bt-dev-redis-master.bt.svc.cluster.local:6379 \
               --set nodeEnv=development
+
+            # Check container status
+            kubectl rollout status --timeout=180s deployment bt-dev-app-${{ needs.compute-sha.outputs.sha_short }}-backend
+            kubectl rollout status --timeout=180s deployment bt-dev-app-${{ needs.compute-sha.outputs.sha_short }}-frontend

.github/workflows/cd-prod.yaml

@@ -36,11 +36,19 @@ jobs:
           username: root
           key: ${{ secrets.SSH_KEY }}
           script: |
+            set -e  # Exit immediately if a command fails
             cd ./infra
+
+            # Check if the Helm release exists
             if helm status bt-prod-app ; then
+              # Restart deployments if the Helm release exists
               kubectl rollout restart deployment bt-prod-app-backend
               kubectl rollout restart deployment bt-prod-app-frontend
             else
+              # Install the Helm release if it doesn't exist
               helm install bt-prod-app ./app --namespace=bt \
                 --set host=stanfurdtime.com
-            fi
+
+            # Check container status
+            kubectl rollout status --timeout=180s deployment bt-prod-app-backend
+            kubectl rollout status --timeout=180s deployment bt-prod-app-frontend

.github/workflows/cd-stage.yaml

@@ -28,16 +28,24 @@ jobs:
           username: root
           key: ${{ secrets.SSH_KEY }}
           script: |
+            set -e  # Exit immediately if a command fails
             cd ./infra
+
+            # Check if the Helm release exists
             if helm status bt-stage-app ; then
+              # Restart deployments if the Helm release exists
               kubectl rollout restart deployment bt-stage-app-backend
               kubectl rollout restart deployment bt-stage-app-frontend
             else
+              # Install the Helm release if it doesn't exist
               helm install bt-stage-app ./app --namespace=bt \
                 --set env=stage \
                 --set frontend.image.tag=latest \
                 --set backend.image.tag=latest \
                 --set host=staging.stanfurdtime.com \
                 --set mongoUri=mongodb://bt-stage-mongo-mongodb.bt.svc.cluster.local:27017/bt \
                 --set redisUri=redis://bt-stage-redis-master.bt.svc.cluster.local:6379
-            fi
+
+            # Check container status
+            kubectl rollout status --timeout=180s deployment bt-stage-app-backend
+            kubectl rollout status --timeout=180s deployment bt-stage-app-frontend

apps/backend/src/bootstrap/index.ts

@@ -7,6 +7,9 @@ import loaders from "./loaders";
 export default async (config: Config) => {
   const app = express();
   app.set("trust proxy", 1);
+  app.get("/healthz", (_, res) => {
+    res.status(200).send("OK");
+  });
 
   await loaders(app);
 

infra/app/templates/backend.yaml

@@ -49,6 +49,12 @@ spec:
               value: "_"
             - name: SESSION_SECRET
               value: "_"
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: {{ .Values.backend.port }}
+            initialDelaySeconds: 15
+            periodSeconds: 3
 
 ---
 

infra/app/templates/frontend.yaml

@@ -20,6 +20,16 @@ spec:
           imagePullPolicy: Always
           ports:
             - containerPort: {{ .Values.frontend.port }}
+          readinessProbe:
+            exec:
+              command:
+                - /bin/sh
+                - -c
+                - >
+                  CONTENT=$(curl -s https://localhost:{{ .Values.frontend.port }});
+                  if [[ "$CONTENT" != *"<div id="root"></div>"* ]]; then exit 1; else exit 0; fi
+            initialDelaySeconds: 15
+            periodSeconds: 3
 
 ---