From ff89199158587efba54f738414b053f2b3f669d1 Mon Sep 17 00:00:00 2001 From: Leo Huang Date: Tue, 5 Nov 2024 19:33:51 -0800 Subject: [PATCH] cicd actions - action fails on deploy failure (#736) Initial Changes (for build failures): * Update cd-dev.yaml * Update App.tsx * Update cd-prod.yaml * Update cd-stage.yaml * Revert App.tsx * update cd-dev/prod/stage to check k8 rollout status * break helm thru liveness probe, timeout reduced to 300s for testing * revert values.yaml changes, timeout reset to 1200s Testing Liveliness Probe (for runtime failures): no progress * testing liveness probe, should pass * add runtime errors, backend testing * change backend * frontend break * try this for backend * frontend fix * try: backend should break, frontend should render, but not have anything * weiofjwefi * ????????????? * try * weofij * weffff * wefoij * fix frontend, backend hope break * fix backend, frontend changes * hardcoding a failing livenessProbe Readiness Probe (working): * healthz on backend * use readiness * set timeout to 3 minutes * remove comments --------- Co-authored-by: maxmwang --- .github/workflows/cd-dev.yaml | 9 +++++++++ .github/workflows/cd-prod.yaml | 10 +++++++++- .github/workflows/cd-stage.yaml | 10 +++++++++- apps/backend/src/bootstrap/index.ts | 3 +++ infra/app/templates/backend.yaml | 6 ++++++ infra/app/templates/frontend.yaml | 10 ++++++++++ 6 files changed, 46 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd-dev.yaml b/.github/workflows/cd-dev.yaml index b68d421b3..04249bde5 100644 --- a/.github/workflows/cd-dev.yaml +++ b/.github/workflows/cd-dev.yaml @@ -51,8 +51,13 @@ jobs: username: root key: ${{ secrets.SSH_KEY }} script: | + set -e # Exit immediately if a command fails cd ./infra + + # Uninstall the old helm chart if it exists helm uninstall bt-dev-app-${{ needs.compute-sha.outputs.sha_short }} || true + + # Install new chart helm install bt-dev-app-${{ needs.compute-sha.outputs.sha_short }} ./app --namespace=bt \ --set env=dev \ --set ttl=${{ inputs.ttl }} \ @@ -62,3 +67,7 @@ jobs: --set mongoUri=mongodb://bt-dev-mongo-mongodb.bt.svc.cluster.local:27017/bt \ --set redisUri=redis://bt-dev-redis-master.bt.svc.cluster.local:6379 \ --set nodeEnv=development + + # Check container status + kubectl rollout status --timeout=180s deployment bt-dev-app-${{ needs.compute-sha.outputs.sha_short }}-backend + kubectl rollout status --timeout=180s deployment bt-dev-app-${{ needs.compute-sha.outputs.sha_short }}-frontend diff --git a/.github/workflows/cd-prod.yaml b/.github/workflows/cd-prod.yaml index 19bb245b2..f67488a1a 100644 --- a/.github/workflows/cd-prod.yaml +++ b/.github/workflows/cd-prod.yaml @@ -36,11 +36,19 @@ jobs: username: root key: ${{ secrets.SSH_KEY }} script: | + set -e # Exit immediately if a command fails cd ./infra + + # Check if the Helm release exists if helm status bt-prod-app ; then + # Restart deployments if the Helm release exists kubectl rollout restart deployment bt-prod-app-backend kubectl rollout restart deployment bt-prod-app-frontend else + # Install the Helm release if it doesn't exist helm install bt-prod-app ./app --namespace=bt \ --set host=stanfurdtime.com - fi + + # Check container status + kubectl rollout status --timeout=180s deployment bt-prod-app-backend + kubectl rollout status --timeout=180s deployment bt-prod-app-frontend diff --git a/.github/workflows/cd-stage.yaml b/.github/workflows/cd-stage.yaml index de5efce44..93ad481b4 100644 --- a/.github/workflows/cd-stage.yaml +++ b/.github/workflows/cd-stage.yaml @@ -28,11 +28,16 @@ jobs: username: root key: ${{ secrets.SSH_KEY }} script: | + set -e # Exit immediately if a command fails cd ./infra + + # Check if the Helm release exists if helm status bt-stage-app ; then + # Restart deployments if the Helm release exists kubectl rollout restart deployment bt-stage-app-backend kubectl rollout restart deployment bt-stage-app-frontend else + # Install the Helm release if it doesn't exist helm install bt-stage-app ./app --namespace=bt \ --set env=stage \ --set frontend.image.tag=latest \ @@ -40,4 +45,7 @@ jobs: --set host=staging.stanfurdtime.com \ --set mongoUri=mongodb://bt-stage-mongo-mongodb.bt.svc.cluster.local:27017/bt \ --set redisUri=redis://bt-stage-redis-master.bt.svc.cluster.local:6379 - fi + + # Check container status + kubectl rollout status --timeout=180s deployment bt-stage-app-backend + kubectl rollout status --timeout=180s deployment bt-stage-app-frontend diff --git a/apps/backend/src/bootstrap/index.ts b/apps/backend/src/bootstrap/index.ts index 2113b374a..5cf3e23f7 100644 --- a/apps/backend/src/bootstrap/index.ts +++ b/apps/backend/src/bootstrap/index.ts @@ -7,6 +7,9 @@ import loaders from "./loaders"; export default async (config: Config) => { const app = express(); app.set("trust proxy", 1); + app.get("/healthz", (_, res) => { + res.status(200).send("OK"); + }); await loaders(app); diff --git a/infra/app/templates/backend.yaml b/infra/app/templates/backend.yaml index e2c74cf71..b85ef8289 100644 --- a/infra/app/templates/backend.yaml +++ b/infra/app/templates/backend.yaml @@ -49,6 +49,12 @@ spec: value: "_" - name: SESSION_SECRET value: "_" + readinessProbe: + httpGet: + path: /healthz + port: {{ .Values.backend.port }} + initialDelaySeconds: 15 + periodSeconds: 3 --- diff --git a/infra/app/templates/frontend.yaml b/infra/app/templates/frontend.yaml index 7c661567c..cfd84f927 100644 --- a/infra/app/templates/frontend.yaml +++ b/infra/app/templates/frontend.yaml @@ -20,6 +20,16 @@ spec: imagePullPolicy: Always ports: - containerPort: {{ .Values.frontend.port }} + readinessProbe: + exec: + command: + - /bin/sh + - -c + - > + CONTENT=$(curl -s https://localhost:{{ .Values.frontend.port }}); + if [[ "$CONTENT" != *"
"* ]]; then exit 1; else exit 0; fi + initialDelaySeconds: 15 + periodSeconds: 3 ---