From ff89199158587efba54f738414b053f2b3f669d1 Mon Sep 17 00:00:00 2001 From: Leo Huang Date: Tue, 5 Nov 2024 19:33:51 -0800 Subject: [PATCH 1/7] cicd actions - action fails on deploy failure (#736) Initial Changes (for build failures): * Update cd-dev.yaml * Update App.tsx * Update cd-prod.yaml * Update cd-stage.yaml * Revert App.tsx * update cd-dev/prod/stage to check k8 rollout status * break helm thru liveness probe, timeout reduced to 300s for testing * revert values.yaml changes, timeout reset to 1200s Testing Liveliness Probe (for runtime failures): no progress * testing liveness probe, should pass * add runtime errors, backend testing * change backend * frontend break * try this for backend * frontend fix * try: backend should break, frontend should render, but not have anything * weiofjwefi * ????????????? * try * weofij * weffff * wefoij * fix frontend, backend hope break * fix backend, frontend changes * hardcoding a failing livenessProbe Readiness Probe (working): * healthz on backend * use readiness * set timeout to 3 minutes * remove comments --------- Co-authored-by: maxmwang --- .github/workflows/cd-dev.yaml | 9 +++++++++ .github/workflows/cd-prod.yaml | 10 +++++++++- .github/workflows/cd-stage.yaml | 10 +++++++++- apps/backend/src/bootstrap/index.ts | 3 +++ infra/app/templates/backend.yaml | 6 ++++++ infra/app/templates/frontend.yaml | 10 ++++++++++ 6 files changed, 46 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd-dev.yaml b/.github/workflows/cd-dev.yaml index b68d421b3..04249bde5 100644 --- a/.github/workflows/cd-dev.yaml +++ b/.github/workflows/cd-dev.yaml @@ -51,8 +51,13 @@ jobs: username: root key: ${{ secrets.SSH_KEY }} script: | + set -e # Exit immediately if a command fails cd ./infra + + # Uninstall the old helm chart if it exists helm uninstall bt-dev-app-${{ needs.compute-sha.outputs.sha_short }} || true + + # Install new chart helm install bt-dev-app-${{ needs.compute-sha.outputs.sha_short }} ./app --namespace=bt \ --set env=dev \ --set ttl=${{ inputs.ttl }} \ @@ -62,3 +67,7 @@ jobs: --set mongoUri=mongodb://bt-dev-mongo-mongodb.bt.svc.cluster.local:27017/bt \ --set redisUri=redis://bt-dev-redis-master.bt.svc.cluster.local:6379 \ --set nodeEnv=development + + # Check container status + kubectl rollout status --timeout=180s deployment bt-dev-app-${{ needs.compute-sha.outputs.sha_short }}-backend + kubectl rollout status --timeout=180s deployment bt-dev-app-${{ needs.compute-sha.outputs.sha_short }}-frontend diff --git a/.github/workflows/cd-prod.yaml b/.github/workflows/cd-prod.yaml index 19bb245b2..f67488a1a 100644 --- a/.github/workflows/cd-prod.yaml +++ b/.github/workflows/cd-prod.yaml @@ -36,11 +36,19 @@ jobs: username: root key: ${{ secrets.SSH_KEY }} script: | + set -e # Exit immediately if a command fails cd ./infra + + # Check if the Helm release exists if helm status bt-prod-app ; then + # Restart deployments if the Helm release exists kubectl rollout restart deployment bt-prod-app-backend kubectl rollout restart deployment bt-prod-app-frontend else + # Install the Helm release if it doesn't exist helm install bt-prod-app ./app --namespace=bt \ --set host=stanfurdtime.com - fi + + # Check container status + kubectl rollout status --timeout=180s deployment bt-prod-app-backend + kubectl rollout status --timeout=180s deployment bt-prod-app-frontend diff --git a/.github/workflows/cd-stage.yaml b/.github/workflows/cd-stage.yaml index de5efce44..93ad481b4 100644 --- a/.github/workflows/cd-stage.yaml +++ b/.github/workflows/cd-stage.yaml @@ -28,11 +28,16 @@ jobs: username: root key: ${{ secrets.SSH_KEY }} script: | + set -e # Exit immediately if a command fails cd ./infra + + # Check if the Helm release exists if helm status bt-stage-app ; then + # Restart deployments if the Helm release exists kubectl rollout restart deployment bt-stage-app-backend kubectl rollout restart deployment bt-stage-app-frontend else + # Install the Helm release if it doesn't exist helm install bt-stage-app ./app --namespace=bt \ --set env=stage \ --set frontend.image.tag=latest \ @@ -40,4 +45,7 @@ jobs: --set host=staging.stanfurdtime.com \ --set mongoUri=mongodb://bt-stage-mongo-mongodb.bt.svc.cluster.local:27017/bt \ --set redisUri=redis://bt-stage-redis-master.bt.svc.cluster.local:6379 - fi + + # Check container status + kubectl rollout status --timeout=180s deployment bt-stage-app-backend + kubectl rollout status --timeout=180s deployment bt-stage-app-frontend diff --git a/apps/backend/src/bootstrap/index.ts b/apps/backend/src/bootstrap/index.ts index 2113b374a..5cf3e23f7 100644 --- a/apps/backend/src/bootstrap/index.ts +++ b/apps/backend/src/bootstrap/index.ts @@ -7,6 +7,9 @@ import loaders from "./loaders"; export default async (config: Config) => { const app = express(); app.set("trust proxy", 1); + app.get("/healthz", (_, res) => { + res.status(200).send("OK"); + }); await loaders(app); diff --git a/infra/app/templates/backend.yaml b/infra/app/templates/backend.yaml index e2c74cf71..b85ef8289 100644 --- a/infra/app/templates/backend.yaml +++ b/infra/app/templates/backend.yaml @@ -49,6 +49,12 @@ spec: value: "_" - name: SESSION_SECRET value: "_" + readinessProbe: + httpGet: + path: /healthz + port: {{ .Values.backend.port }} + initialDelaySeconds: 15 + periodSeconds: 3 --- diff --git a/infra/app/templates/frontend.yaml b/infra/app/templates/frontend.yaml index 7c661567c..cfd84f927 100644 --- a/infra/app/templates/frontend.yaml +++ b/infra/app/templates/frontend.yaml @@ -20,6 +20,16 @@ spec: imagePullPolicy: Always ports: - containerPort: {{ .Values.frontend.port }} + readinessProbe: + exec: + command: + - /bin/sh + - -c + - > + CONTENT=$(curl -s https://localhost:{{ .Values.frontend.port }}); + if [[ "$CONTENT" != *"
"* ]]; then exit 1; else exit 0; fi + initialDelaySeconds: 15 + periodSeconds: 3 --- From 227e8908b87950182359856ff2f38462fe9d577c Mon Sep 17 00:00:00 2001 From: maxmwang Date: Tue, 5 Nov 2024 19:52:04 -0800 Subject: [PATCH 2/7] add env variables to updater.yaml --- infra/app/templates/updater.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/infra/app/templates/updater.yaml b/infra/app/templates/updater.yaml index d52bb5280..d3f6149ec 100644 --- a/infra/app/templates/updater.yaml +++ b/infra/app/templates/updater.yaml @@ -43,6 +43,10 @@ spec: value: "_" - name: SIS_COURSE_APP_KEY value: "_" + - name: SIS_TERM_APP_ID + value: "_" + - name: SIS_TERM_APP_KEY + value: "_" - name: GOOGLE_CLIENT_ID value: "_" - name: GOOGLE_CLIENT_SECRET From a82f5b2a508d8c4498f029ea4b2c857c9e4042fd Mon Sep 17 00:00:00 2001 From: maxmwang Date: Tue, 5 Nov 2024 19:57:46 -0800 Subject: [PATCH 3/7] fix typo --- .github/workflows/cd-prod.yaml | 1 + .github/workflows/cd-stage.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/cd-prod.yaml b/.github/workflows/cd-prod.yaml index f67488a1a..03c61f197 100644 --- a/.github/workflows/cd-prod.yaml +++ b/.github/workflows/cd-prod.yaml @@ -48,6 +48,7 @@ jobs: # Install the Helm release if it doesn't exist helm install bt-prod-app ./app --namespace=bt \ --set host=stanfurdtime.com + fi # Check container status kubectl rollout status --timeout=180s deployment bt-prod-app-backend diff --git a/.github/workflows/cd-stage.yaml b/.github/workflows/cd-stage.yaml index 93ad481b4..fc782067b 100644 --- a/.github/workflows/cd-stage.yaml +++ b/.github/workflows/cd-stage.yaml @@ -45,6 +45,7 @@ jobs: --set host=staging.stanfurdtime.com \ --set mongoUri=mongodb://bt-stage-mongo-mongodb.bt.svc.cluster.local:27017/bt \ --set redisUri=redis://bt-stage-redis-master.bt.svc.cluster.local:6379 + fi # Check container status kubectl rollout status --timeout=180s deployment bt-stage-app-backend From bfedf919d007bae6630dfcc94b971c35bc000e74 Mon Sep 17 00:00:00 2001 From: maxmwang Date: Tue, 5 Nov 2024 21:56:35 -0800 Subject: [PATCH 4/7] readiness probe for frontendd --- infra/app/templates/frontend.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/infra/app/templates/frontend.yaml b/infra/app/templates/frontend.yaml index cfd84f927..d87544660 100644 --- a/infra/app/templates/frontend.yaml +++ b/infra/app/templates/frontend.yaml @@ -21,13 +21,9 @@ spec: ports: - containerPort: {{ .Values.frontend.port }} readinessProbe: - exec: - command: - - /bin/sh - - -c - - > - CONTENT=$(curl -s https://localhost:{{ .Values.frontend.port }}); - if [[ "$CONTENT" != *"
"* ]]; then exit 1; else exit 0; fi + httpGet: + path: / + port: {{ .Values.backend.port }} initialDelaySeconds: 15 periodSeconds: 3 From 2fa51525113b7429b79ea064430f7491d88cd97b Mon Sep 17 00:00:00 2001 From: maxmwang Date: Tue, 5 Nov 2024 22:58:05 -0800 Subject: [PATCH 5/7] fix cleanup job --- infra/app/templates/cleanup.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/infra/app/templates/cleanup.yaml b/infra/app/templates/cleanup.yaml index c5f85eba3..ea25b4512 100644 --- a/infra/app/templates/cleanup.yaml +++ b/infra/app/templates/cleanup.yaml @@ -11,6 +11,12 @@ spec: containers: - name: cleanup image: alpine/helm - command: ['sh', '-c', 'sleep $(( {{ .Values.ttl }} * 60 * 60 )); helm uninstall {{ .Release.Name }}'] + command: + - sh + - -c + - | + echo "Cleaning up {{ .Release.Name }} in {{ .Values.ttl }} hours" + sleep {{ .Values.ttl }}h + helm uninstall {{ .Release.Name }} restartPolicy: Never {{ end }} From 450b55325f352d9f35e2229dbd8b254eb7716d63 Mon Sep 17 00:00:00 2001 From: maxmwang Date: Tue, 5 Nov 2024 22:59:24 -0800 Subject: [PATCH 6/7] fix typo --- infra/app/templates/frontend.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/app/templates/frontend.yaml b/infra/app/templates/frontend.yaml index d87544660..27263fa80 100644 --- a/infra/app/templates/frontend.yaml +++ b/infra/app/templates/frontend.yaml @@ -23,7 +23,7 @@ spec: readinessProbe: httpGet: path: / - port: {{ .Values.backend.port }} + port: {{ .Values.frontend.port }} initialDelaySeconds: 15 periodSeconds: 3 From 3db2d3504be2978fde34a0b792e0f1526e6f7941 Mon Sep 17 00:00:00 2001 From: maxmwang Date: Tue, 5 Nov 2024 23:00:27 -0800 Subject: [PATCH 7/7] add term env vars to backend helm template --- infra/app/templates/backend.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/infra/app/templates/backend.yaml b/infra/app/templates/backend.yaml index b85ef8289..7f0a2c422 100644 --- a/infra/app/templates/backend.yaml +++ b/infra/app/templates/backend.yaml @@ -43,6 +43,10 @@ spec: value: "_" - name: SIS_COURSE_APP_KEY value: "_" + - name: SIS_TERM_APP_ID + value: "_" + - name: SIS_TERM_APP_KEY + value: "_" - name: GOOGLE_CLIENT_ID value: "_" - name: GOOGLE_CLIENT_SECRET