Merge branch 'staging' into update-fix/Analytics-data-export-cleanup

Updates from airqo staging
airqo-platform · Nov 20, 2024 · 8bab12d · 8bab12d
2 parents 8bd88dc + 15a719b
commit 8bab12d
Show file tree

Hide file tree

Showing 41 changed files with 3,287 additions and 1,083 deletions.
diff --git a/.github/workflows/deploy-apis-to-staging.yml b/.github/workflows/deploy-apis-to-staging.yml
@@ -56,6 +56,7 @@ jobs:
       run_incentives: ${{ steps.check_files.outputs.run_incentives }} # incentives
       run_insights: ${{ steps.check_files.outputs.run_insights }} # incentives
       run_spatial: ${{ steps.check_files.outputs.run_spatial }} # spatial
+      run_website: ${{ steps.check_files.outputs.run_website }} # website
       run_kafka_connectors: ${{ steps.check_files.outputs.run_kafka_connectors }} # kafka connectors
       run_nginx: ${{ steps.check_files.outputs.run_nginx }} # nginx ingress
 
@@ -98,6 +99,7 @@ jobs:
           echo "run_incentives=false" >>$GITHUB_OUTPUT
           echo "run_insights=false" >>$GITHUB_OUTPUT
           echo "run_spatial=false" >>$GITHUB_OUTPUT
+          echo "run_website=false" >>$GITHUB_OUTPUT
           echo "run_kafka_connectors=false" >>$GITHUB_OUTPUT
           echo "run_nginx=false" >>$GITHUB_OUTPUT
 
@@ -271,6 +273,14 @@ jobs:
             
             if [[ $file == k8s/spatial/* ]]; then
               echo "run_spatial=true" >>$GITHUB_OUTPUT              
+            fi 
+            
+            if [[ $file == src/website/* ]]; then
+              echo "run_website=true" >>$GITHUB_OUTPUT
+            fi
+            
+            if [[ $file == k8s/website/* ]]; then
+              echo "run_website=true" >>$GITHUB_OUTPUT              
             fi            
 
             if [[ $file == k8s/nginx/staging/* ]]; then
@@ -296,6 +306,7 @@ jobs:
               echo "run_incentives=true" >>$GITHUB_OUTPUT
               echo "run_insights=true" >>$GITHUB_OUTPUT
               echo "run_spatial=true" >>$GITHUB_OUTPUT
+              echo "run_website=true" >>$GITHUB_OUTPUT
               echo "run_view=true" >>$GITHUB_OUTPUT
               echo "run_kafka_connectors=true" >>$GITHUB_OUTPUT
               echo "run_nginx=true" >>$GITHUB_OUTPUT
@@ -1723,6 +1734,66 @@ jobs:
           gcloud secrets versions access latest --secret="sta-key-analytics-service-account" > google_application_credentials.json
           kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json stage-spatial-api-config-files | kubectl replace -f - -n staging
 
+
+  ### website ###
+  website:
+    name: build-push-deploy-website
+    needs: [check, image-tag]
+    if: needs.check.outputs.run_website == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+
+      - name: Login to GCR
+        uses: docker/[email protected]
+        with:
+          registry: ${{ env.REGISTRY_URL }}
+          username: _json_key
+          password: ${{ secrets.GCR_CONFIG }}
+
+      - name: Login to K8S
+        uses: azure/[email protected]
+        with:
+          method: kubeconfig
+          kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }}
+
+      - name: Build and Push Docker Image
+        run: |
+          cd src/website/
+          docker build --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-website-api:${{ needs.image-tag.outputs.build_id }} .
+          docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-website-api:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-website-api:latest
+          docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-website-api:${{ needs.image-tag.outputs.build_id }}
+          docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-website-api:latest
+
+      - name: Update corresponding helm values file(with retry)
+        uses: Wandalen/[email protected] # Retries action on fail
+        with:
+          action: fjogeleit/yaml-update-action@main # Action to retry
+          with: |
+            valueFile: "k8s/website/values-stage.yaml"
+            propertyPath: "image.tag"
+            value: ${{ needs.image-tag.outputs.build_id }}
+            branch: ${{ env.DEPLOY_BRANCH }}
+            token: ${{ secrets.YAML_UPDATER_TOKEN }}
+            message: "Update website staging image tag to ${{ needs.image-tag.outputs.build_id }}"
+
+      - name: Login to GCP
+        uses: google-github-actions/[email protected]
+        with:
+          credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
+
+      - name: Setup Cloud SDK
+        uses: google-github-actions/[email protected]
+
+      - name: Update the corresponding k8s configmap(s)
+        run: |
+          cd src/website/
+          gcloud secrets versions access latest --secret="sta-env-website-backend" > .env
+          kubectl create configmap --dry-run=client -o yaml --from-env-file=.env stage-website-api-config | kubectl replace -f - -n staging
+          gcloud secrets versions access latest --secret="sta-key-analytics-service-account" > google_application_credentials.json
+          kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json stage-website-api-config-files | kubectl replace -f - -n staging          
+
   ### apply nginx ###
   nginx:
     name: apply-nginx

diff --git a/.github/workflows/deploy-previews.yml b/.github/workflows/deploy-previews.yml
@@ -37,6 +37,7 @@ jobs:
       run_calibrate: ${{ steps.check_files.outputs.run_calibrate }} # calibrate
       run_incentives: ${{ steps.check_files.outputs.run_incentives }} # incentives
       run_spatial: ${{ steps.check_files.outputs.run_spatial }} # spatial
+      run_website: ${{ steps.check_files.outputs.run_website }} # website
     runs-on: ubuntu-latest
     steps:
       - name: checkout code
@@ -67,6 +68,7 @@ jobs:
           echo "run_calibrate=false" >>$GITHUB_OUTPUT 
           echo "run_incentives=false" >>$GITHUB_OUTPUT
           echo "run_spatial=false" >>$GITHUB_OUTPUT
+          echo "run_website=false" >>$GITHUB_OUTPUT
 
           while IFS= read -r file
           do
@@ -123,6 +125,10 @@ jobs:
               echo "run_spatial=true" >>$GITHUB_OUTPUT
             fi
 
+            if [[ $file == src/website/* ]]; then
+              echo "run_website=true" >>$GITHUB_OUTPUT
+            fi            
+
 
           done < files.txt
 
@@ -1139,3 +1145,81 @@ jobs:
               repo: context.repo.repo,
               body: 'Spatial changes in this PR available for preview [here](${{ needs.spatial.outputs.url }})'
             })
+
+  ### website ###
+  website:
+    name: build-push-deploy-website
+    needs: [check, branch-name]
+    if: needs.check.outputs.run_website == 'true'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    outputs:
+      url: ${{ steps.preview-url.outputs.url }}
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+
+      - name: Google Auth
+        id: auth
+        uses: google-github-actions/[email protected]
+        with:
+          credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
+
+      - name: Setup Cloud SDK
+        uses: google-github-actions/[email protected]
+
+      - name: Docker Auth
+        id: docker-auth
+        uses: docker/[email protected]
+        with:
+          registry: ${{ env.REGISTRY_URL }}
+          username: _json_key
+          password: ${{ secrets.GCP_SA_CREDENTIALS }}
+
+      - name: Build and Push Container
+        run: |
+          cd src/website/
+          docker build --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/pr-previews/website-pr-previews:${{ github.sha }} ./
+          docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/pr-previews/website-pr-previews:${{ github.sha }}
+
+      - name: Deploy to Cloud Run
+        run: |-
+          gcloud run deploy ${{ needs.branch-name.outputs.lowercase }}-website-preview \
+            --region=${{ secrets.REGION }} \
+            --max-instances=10 \
+            --timeout=60 \
+            --concurrency=10 \
+            --image=${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/pr-previews/website-pr-previews:${{ github.sha }} \
+            --port=8000 \
+            --cpu=1000m \
+            --memory=1024Mi \
+            --update-secrets=/etc/env/.env=sta-env-website-backend:latest,/etc/config/google_application_credentials.json=sta-key-analytics-service-account:latest \
+            --command="/bin/sh","-c","cat /etc/env/.env >> /app/.env; /app/entrypoint.sh" \
+            --allow-unauthenticated
+
+      - name: Get preview service url
+        id: preview-url
+        run: |
+          read service_url < <(gcloud run services describe ${{ needs.branch-name.outputs.lowercase }}-website-preview \
+            --format='value(status.url)' \
+            --platform managed \
+            --region ${{ secrets.REGION }})
+          echo "url=${service_url}" >>$GITHUB_OUTPUT
+
+  website-pr-comment:
+    name: website-preview-link-comment
+    if: needs.check.outputs.run_website == 'true'
+    needs: [website]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/github-script@v6
+        with:
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: 'website changes in this PR available for preview [here](${{ needs.website.outputs.url }})'
+            })           
diff --git a/.github/workflows/remove-deploy-previews.yml b/.github/workflows/remove-deploy-previews.yml
@@ -35,6 +35,7 @@ jobs:
       remove_preview__calibrate: ${{ steps.check_files.outputs.remove_preview__calibrate }} # calibrate
       remove_preview__incentives: ${{ steps.check_files.outputs.remove_preview__incentives }} #incentives
       remove_preview__spatial: ${{ steps.check_files.outputs.remove_preview__spatial }} #spatial
+      remove_preview__website: ${{ steps.check_files.outputs.remove_preview__website }} #website
 
     runs-on: ubuntu-latest
     steps:
@@ -66,6 +67,7 @@ jobs:
           echo "remove_preview__calibrate=false" >>$GITHUB_OUTPUT 
           echo "remove_preview__incentives=false" >>$GITHUB_OUTPUT
           echo "remove_preview__spatial=false" >>$GITHUB_OUTPUT 
+          echo "remove_preview__website=false" >>$GITHUB_OUTPUT 
 
           while IFS= read -r file
           do
@@ -120,7 +122,11 @@ jobs:
             
             if [[ $file == src/spatial/* ]]; then
               echo "remove_preview__spatial=true" >>$GITHUB_OUTPUT
-            fi            
+            fi 
+            
+            if [[ $file == src/website/* ]]; then
+              echo "remove_preview__website=true" >>$GITHUB_OUTPUT
+            fi             
 
           done < files.txt
 
@@ -409,3 +415,25 @@ jobs:
           gcloud run services delete ${{ needs.branch-name.outputs.lowercase }}-spatial-preview \
             --region=${{ secrets.REGION }} \
             --quiet
+
+  ### website ###
+  website:
+    name: build-push-website
+    needs: [check, branch-name]
+    if: needs.check.outputs.remove_preview__website == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Google Auth
+        id: auth
+        uses: google-github-actions/[email protected]
+        with:
+          credentials_json: "${{ secrets.GCP_SA_CREDENTIALS }}"
+
+      - name: Setup Cloud SDK
+        uses: "google-github-actions/[email protected]"
+
+      - name: Delete PR deploy preview
+        run: |-
+          gcloud run services delete ${{ needs.branch-name.outputs.lowercase }}-website-preview \
+            --region=${{ secrets.REGION }} \
+            --quiet            
diff --git a/k8s/analytics/values-prod.yaml b/k8s/analytics/values-prod.yaml
@@ -8,7 +8,7 @@ images:
     celeryWorker: eu.gcr.io/airqo-250220/airqo-analytics-celery-worker
     reportJob: eu.gcr.io/airqo-250220/airqo-analytics-report-job
     devicesSummaryJob: eu.gcr.io/airqo-250220/airqo-analytics-devices-summary-job
-  tag: prod-c1581d38-1731588486
+  tag: prod-24272a02-1731937745
 api:
   name: airqo-analytics-api
   label: analytics-api

diff --git a/k8s/auth-service/values-prod.yaml b/k8s/auth-service/values-prod.yaml
@@ -6,7 +6,7 @@ app:
 replicaCount: 3
 image:
   repository: eu.gcr.io/airqo-250220/airqo-auth-api
-  tag: prod-c1581d38-1731588486
+  tag: prod-24272a02-1731937745
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

diff --git a/k8s/auth-service/values-stage.yaml b/k8s/auth-service/values-stage.yaml
@@ -6,7 +6,7 @@ app:
 replicaCount: 2
 image:
   repository: eu.gcr.io/airqo-250220/airqo-stage-auth-api
-  tag: stage-96bf4723-1731358545
+  tag: stage-e0815cc8-1731824148
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

diff --git a/k8s/calibrate/values-prod.yaml b/k8s/calibrate/values-prod.yaml
@@ -6,11 +6,11 @@ app:
 initContainer:
   image:
     repository: eu.gcr.io/airqo-250220/airqo-calibrate-pickle-file
-    tag: prod-c1581d38-1731588486
+    tag: prod-24272a02-1731937745
 replicaCount: 3
 image:
   repository: eu.gcr.io/airqo-250220/airqo-calibrate-api
-  tag: prod-c1581d38-1731588486
+  tag: prod-24272a02-1731937745
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

diff --git a/k8s/device-registry/values-prod.yaml b/k8s/device-registry/values-prod.yaml
@@ -6,7 +6,7 @@ app:
 replicaCount: 3
 image:
   repository: eu.gcr.io/airqo-250220/airqo-device-registry-api
-  tag: prod-c1581d38-1731588486
+  tag: prod-24272a02-1731937745
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

diff --git a/k8s/device-registry/values-stage.yaml b/k8s/device-registry/values-stage.yaml
@@ -6,7 +6,7 @@ app:
 replicaCount: 2
 image:
   repository: eu.gcr.io/airqo-250220/airqo-stage-device-registry-api
-  tag: stage-d40838b5-1731501583
+  tag: stage-4292371f-1731851989
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

diff --git a/k8s/exceedance/values-prod-airqo.yaml b/k8s/exceedance/values-prod-airqo.yaml
@@ -4,6 +4,6 @@ app:
   configmap: env-exceedance-production
 image:
   repository: eu.gcr.io/airqo-250220/airqo-exceedance-job
-  tag: prod-c1581d38-1731588486
+  tag: prod-24272a02-1731937745
 nameOverride: ''
 fullnameOverride: ''
diff --git a/k8s/exceedance/values-prod-kcca.yaml b/k8s/exceedance/values-prod-kcca.yaml
@@ -4,6 +4,6 @@ app:
   configmap: env-exceedance-production
 image:
   repository: eu.gcr.io/airqo-250220/kcca-exceedance-job
-  tag: prod-c1581d38-1731588486
+  tag: prod-24272a02-1731937745
 nameOverride: ''
 fullnameOverride: ''
diff --git a/k8s/locate/values-prod.yaml b/k8s/locate/values-prod.yaml
@@ -6,7 +6,7 @@ app:
 replicaCount: 3
 image:
   repository: eu.gcr.io/airqo-250220/airqo-locate-api
-  tag: prod-c1581d38-1731588486
+  tag: prod-24272a02-1731937745
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

diff --git a/k8s/nginx/staging/analytics-vs.yaml b/k8s/nginx/staging/analytics-vs.yaml
@@ -190,5 +190,6 @@ spec:
               - name: Content-Type
                 value: application/json  
 
+
 
 
diff --git a/k8s/nginx/staging/api-vs.yaml b/k8s/nginx/staging/api-vs.yaml
@@ -53,7 +53,7 @@ spec:
       port: 8080
     - name: spatial
       service: airqo-spatial-notification-api-svc
-      port: 5000      
+      port: 5000          
   routes:
     - path: ~ /api\/v[1-2]\/users
       action:
@@ -183,3 +183,4 @@ spec:
             add:
               - name: Content-Type
                 value: application/json                
+
diff --git a/k8s/nginx/staging/netmanager-vs.yaml b/k8s/nginx/staging/netmanager-vs.yaml
@@ -56,7 +56,7 @@ spec:
       port: 8080  
     - name: spatial
       service: airqo-stage-spatial-api-svc
-      port: 5000           
+      port: 5000                
   routes:
     - path: /
       action:
@@ -188,4 +188,5 @@ spec:
           responseHeaders:
             add:
               - name: Content-Type
-                value: application/json 
+                value: application/json 
+
Original file line number	Diff line number	Diff line change
Expand Up		@@ -190,5 +190,6 @@ spec:
		- name: Content-Type
		value: application/json