move to production #360
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: deploy-apis-to-production | |
| on: | |
| pull_request: | |
| branches: | |
| - master | |
| types: | |
| - closed | |
| env: | |
| REGISTRY_URL: eu.gcr.io | |
| PROJECT_ID: airqo-250220 | |
| DEPLOY_BRANCH: staging | |
| jobs: | |
| image-tag: | |
| if: github.event.pull_request.merged == true | |
| name: create image tag | |
| runs-on: ubuntu-latest | |
| outputs: | |
| build_id: ${{ steps.prep.outputs.build_id }} # build id | |
| datetime: ${{ steps.prep.outputs.datetime }} # build date | |
| steps: | |
| - name: generate build ID | |
| id: prep | |
| run: | | |
| sha=${GITHUB_SHA::8} | |
| timestamp=$(date +%s) | |
| datetime=$(date) | |
| echo "build_id=prod-${sha}-${timestamp}" >>$GITHUB_OUTPUT | |
| echo "datetime=${datetime}" >>$GITHUB_OUTPUT | |
| check: | |
| if: github.event.pull_request.merged == true | |
| name: check changed microservice(s) | |
| outputs: | |
| run_auth_service: ${{ steps.check_files.outputs.run_auth_service }} # auth service | |
| run_device_registry: ${{ steps.check_files.outputs.run_device_registry }} # device registry | |
| run_device_monitoring: ${{ steps.check_files.outputs.run_device_monitoring }} # device monitoring | |
| run_data_mgt: ${{ steps.check_files.outputs.run_data_mgt }} # data mgt | |
| run_analytics: ${{ steps.check_files.outputs.run_analytics }} # analytics | |
| run_device_uptime: ${{ steps.check_files.outputs.run_device_uptime }} # device uptime | |
| run_device_status: ${{ steps.check_files.outputs.run_device_status }} # device status | |
| run_locate: ${{ steps.check_files.outputs.run_locate }} # locate | |
| run_predict: ${{ steps.check_files.outputs.run_predict }} # predict | |
| run_gp_model: ${{ steps.check_files.outputs.run_gp_model }} # gp_model | |
| run_exceedances: ${{ steps.check_files.outputs.run_exceedances }} # exceedances | |
| run_meta_data: ${{ steps.check_files.outputs.run_meta_data }} # meta data | |
| run_view: ${{ steps.check_files.outputs.run_view }} # view | |
| run_calibrate: ${{ steps.check_files.outputs.run_calibrate }} # calibrate | |
| run_kafka_cluster_operator: ${{ steps.check_files.outputs.run_kafka_cluster_operator }} # kafka cluster operator | |
| run_kafka_cluster: ${{ steps.check_files.outputs.run_kafka_cluster }} # kafka cluster | |
| run_kafka_topics: ${{ steps.check_files.outputs.run_kafka_topics }} # kafka topics | |
| run_airflow: ${{ steps.check_files.outputs.run_airflow }} # airflow | |
| run_incentives: ${{ steps.check_files.outputs.run_incentives }} # incentives | |
| run_kafka_connectors: ${{ steps.check_files.outputs.run_kafka_connectors }} # kafka connectors | |
| run_nginx: ${{ steps.check_files.outputs.run_nginx }} # nginx | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: checkout code | |
| uses: actions/[email protected] | |
| with: | |
| fetch-depth: 2 | |
| #### all microservices ###### | |
| - name: check modified microserivce | |
| id: check_files | |
| run: | | |
| echo "=============== list modified files ===============" | |
| git diff --name-only HEAD^ HEAD | |
| echo "========== check paths of modified files ==========" | |
| git diff --name-only HEAD^ HEAD > files.txt | |
| echo "run_auth_service=false" >>$GITHUB_OUTPUT | |
| echo "run_device_registry=false" >>$GITHUB_OUTPUT | |
| echo "run_device_monitoring=false" >>$GITHUB_OUTPUT | |
| echo "run_data_mgt=false" >>$GITHUB_OUTPUT | |
| echo "run_analytics=false" >>$GITHUB_OUTPUT | |
| echo "run_device_uptime=false" >>$GITHUB_OUTPUT | |
| echo "run_device_status=false" >>$GITHUB_OUTPUT | |
| echo "run_locate=false" >>$GITHUB_OUTPUT | |
| echo "run_predict=false" >>$GITHUB_OUTPUT | |
| echo "run_gp_model=false" >>$GITHUB_OUTPUT | |
| echo "run_exceedances=false" >>$GITHUB_OUTPUT | |
| echo "run_meta_data=false" >>$GITHUB_OUTPUT | |
| echo "run_view=false" >>$GITHUB_OUTPUT | |
| echo "run_calibrate=false" >>$GITHUB_OUTPUT | |
| echo "run_kafka_cluster_operator=false" >>$GITHUB_OUTPUT | |
| echo "run_kafka_cluster=false" >>$GITHUB_OUTPUT | |
| echo "run_kafka_topics=false" >>$GITHUB_OUTPUT | |
| echo "run_airflow=false" >>$GITHUB_OUTPUT | |
| echo "run_incentives=false" >>$GITHUB_OUTPUT | |
| echo "run_kafka_connectors=false" >>$GITHUB_OUTPUT | |
| echo "run_nginx=false" >>$GITHUB_OUTPUT | |
| while IFS= read -r file | |
| do | |
| echo $file | |
| if [[ $file == src/auth-service/* ]]; then | |
| echo "run_auth_service=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/auth-service/* ]]; then | |
| echo "run_auth_service=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/device-registry/* ]]; then | |
| echo "run_device_registry=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/device-registry/* ]]; then | |
| echo "run_device_registry=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/device-monitoring/* ]]; then | |
| echo "run_device_monitoring=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/device-monitor/* ]]; then | |
| echo "run_device_monitoring=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/data-mgt/node/* ]]; then | |
| echo "run_data_mgt=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/data-mgt/* ]]; then | |
| echo "run_data_mgt=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/analytics/* ]]; then | |
| echo "run_analytics=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/analytics/* ]]; then | |
| echo "run_analytics=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/device-uptime/* ]]; then | |
| echo "run_device_uptime=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/device-uptime/* ]]; then | |
| echo "run_device_uptime=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/device-status/* ]]; then | |
| echo "run_device_status=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/device-status/* ]]; then | |
| echo "run_device_status=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/locate/* ]]; then | |
| echo "run_locate=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/locate/* ]]; then | |
| echo "run_locate=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/predict/* ]]; then | |
| echo "run_predict=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/predict/* ]]; then | |
| echo "run_predict=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/gp-model/* ]]; then | |
| echo "run_gp_model=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/gp-model/* ]]; then | |
| echo "run_gp_model=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/exceedances/* ]]; then | |
| echo "run_exceedances=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/exceedance/* ]]; then | |
| echo "run_exceedances=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/meta-data/* ]]; then | |
| echo "run_meta_data=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/meta-data/* ]]; then | |
| echo "run_meta_data=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/view/* ]]; then | |
| echo "run_view=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/view/* ]]; then | |
| echo "run_view=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/calibrate/* ]]; then | |
| echo "run_calibrate=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/calibrate/* ]]; then | |
| echo "run_calibrate=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/kafka/clusters/* ]]; then | |
| echo "run_kafka_cluster=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/kafka/operator/* ]]; then | |
| echo "run_kafka_cluster_operator=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/kafka/topics/* ]]; then | |
| echo "run_kafka_topics=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/kafka-connectors/* ]]; then | |
| echo "run_kafka_connectors=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/airflow/* ]]; then | |
| echo "run_airflow=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/airflow/* ]]; then | |
| echo "run_airflow=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/incentives/* ]]; then | |
| echo "run_incentives=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/incentives/* ]]; then | |
| echo "run_incentives=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/nginx/production/* ]]; then | |
| echo "run_nginx=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == workflow-trigger ]]; then | |
| echo "run_calibrate=true" >>$GITHUB_OUTPUT | |
| echo "run_meta_data=true" >>$GITHUB_OUTPUT | |
| echo "run_exceedances=true" >>$GITHUB_OUTPUT | |
| echo "run_gp_model=true" >>$GITHUB_OUTPUT | |
| echo "run_predict=true" >>$GITHUB_OUTPUT | |
| echo "run_locate=true" >>$GITHUB_OUTPUT | |
| echo "run_device_status=true" >>$GITHUB_OUTPUT | |
| echo "run_device_uptime=true" >>$GITHUB_OUTPUT | |
| echo "run_analytics=true" >>$GITHUB_OUTPUT | |
| echo "run_data_mgt=true" >>$GITHUB_OUTPUT | |
| echo "run_device_monitoring=true" >>$GITHUB_OUTPUT | |
| echo "run_device_registry=true" >>$GITHUB_OUTPUT | |
| echo "run_auth_service=true" >>$GITHUB_OUTPUT | |
| echo "run_airflow=true" >>$GITHUB_OUTPUT | |
| echo "run_incentives=true" >>$GITHUB_OUTPUT | |
| echo "run_view=true" >>$GITHUB_OUTPUT | |
| echo "run_kafka_connectors=true" >>$GITHUB_OUTPUT | |
| echo "run_nginx=true" >>$GITHUB_OUTPUT | |
| fi | |
| done < files.txt | |
| #### Kafka Connectors ###### | |
| kafka-connectors: | |
| name: build-push-deploy-kafka-connectors | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_kafka_connectors == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Set up QEMU | |
| uses: docker/[email protected] | |
| - name: Set up Docker Buildx | |
| uses: docker/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Build and push measurements source connectors | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/kafka-connectors/measurements-source-connector | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/measurements-connect:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/measurements-connect:latest | |
| - name: Build and push biq query connector | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/kafka-connectors/bigquery-connector | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/bigquery-connector:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/bigquery-connector:latest | |
| - name: Login to k8s cluster | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_PROD }} | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/kafka/BigQuery-connectors/values-prod.yaml" | |
| propertyPath: "image.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update BigQuery-connectors production image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Deploy to k8s cluster | |
| run: | | |
| export MEASUREMENTS_CONNECT_IMAGE=${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/measurements-connect:${{ needs.image-tag.outputs.build_id }} | |
| export PROD_KAFKA_CLUSTER=${{ secrets.PROD_KAFKA_CLUSTER }} | |
| cat k8s/kafka/connectors/measurements-connect.yaml | sed "s/{{MEASUREMENTS_CONNECT_IMAGE}}/$MEASUREMENTS_CONNECT_IMAGE/g" | sed "s/{{KAFKA_CLUSTER}}/$PROD_KAFKA_CLUSTER/g" | kubectl apply -n message-broker -f- | |
| kubectl apply -f k8s/kafka/connectors/purple-air-connector.yaml -n message-broker | |
| - name: Set up GCP credentials and Kubernetes configmaps | |
| run : | | |
| gcloud secrets versions access latest --secret="prod-env-kafka-bigquery-connectors" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env bigquery-connectors | kubectl replace -f - -n production | |
| gcloud secrets versions access latest --secret="prod-key-kafka-bigquery-connectors" > google_application_credentials.json | |
| kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json bigquery-connectors-files | kubectl replace -f - -n production | |
| ### auth service ### | |
| auth-service: | |
| name: build-push-deploy-auth | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_auth_service == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_PROD }} | |
| - name: Build and Push Docker Image | |
| run: | | |
| cd src/auth-service/ | |
| docker build --target=production --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-auth-api:${{ needs.image-tag.outputs.build_id }} . | |
| docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-auth-api:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-auth-api:latest | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-auth-api:${{ needs.image-tag.outputs.build_id }} | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-auth-api:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/auth-service/values-prod.yaml" | |
| propertyPath: "image.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update auth service production image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/auth-service/ | |
| gcloud secrets versions access latest --secret="prod-env-auth-service" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env prod-auth-api-config | kubectl replace -f - -n production | |
| gcloud secrets versions access latest --secret="prod-key-auth-service-firebase-admin-sdk" > firebase_admin_sdk.json | |
| kubectl create configmap --dry-run=client -o yaml --from-file=firebase_admin_sdk.json prod-auth-api-config-files | kubectl replace -f - -n production | |
| ### airflow ### | |
| airflow: | |
| name: build-push-deploy-airflow | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_airflow == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Set up QEMU | |
| uses: docker/[email protected] | |
| - name: Set up Docker Buildx | |
| uses: docker/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Build and push app | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/airflow | |
| target: deployment | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-apache-airflow:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-apache-airflow:latest | |
| - name: Build and push XCom | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/airflow | |
| target: xcom-setup | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-apache-airflow-xcom:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-apache-airflow-xcom:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/airflow/values-prod.yaml" | |
| propertyPath: "images.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update airflow prod image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_PROD }} | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/airflow/ | |
| gcloud secrets versions access latest --secret="prod-env-airflow" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env airflow-config | kubectl replace -f - -n pipeline | |
| #### Device Registry ###### | |
| device-registry: | |
| name: build-push-deploy-device-registry | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_device_registry == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_PROD }} | |
| - name: Build and Push Docker Image | |
| run: | | |
| cd src/device-registry/ | |
| docker build --target=production --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-registry-api:${{ needs.image-tag.outputs.build_id }} . | |
| docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-registry-api:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-registry-api:latest | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-registry-api:${{ needs.image-tag.outputs.build_id }} | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-registry-api:latest | |
| - name: Update corresponding helm values file | |
| uses: Wandalen/[email protected] | |
| with: | |
| action: fjogeleit/yaml-update-action@main | |
| with: | | |
| valueFile: "k8s/device-registry/values-prod.yaml" | |
| propertyPath: "image.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update device registry production image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/device-registry/ | |
| gcloud secrets versions access latest --secret="prod-env-device-registry" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-device-registry-production | kubectl replace -f - -n production | |
| gcloud secrets versions access latest --secret="prod-key-device-registry-service-account" > google_application_credentials.json | |
| kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json device-registry-config-files | kubectl replace -f - -n production | |
| ### device monitoring ### | |
| device-monitoring: | |
| name: build-push-deploy-device-monitoring | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_device_monitoring == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_PROD }} | |
| - name: Build and push API | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/device-monitoring/ | |
| target: production | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-monitor-api:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-monitor-api:latest | |
| - name: Build and push celery-beat | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/device-monitoring/ | |
| target: celery-beat | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-monitor-celery-beat:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-monitor-celery-beat:latest | |
| - name: Build and push celery-worker | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/device-monitoring/ | |
| target: celery-worker | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-monitor-celery-worker:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-monitor-celery-worker:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/device-monitor/values-prod.yaml" | |
| propertyPath: "images.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update device monitor production images' tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/device-monitoring/ | |
| gcloud secrets versions access latest --secret="prod-env-device-monitoring" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-device-monitoring-production | kubectl replace -f - -n production | |
| gcloud secrets versions access latest --secret="prod-key-device-monitoring-service-account" > google_application_credentials.json | |
| kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json device-monitor-config-files | kubectl replace -f - -n production | |
| ### data-mgt ### | |
| data-mgt: | |
| name: build-push-deploy-data-mgt | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_data_mgt == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_PROD }} | |
| - name: Build and Push Docker Image | |
| run: | | |
| cd src/data-mgt/node/ | |
| docker build --target=production --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-data-mgt-api:${{ needs.image-tag.outputs.build_id }} . | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-data-mgt-api:${{ needs.image-tag.outputs.build_id }} | |
| docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-data-mgt-api:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-data-mgt-api:latest | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-data-mgt-api:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/data-mgt/values-prod.yaml" | |
| propertyPath: "image.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update data mgt production image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/data-mgt/node/ | |
| gcloud secrets versions access latest --secret="prod-env-data-mgt-nodejs" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env data-mgt-api-config | kubectl replace -f - -n production | |
| ### analytics ### | |
| analytics: | |
| name: build-push-deploy-analytics | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_analytics == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| ### run unit tests ### | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_PROD }} | |
| - name: Build and push API Docker Image | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/analytics/ | |
| target: production | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-analytics-api:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-analytics-api:latest | |
| - name: Build and push Celery beat Docker Image | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/analytics/ | |
| target: celery-beat | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-analytics-celery-beat:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-analytics-celery-beat:latest | |
| - name: Build and push Celery worker Docker Image | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/analytics/ | |
| target: celery-worker | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-analytics-celery-worker:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-analytics-celery-worker:latest | |
| - name: Build and push device summary job Docker Image | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/analytics/ | |
| target: devices-summary-job | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-analytics-devices-summary-job:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-analytics-devices-summary-job:latest | |
| - name: Build and push reports job Docker Image | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/analytics/jobs/reports | |
| target: production | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-analytics-report-job:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-analytics-report-job:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/analytics/values-prod.yaml" | |
| propertyPath: "images.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update analytics production image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/analytics/ | |
| gcloud secrets versions access latest --secret="prod-env-analytics" > api.env | |
| gcloud secrets versions access latest --secret="prod-env-analytics-report-job" > reports.env | |
| gcloud secrets versions access latest --secret="prod-key-analytics-service-account" > google_application_credentials.json | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=api.env env-analytics-production | kubectl replace -f - -n production | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=reports.env env-analytics-report-production | kubectl replace -f - -n production | |
| kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json prod-analytics-config-files | kubectl replace -f - -n production | |
| ### device uptime ### | |
| device-uptime: | |
| name: build-push-deploy-device-uptime | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_device_uptime == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| ### run unit tests ### | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_PROD }} | |
| - name: Build and push device uptime v1 docker image | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/device-uptime/ | |
| target: production | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-uptime-job:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-uptime-job:latest | |
| - name: Build and push device uptime v2 docker image | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/device-uptime/ | |
| target: uptime | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-uptime-job-v2:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-uptime-job-v2:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/device-uptime/values-prod.yaml" | |
| propertyPath: "image.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update device uptime production image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/device-uptime/ | |
| gcloud secrets versions access latest --secret="prod-env-device-uptime" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-device-uptime-production | kubectl replace -f - -n production | |
| gcloud secrets versions access latest --secret="prod-key-device-uptime-v2-service-account" > google_application_credentials.json | |
| kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json device-uptime-v2-files | kubectl replace -f - -n production | |
| ### device status ### | |
| device-status: | |
| name: build-push-deploy-device-status | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_device_status == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| ### run unit tests ### | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_PROD }} | |
| - name: Build and Push Docker Image | |
| run: | | |
| cd src/device-status/ | |
| docker build --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-status-job:${{ needs.image-tag.outputs.build_id }} -f Dockerfile . | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-status-job:${{ needs.image-tag.outputs.build_id }} | |
| docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-status-job:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-status-job:latest | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-status-job:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/device-status/values-prod.yaml" | |
| propertyPath: "image.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update device status production image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/device-status/ | |
| gcloud secrets versions access latest --secret="prod-env-device-status" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-device-status-production | kubectl replace -f - -n production | |
| ### kafka cluster operator ### | |
| kafka-cluster-operator: | |
| name: build-push-deploy-kafka-cluster-operator | |
| needs: [check] | |
| if: needs.check.outputs.run_kafka_cluster_operator == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_PROD }} | |
| - name: Deploy to K8S | |
| run: | | |
| kubectl apply -f k8s/namespaces/message-broker.yaml | |
| bash k8s/kafka/operator/deployment-script.sh | |
| ### kafka cluster ### | |
| kafka-cluster: | |
| name: build-push-deploy-kafka-cluster | |
| needs: [check] | |
| if: ${{ (needs.check.outputs.run_kafka_cluster == 'true') || (needs.check.outputs.run_kafka_cluster_operator == 'true') }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_PROD }} | |
| - name: Deploy to K8S | |
| run: | | |
| export CLUSTER_CONTROLLER_IP=${{ secrets.PROD_CLUSTER_CONTROLLER_IP}} | |
| sed -i 's/CLUSTER_CONTROLLER_IP/'"$CLUSTER_CONTROLLER_IP"'/g' k8s/kafka/clusters/kafka-cluster.yaml | |
| kubectl apply -n message-broker -f k8s/kafka/clusters/kafka-cluster.yaml | |
| kubectl rollout restart deployment kafka-cluster-entity-operator -n message-broker | |
| kubectl rollout restart deployment strimzi-cluster-operator -n message-broker | |
| ### kafka topics ### | |
| kafka-topics: | |
| name: build-push-deploy-kafka-topics | |
| needs: [check] | |
| if: ${{ (needs.check.outputs.run_kafka_cluster == 'true') || (needs.check.outputs.run_kafka_topics == 'true') }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_PROD }} | |
| - name: Deploy to K8S | |
| run: | | |
| kubectl apply -f k8s/kafka/topics/kafka-topics.yaml -n message-broker | |
| ### locate ### | |
| locate: | |
| name: build-push-deploy-locate | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_locate == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| ### run unit tests ### | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_PROD }} | |
| - name: Build and Push Docker Image | |
| run: | | |
| cd src/locate/ | |
| docker build --target=production --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-locate-api:${{ needs.image-tag.outputs.build_id }} . | |
| docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-locate-api:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-locate-api:latest | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-locate-api:${{ needs.image-tag.outputs.build_id }} | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-locate-api:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/locate/values-prod.yaml" | |
| propertyPath: "image.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update locate production image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/locate/ | |
| gcloud secrets versions access latest --secret="prod-env-locate-api" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-locate-production | kubectl replace -f - -n production | |
| ### gp model ### | |
| gp-model: | |
| name: build-push-deploy-gp-model | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_gp_model == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_PROD }} | |
| - name: Build and push cron job | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/gp-model | |
| target: production | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-gp-model-job:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-gp-model-job:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/gp-model/values-prod.yaml" | |
| propertyPath: "images.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update gp-model production image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/gp-model/ | |
| gcloud secrets versions access latest --secret="prod-env-gp-model" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-gp-model-production | kubectl replace -f - -n production | |
| gcloud secrets versions access latest --secret="prod-key-gp-model-service-account" > google_application_credentials.json | |
| kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json gp-model-job-config-files | kubectl replace -f - -n production | |
| ### exceddences ### | |
| exceedance: | |
| name: build-push-deploy-exceedances | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_exceedances == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| ### run unit tests ### | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_PROD }} | |
| - name: Build and push AirQo Docker image | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/exceedances | |
| target: airqo-production | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-exceedance-job:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-exceedance-job:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/exceedance/values-prod-airqo.yaml" | |
| propertyPath: "image.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update AirQo exceedance production image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Build and push KCCA Docker image | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/exceedances | |
| target: kcca-production | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/kcca-exceedance-job:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/kcca-exceedance-job:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/exceedance/values-prod-kcca.yaml" | |
| propertyPath: "image.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update KCCA exceedance production image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/exceedances/ | |
| gcloud secrets versions access latest --secret="prod-env-exceedances" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-exceedance-production | kubectl replace -f - -n production | |
| ### meta-data ### | |
| meta-data: | |
| name: build-push-deploy-meta-data | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_meta_data == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| ### run unit tests ### | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_PROD }} | |
| - name: Build and push API | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/meta-data | |
| target: api | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-meta-data-api:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-meta-data-api:latest | |
| - name: Build and push sites consumer | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/meta-data | |
| target: sites-consumer | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-meta-data-sites-consumer:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-meta-data-sites-consumer:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/meta-data/values-prod.yaml" | |
| propertyPath: "images.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update meta-data production image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/meta-data/ | |
| gcloud secrets versions access latest --secret="prod-env-metadata" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env prod-meta-data-api-config | kubectl replace -f - -n production | |
| gcloud secrets versions access latest --secret="prod-key-metadata-service-account" > google_application_credentials.json | |
| kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json prod-meta-data-api-config-files | kubectl replace -f - -n production | |
| ### view ### | |
| view: | |
| name: build-push-deploy-view | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_view == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Set up QEMU | |
| uses: docker/[email protected] | |
| - name: Set up Docker Buildx | |
| uses: docker/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_PROD }} | |
| - name: Build and push API | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-view-api:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-view-api:latest | |
| target: api | |
| context: src/view | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/view/values-prod.yaml" | |
| propertyPath: "images.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update view production image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/view/ | |
| gcloud secrets versions access latest --secret="prod-env-view-api" > api.env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=api.env prod-view-api-config | kubectl replace -f - -n production | |
| gcloud secrets versions access latest --secret="prod-key-view-api-google-credentials" > google_application_credentials.json | |
| kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json view-api-config-files | kubectl replace -f - -n production | |
| #### Calibrate ###### | |
| calibrate: | |
| name: build-push-deploy-calibrate | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_calibrate == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Set up QEMU | |
| uses: docker/[email protected] | |
| - name: Set up Docker Buildx | |
| uses: docker/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Build and push calibrate API | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/calibrate | |
| target: production | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-calibrate-api:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-calibrate-api:latest | |
| - name: Build and push calibrate pickle file | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/calibrate | |
| target: pickle-file | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-calibrate-pickle-file:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-calibrate-pickle-file:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/calibrate/values-prod.yaml" | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update calibrate production tags to ${{ needs.image-tag.outputs.build_id }}" | |
| changes: | | |
| { | |
| "image.tag": "${{ needs.image-tag.outputs.build_id }}", | |
| "initContainer.image.tag": "${{ needs.image-tag.outputs.build_id }}" | |
| } | |
| - name: Login to Main k8s cluster | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_PROD }} | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/calibrate/ | |
| gcloud secrets versions access latest --secret="prod-env-calibrate-api" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env prod-calibrate-api-config | kubectl replace -f - -n production | |
| gcloud secrets versions access latest --secret="prod-key-calibrate-service-account" > google_application_credentials.json | |
| kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json prod-calibrate-api-config-files | kubectl replace -f - -n production | |
| ### predict api and jobs ### | |
| predict-service: | |
| name: build-push-deploy-predict-service | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_predict == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Set up QEMU | |
| uses: docker/[email protected] | |
| - name: Set up Docker Buildx | |
| uses: docker/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_PROD }} | |
| - name: Build and push API Docker Image | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/predict/api | |
| target: production | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-prediction-api:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-prediction-api:latest | |
| - name: Build and push predict places image | |
| uses: docker/build-push-action@v3 | |
| with: | |
| push: true | |
| context: src/predict/jobs/predict_places_air_quality | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-predict-places-air-quality:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-predict-places-air-quality:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/predict/values-prod.yaml" | |
| propertyPath: "images.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update predict production image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/predict/ | |
| echo "=============== Creating the required secret files ===============" | |
| gcloud secrets versions access latest --secret="prod-env-predict-api" > api.env | |
| gcloud secrets versions access latest --secret="prod-env-predict-predict-places-air-quality" > predict_places_air_quality.env | |
| gcloud secrets versions access latest --secret="prod-key-predict-service-account" > google_application_credentials.json | |
| echo "=============== Updating the respective .env files ===============" | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=api.env env-predict-production | kubectl replace -f - -n production | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=predict_places_air_quality.env env-predict-places-air-quality-production | kubectl replace -f - -n production | |
| echo "=============== Updating the respective credentials files ===============" | |
| kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json env-predict-production-files | kubectl replace -f - -n production | |
| ### incentives ### | |
| incentives: | |
| name: build-push-incentives | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_incentives == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_PROD }} | |
| - name: Build and Push Docker Image | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/incentives/ | |
| target: production | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-incentives-api:${{ needs.image-tag.outputs.build_id }}, ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-incentives-api:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/incentives/values-prod.yaml" | |
| propertyPath: "image.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update incentives production image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/incentives/ | |
| gcloud secrets versions access latest --secret="prod-env-incentives" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env prod-incentives-api-config | kubectl replace -f - -n production | |
| ### apply nginx ### | |
| nginx: | |
| name: apply-nginx | |
| needs: [check] | |
| if: needs.check.outputs.run_nginx == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_PROD }} | |
| - name: Apply nginx changes | |
| run: kubectl apply -f k8s/nginx/production/ |