AN-297-Adding unit tests for Device Registry. #2414
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: deploy-apis-to-staging | |
| on: | |
| pull_request: | |
| branches: | |
| - staging | |
| types: | |
| - closed | |
| env: | |
| REGISTRY_URL: eu.gcr.io | |
| PROJECT_ID: airqo-250220 | |
| DEPLOY_BRANCH: staging | |
| jobs: | |
| image-tag: | |
| if: github.event.pull_request.merged == true | |
| name: create image tag | |
| runs-on: ubuntu-latest | |
| outputs: | |
| build_id: ${{ steps.prep.outputs.build_id }} # build id | |
| datetime: ${{ steps.prep.outputs.datetime }} # build date | |
| steps: | |
| - name: generate build ID | |
| id: prep | |
| run: | | |
| sha=${GITHUB_SHA::8} | |
| timestamp=$(date +%s) | |
| datetime=$(date) | |
| echo "build_id=stage-${sha}-${timestamp}" >>$GITHUB_OUTPUT | |
| echo "datetime=${datetime}" >>$GITHUB_OUTPUT | |
| check: | |
| if: github.event.pull_request.merged == true | |
| name: check changed microservice(s) | |
| outputs: | |
| run_auth_service: ${{ steps.check_files.outputs.run_auth_service }} # auth service | |
| run_device_registry: ${{ steps.check_files.outputs.run_device_registry }} # device registry | |
| run_device_monitoring: ${{ steps.check_files.outputs.run_device_monitoring }} # device monitoring | |
| run_data_mgt: ${{ steps.check_files.outputs.run_data_mgt }} # data mgt | |
| run_analytics: ${{ steps.check_files.outputs.run_analytics }} # analytics | |
| run_device_uptime: ${{ steps.check_files.outputs.run_device_uptime }} # device uptime | |
| run_device_status: ${{ steps.check_files.outputs.run_device_status }} # device status | |
| run_locate: ${{ steps.check_files.outputs.run_locate }} # locate | |
| run_predict: ${{ steps.check_files.outputs.run_predict }} # predict | |
| run_gp_model: ${{ steps.check_files.outputs.run_gp_model }} # gp_model | |
| run_exceedances: ${{ steps.check_files.outputs.run_exceedances }} # exceedances | |
| run_meta_data: ${{ steps.check_files.outputs.run_meta_data }} # meta data | |
| run_view: ${{ steps.check_files.outputs.run_view }} # view | |
| run_calibrate: ${{ steps.check_files.outputs.run_calibrate }} # calibrate | |
| run_average_jobs: ${{ steps.check_files.outputs.run_average_jobs }} # average and calibration jobs | |
| run_events_check: ${{ steps.check_files.outputs.run_events_check }} # event checks | |
| run_notifications: ${{ steps.check_files.outputs.run_notifications }} # notification | |
| run_kafka_cluster_operator: ${{ steps.check_files.outputs.run_kafka_cluster_operator }} # kafka cluster operator | |
| run_kafka_cluster: ${{ steps.check_files.outputs.run_kafka_cluster }} # kafka cluster | |
| run_kafka_topics: ${{ steps.check_files.outputs.run_kafka_topics }} # kafka topics | |
| run_airflow: ${{ steps.check_files.outputs.run_airflow }} # airflow | |
| run_network_uptime: ${{ steps.check_files.outputs.run_network_uptime }} # network_uptime | |
| run_channel_data: ${{ steps.check_files.outputs.run_channel_data }} # channel data job | |
| run_kafka_connectors: ${{ steps.check_files.outputs.run_kafka_connectors }} # kafka connectors | |
| run_fault_detection: ${{ steps.check_files.outputs.run_fault_detection }} # fault detection | |
| run_nginx: ${{ steps.check_files.outputs.run_nginx }} # nginx ingress | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: checkout code | |
| uses: actions/[email protected] | |
| with: | |
| fetch-depth: 2 | |
| #### all microservices ###### | |
| - name: check modified microserivce | |
| id: check_files | |
| run: | | |
| echo "=============== list modified files ===============" | |
| git diff --name-only HEAD^ HEAD | |
| echo "========== check paths of modified files ==========" | |
| git diff --name-only HEAD^ HEAD > files.txt | |
| echo "run_auth_service=false" >>$GITHUB_OUTPUT | |
| echo "run_device_registry=false" >>$GITHUB_OUTPUT | |
| echo "run_device_monitoring=false" >>$GITHUB_OUTPUT | |
| echo "run_data_mgt=false" >>$GITHUB_OUTPUT | |
| echo "run_analytics=false" >>$GITHUB_OUTPUT | |
| echo "run_device_uptime=false" >>$GITHUB_OUTPUT | |
| echo "run_device_status=false" >>$GITHUB_OUTPUT | |
| echo "run_locate=false" >>$GITHUB_OUTPUT | |
| echo "run_predict=false" >>$GITHUB_OUTPUT | |
| echo "run_gp_model=false" >>$GITHUB_OUTPUT | |
| echo "run_exceedances=false" >>$GITHUB_OUTPUT | |
| echo "run_meta_data=false" >>$GITHUB_OUTPUT | |
| echo "run_view=false" >>$GITHUB_OUTPUT | |
| echo "run_calibrate=false" >>$GITHUB_OUTPUT | |
| echo "run_average_jobs=false" >>$GITHUB_OUTPUT | |
| echo "run_events_check=false" >>$GITHUB_OUTPUT | |
| echo "run_notifications=false" >>$GITHUB_OUTPUT | |
| echo "run_kafka_cluster_operator=false" >>$GITHUB_OUTPUT | |
| echo "run_kafka_cluster=false" >>$GITHUB_OUTPUT | |
| echo "run_kafka_topics=false" >>$GITHUB_OUTPUT | |
| echo "run_airflow=false" >>$GITHUB_OUTPUT | |
| echo "run_network_uptime=false" >>$GITHUB_OUTPUT | |
| echo "run_channel_data=false" >>$GITHUB_OUTPUT | |
| echo "run_kafka_connectors=false" >>$GITHUB_OUTPUT | |
| echo "run_fault_detection=false" >>$GITHUB_OUTPUT | |
| echo "run_nginx=false" >>$GITHUB_OUTPUT | |
| while IFS= read -r file | |
| do | |
| echo $file | |
| if [[ $file == src/auth-service/* ]]; then | |
| echo "run_auth_service=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/auth-service/* ]]; then | |
| echo "run_auth_service=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/device-registry/* ]]; then | |
| echo "run_device_registry=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/device-registry/* ]]; then | |
| echo "run_device_registry=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/device-monitoring/* ]]; then | |
| echo "run_device_monitoring=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/device-monitor/* ]]; then | |
| echo "run_device_monitoring=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/data-mgt/node/* ]]; then | |
| echo "run_data_mgt=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/data-mgt/* ]]; then | |
| echo "run_data_mgt=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/analytics/* ]]; then | |
| echo "run_analytics=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/analytics/* ]]; then | |
| echo "run_analytics=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/device-uptime/* ]]; then | |
| echo "run_device_uptime=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/device-uptime/* ]]; then | |
| echo "run_device_uptime=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/device-status/* ]]; then | |
| echo "run_device_status=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/device-status/* ]]; then | |
| echo "run_device_status=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/locate/* ]]; then | |
| echo "run_locate=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/locate/* ]]; then | |
| echo "run_locate=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/predict/* ]]; then | |
| echo "run_predict=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/predict/* ]]; then | |
| echo "run_predict=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/gp-model/* ]]; then | |
| echo "run_gp_model=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/gp-model/* ]]; then | |
| echo "run_gp_model=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/exceedances/* ]]; then | |
| echo "run_exceedances=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/exceedance/* ]]; then | |
| echo "run_exceedances=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/meta-data/* ]]; then | |
| echo "run_meta_data=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/meta-data/* ]]; then | |
| echo "run_meta_data=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/view/* ]]; then | |
| echo "run_view=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/view/* ]]; then | |
| echo "run_view=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/calibrate/* ]]; then | |
| echo "run_calibrate=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/calibrate/* ]]; then | |
| echo "run_calibrate=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/data-mgt/python/cron-jobs/* ]]; then | |
| echo "run_average_jobs=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/data-mgt/python/events-monitor/* ]]; then | |
| echo "run_events_check=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/notifications/* ]]; then | |
| echo "run_notifications=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/notification-service/* ]]; then | |
| echo "run_notifications=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/kafka/clusters/* ]]; then | |
| echo "run_kafka_cluster=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/kafka/operator/* ]]; then | |
| echo "run_kafka_cluster_operator=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/kafka/topics/* ]]; then | |
| echo "run_kafka_topics=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/kafka-connectors/* ]]; then | |
| echo "run_kafka_connectors=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/airflow/* ]]; then | |
| echo "run_airflow=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/airflow/* ]]; then | |
| echo "run_airflow=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/network-uptime/* ]]; then | |
| echo "run_network_uptime=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/network-uptime/* ]]; then | |
| echo "run_network_uptime=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/channel-data/* ]]; then | |
| echo "run_channel_data=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/channel-data/* ]]; then | |
| echo "run_channel_data=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == src/fault-detection/* ]]; then | |
| echo "run_fault_detection=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/fault-detection/* ]]; then | |
| echo "run_fault_detection=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == k8s/nginx/staging/* ]]; then | |
| echo "run_nginx=true" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ $file == workflow-trigger ]]; then | |
| echo "run_notifications=true" >>$GITHUB_OUTPUT | |
| echo "run_events_check=true" >>$GITHUB_OUTPUT | |
| echo "run_calibrate=true" >>$GITHUB_OUTPUT | |
| echo "run_average_jobs=true" >>$GITHUB_OUTPUT | |
| echo "run_meta_data=true" >>$GITHUB_OUTPUT | |
| echo "run_exceedances=true" >>$GITHUB_OUTPUT | |
| echo "run_gp_model=true" >>$GITHUB_OUTPUT | |
| echo "run_predict=true" >>$GITHUB_OUTPUT | |
| echo "run_locate=true" >>$GITHUB_OUTPUT | |
| echo "run_device_status=true" >>$GITHUB_OUTPUT | |
| echo "run_device_uptime=true" >>$GITHUB_OUTPUT | |
| echo "run_analytics=true" >>$GITHUB_OUTPUT | |
| echo "run_data_mgt=true" >>$GITHUB_OUTPUT | |
| echo "run_device_monitoring=true" >>$GITHUB_OUTPUT | |
| echo "run_device_registry=true" >>$GITHUB_OUTPUT | |
| echo "run_auth_service=true" >>$GITHUB_OUTPUT | |
| echo "run_airflow=true" >>$GITHUB_OUTPUT | |
| echo "run_network_uptime=true" >>$GITHUB_OUTPUT | |
| echo "run_channel_data=true" >>$GITHUB_OUTPUT | |
| echo "run_fault_detection=true" >>$GITHUB_OUTPUT | |
| echo "run_view=true" >>$GITHUB_OUTPUT | |
| echo "run_kafka_connectors=true" >>$GITHUB_OUTPUT | |
| echo "run_nginx=true" >>$GITHUB_OUTPUT | |
| fi | |
| done < files.txt | |
| ### auth service ### | |
| auth-service: | |
| name: build-push-deploy-auth | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_auth_service == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Build and Push Docker Image | |
| run: | | |
| cd src/auth-service/ | |
| docker build --target=staging --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-auth-api:${{ needs.image-tag.outputs.build_id }} . | |
| docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-auth-api:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-auth-api:latest | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-auth-api:${{ needs.image-tag.outputs.build_id }} | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-auth-api:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/auth-service/values-stage.yaml" | |
| propertyPath: "image.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update auth service staging image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/auth-service/ | |
| gcloud secrets versions access latest --secret="sta-env-auth-service" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env stage-auth-api-config | kubectl replace -f - -n staging | |
| gcloud secrets versions access latest --secret="sta-key-auth-service-firebase-admin-sdk" > firebase_admin_sdk.json | |
| kubectl create configmap --dry-run=client -o yaml --from-file=firebase_admin_sdk.json stage-auth-api-config-files | kubectl replace -f - -n staging | |
| #### Kafka Connectors ###### | |
| kafka-connectors: | |
| name: build-push-deploy-kafka-connectors | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_kafka_connectors == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Set up QEMU | |
| uses: docker/[email protected] | |
| - name: Set up Docker Buildx | |
| uses: docker/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Build and push measurements source connectors | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/kafka-connectors/measurements-source-connector | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/stage-measurements-connect:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/stage-measurements-connect:latest | |
| - name: Build and push biq query connector | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/kafka-connectors/bigquery-connector | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/stage-bigquery-connector:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/stage-bigquery-connector:latest | |
| - name: Login to k8s cluster | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/kafka/BigQuery-connectors/values-stage.yaml" | |
| propertyPath: "image.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update BigQuery-connectors staging image tag to ${{ needs.image-tag.outputs.build_id }} | |
| - name: Deploy to k8s cluster | |
| run: | | |
| export MEASUREMENTS_CONNECT_IMAGE=${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/stage-measurements-connect:${{ needs.image-tag.outputs.build_id }} | |
| export STAGE_KAFKA_CLUSTER=${{ secrets.STAGE_KAFKA_CLUSTER }} | |
| cat k8s/kafka/connectors/measurements-connect.yaml | sed "s/{{MEASUREMENTS_CONNECT_IMAGE}}/$MEASUREMENTS_CONNECT_IMAGE/g" | sed "s/{{KAFKA_CLUSTER}}/$STAGE_KAFKA_CLUSTER/g" | kubectl apply -n message-broker -f- | |
| kubectl apply -f k8s/kafka/connectors/purple-air-connector.yaml -n message-broker | |
| - name: Set up GCP credentials and Kubernetes configmaps | |
| run : | | |
| gcloud secrets versions access latest --secret="sta-env-kafka-bigquery-connectors" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env bigquery-connectors | kubectl replace -f - -n staging | |
| gcloud secrets versions access latest --secret="sta-key-kafka-bigquery-connectors" > google_application_credentials.json | |
| kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json bigquery-connectors-files | kubectl replace -f - -n staging | |
| #### Fault Detection ###### | |
| fault-detection: | |
| name: build-push-deploy-fault-detection | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_fault_detection == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Set up QEMU | |
| uses: docker/[email protected] | |
| - name: Set up Docker Buildx | |
| uses: docker/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Build and push train catboost model | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/fault-detection | |
| target: train-catboost-model | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/stage-train-catboost-model:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/stage-train-catboost-model:latest | |
| - name: Build and push train lstm model | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/fault-detection | |
| target: train-lstm-model | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/stage-train-lstm-model:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/stage-train-lstm-model:latest | |
| - name: Build and push fault detection API | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/fault-detection | |
| target: staging | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/stage-fault-detection-api:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/stage-fault-detection-api:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/fault-detection/values-stage.yaml" | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update fault detection staging tags to ${{ needs.image-tag.outputs.build_id }}" | |
| changes: | | |
| { | |
| "image.tag": "${{ needs.image-tag.outputs.build_id }}", | |
| "initContainers.container1.image.tag": "${{ needs.image-tag.outputs.build_id }}", | |
| "initContainers.container2.image.tag": "${{ needs.image-tag.outputs.build_id }}" | |
| } | |
| - name: Login to k8s cluster | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/fault-detection/ | |
| gcloud secrets versions access latest --secret="sta-env-fault-detection" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env stage-fault-detection-api-config | kubectl replace -f - -n staging | |
| ### airflow ### | |
| airflow: | |
| name: build-push-deploy-airflow | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_airflow == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Set up QEMU | |
| uses: docker/[email protected] | |
| - name: Set up Docker Buildx | |
| uses: docker/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Build and push app | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/airflow | |
| target: deployment | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-apache-airflow:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-apache-airflow:latest | |
| - name: Build and push XCom | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/airflow | |
| target: xcom-setup | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-apache-airflow-xcom:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-apache-airflow-xcom:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/airflow/values-stage.yaml" | |
| propertyPath: "images.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update airflow staging image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/airflow/ | |
| gcloud secrets versions access latest --secret="sta-env-airflow" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env airflow-config | kubectl replace -f - -n pipeline | |
| ### predict api and jobs ### | |
| predict-service: | |
| name: build-push-deploy-predict-service | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_predict == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Set up QEMU | |
| uses: docker/[email protected] | |
| - name: Set up Docker Buildx | |
| uses: docker/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Build and push API Docker Image | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/predict/api | |
| target: staging | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-prediction-api:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-prediction-api:latest | |
| - name: Build and push predict Job Docker Image | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/predict/jobs/forecast | |
| target: staging | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/stage-airqo-predict-job:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/stage-airqo-predict-job:latest | |
| - name: Build and push train Job Docker Image | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/predict/jobs/forecast_training | |
| target: staging | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/stage-airqo-train-job:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/stage-airqo-train-job:latest | |
| - name: Build and push predict places image | |
| uses: docker/build-push-action@v3 | |
| with: | |
| push: true | |
| context: src/predict/jobs/predict_places_air_quality | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/stage-airqo-predict-places-air-quality:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/stage-airqo-predict-places-air-quality:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/predict/values-stage.yaml" | |
| propertyPath: "images.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update predict staging image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/predict/ | |
| echo "=============== Downloading the required secrets from Secret Manager ===============" | |
| gcloud secrets versions access latest --secret="sta-env-predict" > api.env | |
| gcloud secrets versions access latest --secret="sta-env-predict-job" > predict_job.env | |
| gcloud secrets versions access latest --secret="sta-env-predict-train-job" > train_job.env | |
| gcloud secrets versions access latest --secret="sta-env-predict-predict-places-air-quality" > predict_places_air_quality.env | |
| gcloud secrets versions access latest --secret="sta-key-predict-service-account" > google_application_credentials.json | |
| echo "=============== Updating the respective .env files ===============" | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=api.env env-predict-staging | kubectl replace -f - -n staging | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=predict_job.env env-predict-job-staging | kubectl replace -f - -n staging | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=train_job.env env-train-job-staging | kubectl replace -f - -n staging | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=predict_places_air_quality.env env-predict-places-air-quality | kubectl replace -f - -n staging | |
| echo "=============== Updating the respective credentials files ===============" | |
| kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json env-predict-staging-files | kubectl replace -f - -n staging | |
| ### kafka cluster operator ### | |
| kafka-cluster-operator: | |
| name: build-push-deploy-kafka-cluster-operator | |
| needs: [check] | |
| if: needs.check.outputs.run_kafka_cluster_operator == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Deploy to K8S | |
| run: | | |
| kubectl apply -f k8s/namespaces/message-broker.yaml | |
| bash k8s/kafka/operator/deployment-script.sh | |
| ### kafka cluster ### | |
| kafka-cluster: | |
| name: build-push-deploy-kafka-cluster | |
| needs: [check] | |
| if: ${{ (needs.check.outputs.run_kafka_cluster == 'true') || (needs.check.outputs.run_kafka_cluster_operator == 'true') }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Deploy to K8S | |
| run: | | |
| export CLUSTER_CONTROLLER_IP=${{ secrets.STAGE_CLUSTER_CONTROLLER_IP}} | |
| sed -i 's/CLUSTER_CONTROLLER_IP/'"$CLUSTER_CONTROLLER_IP"'/g' k8s/kafka/clusters/kafka-cluster.yaml | |
| kubectl apply -n message-broker -f k8s/kafka/clusters/kafka-cluster.yaml | |
| kubectl rollout restart deployment kafka-cluster-entity-operator -n message-broker | |
| kubectl rollout restart deployment strimzi-cluster-operator -n message-broker | |
| ### kafka topics ### | |
| kafka-topics: | |
| name: build-push-deploy-kafka-topics | |
| needs: [check] | |
| if: ${{ (needs.check.outputs.run_kafka_cluster == 'true') || (needs.check.outputs.run_kafka_topics == 'true') }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Deploy to K8S | |
| run: | | |
| kubectl apply -f k8s/kafka/topics/kafka-topics.yaml -n message-broker | |
| #### Device Registry ###### | |
| device-registry: | |
| name: build-push-deploy-device-registry | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_device_registry == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Build and Push Docker Image | |
| run: | | |
| cd src/device-registry/ | |
| docker build --target=staging --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-device-registry-api:${{ needs.image-tag.outputs.build_id }} . | |
| docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-device-registry-api:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-device-registry-api:latest | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-device-registry-api:${{ needs.image-tag.outputs.build_id }} | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-device-registry-api:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/device-registry/values-stage.yaml" | |
| propertyPath: "image.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update device registry staging image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/device-registry/ | |
| gcloud secrets versions access latest --secret="sta-env-device-registry" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-device-registry-staging | kubectl replace -f - -n staging | |
| gcloud secrets versions access latest --secret="sta-key-device-registry-service-account" > google_application_credentials.json | |
| kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json device-registry-config-files | kubectl replace -f - -n staging | |
| ### device monitoring ### | |
| device-monitoring: | |
| name: build-push-deploy-device-monitoring | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_device_monitoring == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| ### run unit tests ### | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Build and push API | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/device-monitoring/ | |
| target: staging | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-device-monitor-api:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-device-monitor-api:latest | |
| - name: Build and push celery-beat | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/device-monitoring/ | |
| target: celery-beat | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-device-monitor-celery-beat:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-device-monitor-celery-beat:latest | |
| - name: Build and push celery-worker | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/device-monitoring/ | |
| target: celery-worker | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-device-monitor-celery-worker:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-device-monitor-celery-worker:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/device-monitor/values-stage.yaml" | |
| propertyPath: "images.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update device monitor staging images' tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/device-monitoring/ | |
| gcloud secrets versions access latest --secret="sta-env-device-monitoring" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-device-monitoring-staging | kubectl replace -f - -n staging | |
| gcloud secrets versions access latest --secret="sta-key-device-monitoring-service-account" > google_application_credentials.json | |
| kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json device-monitor-config-files | kubectl replace -f - -n staging | |
| ### data-mgt ### | |
| data-mgt: | |
| name: build-push-deploy-data-mgt | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_data_mgt == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Build and Push Docker Image | |
| run: | | |
| cd src/data-mgt/node/ | |
| docker build --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-data-mgt-api:${{ needs.image-tag.outputs.build_id }} -f Dockerfile.stage . | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-data-mgt-api:${{ needs.image-tag.outputs.build_id }} | |
| docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-data-mgt-api:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-data-mgt-api:latest | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-data-mgt-api:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/data-mgt/values-stage.yaml" | |
| propertyPath: "image.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update data mgt staging image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/data-mgt/node/ | |
| gcloud secrets versions access latest --secret="sta-env-data-mgt-nodejs" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env data-mgt-api-config | kubectl replace -f - -n staging | |
| ### analytics ### | |
| analytics: | |
| name: build-push-deploy-analytics | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_analytics == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| ### run unit tests ### | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Build and push API Docker Image | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/analytics/ | |
| target: staging | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-analytics-api:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-analytics-api:latest | |
| - name: Build and push Celery beat Docker Image | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/analytics/ | |
| target: celery-beat | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-analytics-celery-beat:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-analytics-celery-beat:latest | |
| - name: Build and push Celery worker Docker Image | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/analytics/ | |
| target: celery-worker | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-analytics-celery-worker:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-analytics-celery-worker:latest | |
| - name: Build and push device summary job Docker Image | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/analytics/ | |
| target: devices-summary-job | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-analytics-devices-summary-job:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-analytics-devices-summary-job:latest | |
| - name: Build and push reports job Docker Image | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/analytics/jobs/reports | |
| target: staging | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-analytics-report-job:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-analytics-report-job:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/analytics/values-stage.yaml" | |
| propertyPath: "images.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update analytics staging images tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/analytics/ | |
| gcloud secrets versions access latest --secret="sta-env-analytics" > api.env | |
| gcloud secrets versions access latest --secret="sta-env-analytics-report-job" > reports.env | |
| gcloud secrets versions access latest --secret="sta-key-analytics-service-account" > google_application_credentials.json | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=api.env env-analytics-staging | kubectl replace -f - -n staging | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=reports.env env-analytics-report-staging | kubectl replace -f - -n staging | |
| kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json stage-analytics-config-files | kubectl replace -f - -n staging | |
| ### device uptime ### | |
| device-uptime: | |
| name: build-push-deploy-device-uptime | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_device_uptime == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| ### run unit tests ### | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Build and push device uptime v1 docker image | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/device-uptime/ | |
| target: staging | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-device-uptime-job:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-device-uptime-job:latest | |
| - name: Build and push device uptime v2 docker image | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/device-uptime/ | |
| target: uptime | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-device-uptime-job-v2:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-device-uptime-job-v2:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/device-uptime/values-stage.yaml" | |
| propertyPath: "image.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update device uptime staging image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/device-uptime/ | |
| gcloud secrets versions access latest --secret="sta-env-device-uptime" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-device-uptime-staging | kubectl replace -f - -n staging | |
| gcloud secrets versions access latest --secret="sta-key-device-uptime-v2-service-account" > google_application_credentials.json | |
| kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json device-uptime-v2-files | kubectl replace -f - -n staging | |
| ### device status ### | |
| device-status: | |
| name: build-push-deploy-device-status | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_device_status == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| ### run unit tests ### | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Build and Push Docker Image | |
| run: | | |
| cd src/device-status/ | |
| docker build --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-device-status-job:${{ needs.image-tag.outputs.build_id }} -f Dockerfile.stage . | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-device-status-job:${{ needs.image-tag.outputs.build_id }} | |
| docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-device-status-job:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-device-status-job:latest | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-device-status-job:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/device-status/values-stage.yaml" | |
| propertyPath: "image.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update device uptime staging image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/device-status/ | |
| gcloud secrets versions access latest --secret="sta-env-device-status" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-device-status-staging | kubectl replace -f - -n staging | |
| ### locate ### | |
| locate: | |
| name: build-push-deploy-locate | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_locate == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| ### run unit tests ### | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Build and Push Docker Image | |
| run: | | |
| cd src/locate/ | |
| docker build --target=staging --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-locate-api:${{ needs.image-tag.outputs.build_id }} . | |
| docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-locate-api:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-locate-api:latest | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-locate-api:${{ needs.image-tag.outputs.build_id }} | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-locate-api:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/locate/values-stage.yaml" | |
| propertyPath: "image.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update locate staging image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/locate/ | |
| gcloud secrets versions access latest --secret="sta-env-locate" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-locate-staging | kubectl replace -f - -n staging | |
| ### gp model ### | |
| gp-model: | |
| name: build-push-deploy-gp-model | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_gp_model == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Build and push cron job | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/gp-model | |
| target: staging | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-gp-model-job:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-gp-model-job:latest | |
| - name: Build and push broker consumer | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/gp-model | |
| target: message-broker-consumers | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-gp-model-broker-consumer:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-gp-model-broker-consumer:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/gp-model/values-stage.yaml" | |
| propertyPath: "images.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update gp-model staging image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/gp-model/ | |
| gcloud secrets versions access latest --secret="sta-env-gp-model" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-gp-model-staging | kubectl replace -f - -n staging | |
| gcloud secrets versions access latest --secret="sta-key-gp-model-service-account" > google_application_credentials.json | |
| kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json gp-model-job-config-files | kubectl replace -f - -n staging | |
| ### exceddences ### | |
| exceedance: | |
| name: build-push-deploy-exceedances | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_exceedances == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| ### run unit tests ### | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Build and push AirQo Docker image | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/exceedances | |
| target: airqo-staging | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/stage-airqo-exceedance-job:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/stage-airqo-exceedance-job:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/exceedance/values-stage-airqo.yaml" | |
| propertyPath: "image.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update AirQo exceedance staging image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Build and push KCCA Docker image | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/exceedances | |
| target: kcca-staging | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/stage-kcca-exceedance-job:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/stage-kcca-exceedance-job:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/exceedance/values-stage-kcca.yaml" | |
| propertyPath: "image.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update KCCA exceedance staging image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/exceedances/ | |
| gcloud secrets versions access latest --secret="sta-env-exceedances" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-exceedance-staging | kubectl replace -f - -n staging | |
| ### meta-data ### | |
| meta-data: | |
| name: build-push-deploy-meta-data | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_meta_data == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Set up QEMU | |
| uses: docker/[email protected] | |
| - name: Set up Docker Buildx | |
| uses: docker/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Build and push API | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/meta-data | |
| target: api | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-meta-data-api:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-meta-data-api:latest | |
| - name: Build and push sites consumer | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/meta-data | |
| target: sites-consumer | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-meta-data-sites-consumer:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-meta-data-sites-consumer:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/meta-data/values-stage.yaml" | |
| propertyPath: "images.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update meta-data staging image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/meta-data/ | |
| gcloud secrets versions access latest --secret="sta-env-metadata" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env stage-meta-data-api-config | kubectl replace -f - -n staging | |
| gcloud secrets versions access latest --secret="sta-key-metadata-service-account" > google_application_credentials.json | |
| kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json stage-meta-data-api-config-files | kubectl replace -f - -n staging | |
| ### view ### | |
| view: | |
| name: build-push-deploy-view | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_view == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Set up QEMU | |
| uses: docker/[email protected] | |
| - name: Set up Docker Buildx | |
| uses: docker/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Build and push API | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-view-api:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-view-api:latest | |
| target: api | |
| context: src/view | |
| - name: Build and push Message Broker | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-view-message-broker:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-view-message-broker:latest | |
| target: message-broker | |
| context: src/view | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/view/values-stage.yaml" | |
| propertyPath: "images.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update view staging image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/view/ | |
| gcloud secrets versions access latest --secret="sta-env-view-api" > api.env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=api.env stage-view-api-config | kubectl replace -f - -n staging | |
| gcloud secrets versions access latest --secret="sta-env-view-message-broker" > message_broker.env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=message_broker.env stage-view-message-broker-config | kubectl replace -f - -n staging | |
| gcloud secrets versions access latest --secret="sta-key-view-api-google-credentials" > google_application_credentials.json | |
| kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json view-api-config-files | kubectl replace -f - -n staging | |
| #### Calibrate ###### | |
| calibrate: | |
| name: build-push-deploy-calibrate | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_calibrate == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Set up QEMU | |
| uses: docker/[email protected] | |
| - name: Set up Docker Buildx | |
| uses: docker/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Build and push calibrate API | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/calibrate | |
| target: staging | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-calibrate-api:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-calibrate-api:latest | |
| - name: Build and push calibrate pickle file | |
| uses: docker/[email protected] | |
| with: | |
| push: true | |
| context: src/calibrate | |
| target: pickle-file | |
| tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-calibrate-pickle-file:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-calibrate-pickle-file:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/calibrate/values-stage.yaml" | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update calibrate staging tags to ${{ needs.image-tag.outputs.build_id }}" | |
| changes: | | |
| { | |
| "image.tag": "${{ needs.image-tag.outputs.build_id }}", | |
| "initContainer.image.tag": "${{ needs.image-tag.outputs.build_id }}" | |
| } | |
| - name: Login to Main k8s cluster | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/calibrate/ | |
| gcloud secrets versions access latest --secret="sta-env-calibrate-api" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env stage-calibrate-api-config | kubectl replace -f - -n staging | |
| gcloud secrets versions access latest --secret="sta-key-calibrate-service-account" > google_application_credentials.json | |
| kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json stage-calibrate-api-config-files | kubectl replace -f - -n staging | |
| ### events check ### | |
| events-check: | |
| name: build-push-events-check | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_events_check == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| ### run unit tests ### | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Build and Push Docker Image | |
| run: | | |
| cd src/data-mgt/python/events-monitor/ | |
| docker build --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-events-notifs-job:latest . | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-events-notifs-job:latest | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/data-mgt/python/events-monitor/ | |
| gcloud secrets versions access latest --secret="sta-env-device-registry-events-monitor" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env stage-events-notifs-cronjob-config | kubectl replace -f - -n staging | |
| - name: Deploy to K8S | |
| run: | | |
| kubectl apply -f k8s/events-monitor/stage-events-notifs-cronjob.yaml | |
| #### notification ###### | |
| notification: | |
| name: build-push-deploy-notification | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_notifications == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| ### run unit tests ### | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Build and Push Docker Image | |
| run: | | |
| cd src/notifications/ | |
| docker build --target=staging --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-notification-api:${{ needs.image-tag.outputs.build_id }} . | |
| docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-notification-api:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-notification-api:latest | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-notification-api:${{ needs.image-tag.outputs.build_id }} | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-notification-api:latest | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/notifications/ | |
| gcloud secrets versions access latest --secret="sta-env-notifications" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env stage-notification-api-config | kubectl replace -f - -n staging | |
| gcloud secrets versions access latest --secret="sta-key-notifications-service-account" > google_application_credentials.json | |
| kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json stage-notification-api-config-files | kubectl replace -f - -n staging | |
| - name: Deploy to K8S | |
| run: | | |
| kubectl apply -f k8s/notification-service/stage-notification-api.yaml | |
| kubectl set image deployment/airqo-stage-notification-api sta-notification-api=${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-notification-api:${{ needs.image-tag.outputs.build_id }} -n staging | |
| kubectl annotate deployment/airqo-stage-notification-api kubernetes.io/change-cause="Image updated to ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-notification-api:${{ needs.image-tag.outputs.build_id }} on ${{ needs.image-tag.outputs.datetime }}" -n staging | |
| ### network uptime ### | |
| network-uptime: | |
| name: build-push-network-uptime | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_network_uptime == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Build and Push Docker Image | |
| run: | | |
| cd src/network-uptime/ | |
| docker build --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-network-uptime-api:${{ needs.image-tag.outputs.build_id }} . | |
| docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-network-uptime-api:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-network-uptime-api:latest | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-network-uptime-api:${{ needs.image-tag.outputs.build_id }} | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-network-uptime-api:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/network-uptime/values-stage.yaml" | |
| propertyPath: "image.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update network-uptime staging image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/network-uptime/ | |
| gcloud secrets versions access latest --secret="sta-env-network-uptime" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-network-uptime-staging | kubectl replace -f - -n staging | |
| #### Channel data ###### | |
| channel-data: | |
| name: build-push-deploy-channel-data-job | |
| needs: [check, image-tag] | |
| if: needs.check.outputs.run_channel_data == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Login to GCR | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.REGISTRY_URL }} | |
| username: _json_key | |
| password: ${{ secrets.GCR_CONFIG }} | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Build and Push Docker Image | |
| run: | | |
| cd src/channel-data/ | |
| docker build --target=staging --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-update-channel-data-job:${{ needs.image-tag.outputs.build_id }} . | |
| docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-update-channel-data-job:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-update-channel-data-job:latest | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-update-channel-data-job:${{ needs.image-tag.outputs.build_id }} | |
| docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-update-channel-data-job:latest | |
| - name: Update corresponding helm values file(with retry) | |
| uses: Wandalen/[email protected] # Retries action on fail | |
| with: | |
| action: fjogeleit/yaml-update-action@main # Action to retry | |
| with: | | |
| valueFile: "k8s/channel-data/values-stage.yaml" | |
| propertyPath: "image.tag" | |
| value: ${{ needs.image-tag.outputs.build_id }} | |
| branch: ${{ env.DEPLOY_BRANCH }} | |
| token: ${{ secrets.YAML_UPDATER_TOKEN }} | |
| message: "Update channel-data staging image tag to ${{ needs.image-tag.outputs.build_id }}" | |
| - name: Login to GCP | |
| uses: google-github-actions/[email protected] | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }} | |
| - name: Setup Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Update the corresponding k8s configmap(s) | |
| run: | | |
| cd src/channel-data/ | |
| gcloud secrets versions access latest --secret="sta-env-channel-data" > .env | |
| kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-update-channel-data-staging | kubectl replace -f - -n staging | |
| gcloud secrets versions access latest --secret="sta-key-channel-data-service-account" > google_application_credentials.json | |
| kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json env-update-channel-data-staging-files | kubectl replace -f - -n staging | |
| ### apply nginx ### | |
| nginx: | |
| name: apply-nginx | |
| needs: [check] | |
| if: needs.check.outputs.run_nginx == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Login to K8S | |
| uses: azure/[email protected] | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }} | |
| - name: Apply nginx changes | |
| run: kubectl apply -f k8s/nginx/staging/ |