Skip to content

move to production #972

move to production

move to production #972

name: deploy-apis-to-production
on:
pull_request_target:
branches:
- master
types:
- closed
env:
REGISTRY_URL: eu.gcr.io
PROJECT_ID: airqo-250220
DEPLOY_BRANCH: staging
jobs:
image-tag:
if: github.event.pull_request.merged == true
name: create image tag
runs-on: ubuntu-latest
outputs:
build_id: ${{ steps.prep.outputs.build_id }} # build id
datetime: ${{ steps.prep.outputs.datetime }} # build date
steps:
- name: generate build ID
id: prep
run: |
sha=${GITHUB_SHA::8}
timestamp=$(date +%s)
datetime=$(date)
echo "build_id=prod-${sha}-${timestamp}" >>$GITHUB_OUTPUT
echo "datetime=${datetime}" >>$GITHUB_OUTPUT
check:
if: github.event.pull_request.merged == true
name: check changed microservice(s)
outputs:
run_auth_service: ${{ steps.check_files.outputs.run_auth_service }} # auth service
run_device_registry: ${{ steps.check_files.outputs.run_device_registry }} # device registry
run_device_monitoring: ${{ steps.check_files.outputs.run_device_monitoring }} # device monitoring
run_data_mgt: ${{ steps.check_files.outputs.run_data_mgt }} # data mgt
run_data_proxy: ${{ steps.check_files.outputs.run_data_proxy }} # data proxy
run_analytics: ${{ steps.check_files.outputs.run_analytics }} # analytics
run_device_uptime: ${{ steps.check_files.outputs.run_device_uptime }} # device uptime
run_device_status: ${{ steps.check_files.outputs.run_device_status }} # device status
run_locate: ${{ steps.check_files.outputs.run_locate }} # locate
run_predict: ${{ steps.check_files.outputs.run_predict }} # predict
run_gp_model: ${{ steps.check_files.outputs.run_gp_model }} # gp_model
run_exceedances: ${{ steps.check_files.outputs.run_exceedances }} # exceedances
run_meta_data: ${{ steps.check_files.outputs.run_meta_data }} # meta data
run_view: ${{ steps.check_files.outputs.run_view }} # view
run_calibrate: ${{ steps.check_files.outputs.run_calibrate }} # calibrate
run_insights: ${{ steps.check_files.outputs.run_insights}} # analytics
run_kafka_cluster_operator: ${{ steps.check_files.outputs.run_kafka_cluster_operator }} # kafka cluster operator
run_kafka_cluster: ${{ steps.check_files.outputs.run_kafka_cluster }} # kafka cluster
run_kafka_topics: ${{ steps.check_files.outputs.run_kafka_topics }} # kafka topics
run_workflows: ${{ steps.check_files.outputs.run_workflows }} # workflows
run_incentives: ${{ steps.check_files.outputs.run_incentives }} # incentives
run_spatial: ${{ steps.check_files.outputs.run_spatial }} # spatial
run_website: ${{ steps.check_files.outputs.run_website }} # website
run_kafka_connectors: ${{ steps.check_files.outputs.run_kafka_connectors }} # kafka connectors
run_nginx: ${{ steps.check_files.outputs.run_nginx }} # nginx
runs-on: ubuntu-latest
steps:
- name: checkout code
uses: actions/[email protected]
with:
fetch-depth: 2
#### all microservices ######
- name: check modified microserivce
id: check_files
run: |
echo "=============== list modified files ==============="
git diff --name-only HEAD^ HEAD
echo "========== check paths of modified files =========="
git diff --name-only HEAD^ HEAD > files.txt
echo "run_auth_service=false" >>$GITHUB_OUTPUT
echo "run_device_registry=false" >>$GITHUB_OUTPUT
echo "run_device_monitoring=false" >>$GITHUB_OUTPUT
echo "run_data_mgt=false" >>$GITHUB_OUTPUT
echo "run_data_proxy=false" >>$GITHUB_OUTPUT
echo "run_analytics=false" >>$GITHUB_OUTPUT
echo "run_device_uptime=false" >>$GITHUB_OUTPUT
echo "run_device_status=false" >>$GITHUB_OUTPUT
echo "run_locate=false" >>$GITHUB_OUTPUT
echo "run_predict=false" >>$GITHUB_OUTPUT
echo "run_gp_model=false" >>$GITHUB_OUTPUT
echo "run_exceedances=false" >>$GITHUB_OUTPUT
echo "run_meta_data=false" >>$GITHUB_OUTPUT
echo "run_view=false" >>$GITHUB_OUTPUT
echo "run_calibrate=false" >>$GITHUB_OUTPUT
echo "run_kafka_cluster_operator=false" >>$GITHUB_OUTPUT
echo "run_kafka_cluster=false" >>$GITHUB_OUTPUT
echo "run_kafka_topics=false" >>$GITHUB_OUTPUT
echo "run_workflows=false" >>$GITHUB_OUTPUT
echo "run_incentives=false" >>$GITHUB_OUTPUT
echo "run_insights=false" >>$GITHUB_OUTPUT
echo "run_spatial=false" >>$GITHUB_OUTPUT
echo "run_website=false" >>$GITHUB_OUTPUT
echo "run_kafka_connectors=false" >>$GITHUB_OUTPUT
echo "run_nginx=false" >>$GITHUB_OUTPUT
while IFS= read -r file
do
echo $file
if [[ $file == src/auth-service/* ]]; then
echo "run_auth_service=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/auth-service/* ]]; then
echo "run_auth_service=true" >>$GITHUB_OUTPUT
fi
if [[ $file == src/device-registry/* ]]; then
echo "run_device_registry=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/device-registry/* ]]; then
echo "run_device_registry=true" >>$GITHUB_OUTPUT
fi
if [[ $file == src/device-monitoring/* ]]; then
echo "run_device_monitoring=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/device-monitor/* ]]; then
echo "run_device_monitoring=true" >>$GITHUB_OUTPUT
fi
if [[ $file == src/data-mgt/* ]]; then
echo "run_data_mgt=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/data-mgt/* ]]; then
echo "run_data_mgt=true" >>$GITHUB_OUTPUT
fi
if [[ $file == src/data-proxy/* ]]; then
echo "run_data_proxy=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/data-proxy/* ]]; then
echo "run_data_proxy=true" >>$GITHUB_OUTPUT
fi
if [[ $file == src/analytics/* ]]; then
echo "run_analytics=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/analytics/* ]]; then
echo "run_analytics=true" >>$GITHUB_OUTPUT
fi
if [[ $file == src/device-uptime/* ]]; then
echo "run_device_uptime=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/device-uptime/* ]]; then
echo "run_device_uptime=true" >>$GITHUB_OUTPUT
fi
if [[ $file == src/device-status/* ]]; then
echo "run_device_status=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/device-status/* ]]; then
echo "run_device_status=true" >>$GITHUB_OUTPUT
fi
if [[ $file == src/locate/* ]]; then
echo "run_locate=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/locate/* ]]; then
echo "run_locate=true" >>$GITHUB_OUTPUT
fi
if [[ $file == src/predict/* ]]; then
echo "run_predict=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/predict/* ]]; then
echo "run_predict=true" >>$GITHUB_OUTPUT
fi
if [[ $file == src/gp-model/* ]]; then
echo "run_gp_model=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/gp-model/* ]]; then
echo "run_gp_model=true" >>$GITHUB_OUTPUT
fi
if [[ $file == src/exceedances/* ]]; then
echo "run_exceedances=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/exceedance/* ]]; then
echo "run_exceedances=true" >>$GITHUB_OUTPUT
fi
if [[ $file == src/meta-data/* ]]; then
echo "run_meta_data=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/meta-data/* ]]; then
echo "run_meta_data=true" >>$GITHUB_OUTPUT
fi
if [[ $file == src/view/* ]]; then
echo "run_view=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/view/* ]]; then
echo "run_view=true" >>$GITHUB_OUTPUT
fi
if [[ $file == src/calibrate/* ]]; then
echo "run_calibrate=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/calibrate/* ]]; then
echo "run_calibrate=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/kafka/clusters/* ]]; then
echo "run_kafka_cluster=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/kafka/operator/* ]]; then
echo "run_kafka_cluster_operator=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/kafka/topics/* ]]; then
echo "run_kafka_topics=true" >>$GITHUB_OUTPUT
fi
if [[ $file == src/kafka-connectors/* ]]; then
echo "run_kafka_connectors=true" >>$GITHUB_OUTPUT
fi
if [[ $file == src/workflows/* ]]; then
echo "run_workflows=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/workflows/* ]]; then
echo "run_workflows=true" >>$GITHUB_OUTPUT
fi
if [[ $file == src/incentives/* ]]; then
echo "run_incentives=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/incentives/* ]]; then
echo "run_incentives=true" >>$GITHUB_OUTPUT
fi
if [[ $file == src/insights/* ]]; then
echo "run_insights=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/insights/* ]]; then
echo "run_insights=true" >>$GITHUB_OUTPUT
fi
if [[ $file == src/spatial/* ]]; then
echo "run_spatial=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/spatial/* ]]; then
echo "run_spatial=true" >>$GITHUB_OUTPUT
fi
if [[ $file == src/website/* ]]; then
echo "run_website=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/website/* ]]; then
echo "run_website=true" >>$GITHUB_OUTPUT
fi
if [[ $file == k8s/nginx/production/* ]]; then
echo "run_nginx=true" >>$GITHUB_OUTPUT
fi
if [[ $file == workflow-trigger ]]; then
echo "run_calibrate=true" >>$GITHUB_OUTPUT
echo "run_meta_data=true" >>$GITHUB_OUTPUT
echo "run_exceedances=true" >>$GITHUB_OUTPUT
echo "run_gp_model=true" >>$GITHUB_OUTPUT
echo "run_predict=true" >>$GITHUB_OUTPUT
echo "run_locate=true" >>$GITHUB_OUTPUT
echo "run_device_status=true" >>$GITHUB_OUTPUT
echo "run_device_uptime=true" >>$GITHUB_OUTPUT
echo "run_analytics=true" >>$GITHUB_OUTPUT
echo "run_data_mgt=true" >>$GITHUB_OUTPUT
echo "run_data_proxy=true" >>$GITHUB_OUTPUT
echo "run_device_monitoring=true" >>$GITHUB_OUTPUT
echo "run_device_registry=true" >>$GITHUB_OUTPUT
echo "run_auth_service=true" >>$GITHUB_OUTPUT
echo "run_workflows=true" >>$GITHUB_OUTPUT
echo "run_incentives=true" >>$GITHUB_OUTPUT
echo "run_insights=true" >>$GITHUB_OUTPUT
echo "run_spatial=true" >>$GITHUB_OUTPUT
echo "run_website=true" >>$GITHUB_OUTPUT
echo "run_view=true" >>$GITHUB_OUTPUT
echo "run_kafka_connectors=true" >>$GITHUB_OUTPUT
echo "run_nginx=true" >>$GITHUB_OUTPUT
fi
done < files.txt
#### Kafka Connectors ######
kafka-connectors:
name: build-push-deploy-kafka-connectors
needs: [check, image-tag]
if: needs.check.outputs.run_kafka_connectors == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
- name: Set up QEMU
uses: docker/[email protected]
- name: Set up Docker Buildx
uses: docker/[email protected]
- name: Login to GCR
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Build and push measurements source connectors
uses: docker/[email protected]
with:
push: true
context: src/kafka-connectors/measurements-source-connector
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/measurements-connect:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/measurements-connect:latest
- name: Build and push biq query connector
uses: docker/[email protected]
with:
push: true
context: src/kafka-connectors/bigquery-connector
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/bigquery-connector:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/bigquery-connector:latest
- name: Login to k8s cluster
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Update corresponding helm values file(with retry)
uses: Wandalen/[email protected] # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/kafka/BigQuery-connectors/values-prod.yaml"
propertyPath: "image.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update BigQuery-connectors production image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Deploy to k8s cluster
run: |
export MEASUREMENTS_CONNECT_IMAGE=${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/measurements-connect:${{ needs.image-tag.outputs.build_id }}
export PROD_KAFKA_CLUSTER=${{ secrets.PROD_KAFKA_CLUSTER }}
cat k8s/kafka/connectors/measurements-connect.yaml | sed "s/{{MEASUREMENTS_CONNECT_IMAGE}}/$MEASUREMENTS_CONNECT_IMAGE/g" | sed "s/{{KAFKA_CLUSTER}}/$PROD_KAFKA_CLUSTER/g" | kubectl apply -n message-broker -f-
kubectl apply -f k8s/kafka/connectors/purple-air-connector.yaml -n message-broker
- name: Set up GCP credentials and Kubernetes configmaps
run: |
gcloud secrets versions access latest --secret="prod-env-kafka-bigquery-connectors" > .env
kubectl create configmap --dry-run=client -o yaml --from-env-file=.env bigquery-connectors | kubectl replace -f - -n production
gcloud secrets versions access latest --secret="prod-key-kafka-bigquery-connectors" > google_application_credentials.json
kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json bigquery-connectors-files | kubectl replace -f - -n production
### auth service ###
auth-service:
name: build-push-deploy-auth
needs: [check, image-tag]
if: needs.check.outputs.run_auth_service == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
- name: Login to GCR
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Login to K8S
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Build and Push Docker Image
run: |
cd src/auth-service/
docker build --target=production --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-auth-api:${{ needs.image-tag.outputs.build_id }} .
docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-auth-api:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-auth-api:latest
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-auth-api:${{ needs.image-tag.outputs.build_id }}
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-auth-api:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/[email protected] # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/auth-service/values-prod.yaml"
propertyPath: "image.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update auth service production image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Login to GCP
uses: google-github-actions/[email protected]
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK
uses: google-github-actions/[email protected]
- name: Update the corresponding k8s configmap(s)
run: |
cd src/auth-service/
gcloud secrets versions access latest --secret="prod-env-auth-service" > .env
kubectl create configmap --dry-run=client -o yaml --from-env-file=.env prod-auth-api-config | kubectl replace -f - -n production
gcloud secrets versions access latest --secret="prod-key-auth-service-firebase-admin-sdk" > firebase_admin_sdk.json
kubectl create configmap --dry-run=client -o yaml --from-file=firebase_admin_sdk.json prod-auth-api-config-files | kubectl replace -f - -n production
### workflows ###
workflows:
name: build-push-deploy-workflows
needs: [check, image-tag]
if: needs.check.outputs.run_workflows == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
- name: Set up QEMU
uses: docker/[email protected]
- name: Set up Docker Buildx
uses: docker/[email protected]
- name: Login to GCR
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Build and push app
uses: docker/[email protected]
with:
push: true
context: src/workflows
target: deployment
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-workflows:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-workflows:latest
- name: Build and push Redis
uses: docker/[email protected]
with:
push: true
context: src/workflows
target: redis
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-redis:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-redis:latest
- name: Build and push XCom
uses: docker/[email protected]
with:
push: true
context: src/workflows
target: xcom-setup
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-workflows-xcom:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-workflows-xcom:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/[email protected] # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/workflows/values-prod.yaml"
propertyPath: "images.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update workflows prod image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Login to GCP
uses: google-github-actions/[email protected]
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK
uses: google-github-actions/[email protected]
- name: Login to K8S
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Update the corresponding k8s configmap(s)
run: |
cd src/workflows/
gcloud secrets versions access latest --secret="prod-env-airflow" > .env
kubectl create configmap --dry-run=client -o yaml --from-env-file=.env airflow-config | kubectl replace -f - -n pipeline
#### Device Registry ######
device-registry:
name: build-push-deploy-device-registry
needs: [check, image-tag]
if: needs.check.outputs.run_device_registry == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
- name: Login to GCR
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Login to K8S
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Build and Push Docker Image
run: |
cd src/device-registry/
docker build --target=production --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-registry-api:${{ needs.image-tag.outputs.build_id }} .
docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-registry-api:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-registry-api:latest
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-registry-api:${{ needs.image-tag.outputs.build_id }}
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-registry-api:latest
- name: Update corresponding helm values file
uses: Wandalen/[email protected]
with:
action: fjogeleit/yaml-update-action@main
with: |
valueFile: "k8s/device-registry/values-prod.yaml"
propertyPath: "image.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update device registry production image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Login to GCP
uses: google-github-actions/[email protected]
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK
uses: google-github-actions/[email protected]
- name: Update the corresponding k8s configmap(s)
run: |
cd src/device-registry/
gcloud secrets versions access latest --secret="prod-env-device-registry" > .env
kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-device-registry-production | kubectl replace -f - -n production
gcloud secrets versions access latest --secret="prod-key-device-registry-service-account" > google_application_credentials.json
kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json device-registry-config-files | kubectl replace -f - -n production
### device monitoring ###
device-monitoring:
name: build-push-deploy-device-monitoring
needs: [check, image-tag]
if: needs.check.outputs.run_device_monitoring == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
- name: Login to GCR
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Login to K8S
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Build and push API
uses: docker/[email protected]
with:
push: true
context: src/device-monitoring/
target: production
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-monitor-api:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-monitor-api:latest
- name: Build and push celery-beat
uses: docker/[email protected]
with:
push: true
context: src/device-monitoring/
target: celery-beat
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-monitor-celery-beat:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-monitor-celery-beat:latest
- name: Build and push celery-worker
uses: docker/[email protected]
with:
push: true
context: src/device-monitoring/
target: celery-worker
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-monitor-celery-worker:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-monitor-celery-worker:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/[email protected] # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/device-monitor/values-prod.yaml"
propertyPath: "images.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update device monitor production images' tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Login to GCP
uses: google-github-actions/[email protected]
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK
uses: google-github-actions/[email protected]
- name: Update the corresponding k8s configmap(s)
run: |
cd src/device-monitoring/
gcloud secrets versions access latest --secret="prod-env-device-monitoring" > .env
kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-device-monitoring-production | kubectl replace -f - -n production
gcloud secrets versions access latest --secret="prod-key-device-monitoring-service-account" > google_application_credentials.json
kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json device-monitor-config-files | kubectl replace -f - -n production
### data-mgt ###
data-mgt:
name: build-push-deploy-data-mgt
needs: [check, image-tag]
if: needs.check.outputs.run_data_mgt == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
- name: Login to GCR
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Login to K8S
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Build and Push Docker Image
run: |
cd src/data-mgt/
docker build --target=production --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-data-mgt-api:${{ needs.image-tag.outputs.build_id }} .
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-data-mgt-api:${{ needs.image-tag.outputs.build_id }}
docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-data-mgt-api:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-data-mgt-api:latest
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-data-mgt-api:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/[email protected] # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/data-mgt/values-prod.yaml"
propertyPath: "image.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update data mgt production image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Login to GCP
uses: google-github-actions/[email protected]
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK
uses: google-github-actions/[email protected]
- name: Update the corresponding k8s configmap(s)
run: |
cd src/data-mgt/
gcloud secrets versions access latest --secret="prod-env-data-mgt-nodejs" > .env
kubectl create configmap --dry-run=client -o yaml --from-env-file=.env data-mgt-api-config | kubectl replace -f - -n production
### data-proxy ###
data-proxy:
name: build-push-deploy-data-proxy
needs: [check, image-tag]
if: needs.check.outputs.run_data_proxy == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
- name: Login to GCR
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Login to K8S
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Build and Push Docker Image
run: |
cd src/data-proxy/
docker build --target=production --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-data-proxy-api:${{ needs.image-tag.outputs.build_id }} .
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-data-proxy-api:${{ needs.image-tag.outputs.build_id }}
docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-data-proxy-api:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-data-proxy-api:latest
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-data-proxy-api:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/[email protected] # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/data-proxy/values-prod.yaml"
propertyPath: "image.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update data proxy production image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Login to GCP
uses: google-github-actions/[email protected]
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK
uses: google-github-actions/[email protected]
- name: Update the corresponding k8s configmap(s)
run: |
cd src/data-proxy/
gcloud secrets versions access latest --secret="prod-env-data-proxy" > .env
kubectl create configmap --dry-run=client -o yaml --from-env-file=.env data-proxy-api-config | kubectl replace -f - -n production
### analytics ###
analytics:
name: build-push-deploy-analytics
needs: [check, image-tag]
if: needs.check.outputs.run_analytics == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
### run unit tests ###
- name: Login to GCR
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Login to K8S
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Build and push API Docker Image
uses: docker/[email protected]
with:
push: true
context: src/analytics/
target: production
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-analytics-api:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-analytics-api:latest
- name: Build and push Celery beat Docker Image
uses: docker/[email protected]
with:
push: true
context: src/analytics/
target: celery-beat
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-analytics-celery-beat:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-analytics-celery-beat:latest
- name: Build and push Celery worker Docker Image
uses: docker/[email protected]
with:
push: true
context: src/analytics/
target: celery-worker
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-analytics-celery-worker:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-analytics-celery-worker:latest
- name: Build and push device summary job Docker Image
uses: docker/[email protected]
with:
push: true
context: src/analytics/
target: devices-summary-job
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-analytics-devices-summary-job:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-analytics-devices-summary-job:latest
- name: Build and push reports job Docker Image
uses: docker/[email protected]
with:
push: true
context: src/analytics/jobs/reports
target: production
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-analytics-report-job:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-analytics-report-job:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/[email protected] # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/analytics/values-prod.yaml"
propertyPath: "images.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update analytics production image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Login to GCP
uses: google-github-actions/[email protected]
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK
uses: google-github-actions/[email protected]
- name: Update the corresponding k8s configmap(s)
run: |
cd src/analytics/
gcloud secrets versions access latest --secret="prod-env-analytics" > api.env
gcloud secrets versions access latest --secret="prod-env-analytics-report-job" > reports.env
gcloud secrets versions access latest --secret="prod-key-analytics-service-account" > google_application_credentials.json
kubectl create configmap --dry-run=client -o yaml --from-env-file=api.env env-analytics-production | kubectl replace -f - -n production
kubectl create configmap --dry-run=client -o yaml --from-env-file=reports.env env-analytics-report-production | kubectl replace -f - -n production
kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json prod-analytics-config-files | kubectl replace -f - -n production
insights:
name: build-push-deploy-insights
needs: [check, image-tag]
if: needs.check.outputs.run_insights == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
### run unit tests ###
- name: Login to GCR
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Login to K8S
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Build and push API Docker Image
uses: docker/[email protected]
with:
push: true
context: src/insights/
target: production
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-insights-api:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-insights-api:latest
- name: Build and push Celery worker Image
uses: docker/[email protected]
with:
push: true
context: src/insights/
target: celery
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-insights-celery:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-insights-celery:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/[email protected] # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/insights/values-prod.yaml"
propertyPath: "images.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update insights production image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Login to GCP
uses: google-github-actions/[email protected]
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK
uses: google-github-actions/[email protected]
- name: Update the corresponding k8s configmap(s)
run: |
cd src/insights/
gcloud secrets versions access latest --secret="prod-env-analytics" > .env
gcloud secrets versions access latest --secret="prod-key-analytics-service-account" > google_application_credentials.json
kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-insights-production | kubectl replace -f - -n production
kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json prod-insights-config-files | kubectl replace -f - -n production
### device uptime ###
device-uptime:
name: build-push-deploy-device-uptime
needs: [check, image-tag]
if: needs.check.outputs.run_device_uptime == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
### run unit tests ###
- name: Login to GCR
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Login to K8S
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Build and push device uptime v1 docker image
uses: docker/[email protected]
with:
push: true
context: src/device-uptime/
target: production
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-uptime-job:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-uptime-job:latest
- name: Build and push device uptime v2 docker image
uses: docker/[email protected]
with:
push: true
context: src/device-uptime/
target: uptime
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-uptime-job-v2:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-uptime-job-v2:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/[email protected] # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/device-uptime/values-prod.yaml"
propertyPath: "image.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update device uptime production image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Login to GCP
uses: google-github-actions/[email protected]
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK
uses: google-github-actions/[email protected]
- name: Update the corresponding k8s configmap(s)
run: |
cd src/device-uptime/
gcloud secrets versions access latest --secret="prod-env-device-uptime" > .env
kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-device-uptime-production | kubectl replace -f - -n production
gcloud secrets versions access latest --secret="prod-key-device-uptime-v2-service-account" > google_application_credentials.json
kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json device-uptime-v2-files | kubectl replace -f - -n production
### device status ###
device-status:
name: build-push-deploy-device-status
needs: [check, image-tag]
if: needs.check.outputs.run_device_status == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
### run unit tests ###
- name: Login to GCR
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Login to K8S
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Build and Push Docker Image
run: |
cd src/device-status/
docker build --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-status-job:${{ needs.image-tag.outputs.build_id }} -f Dockerfile .
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-status-job:${{ needs.image-tag.outputs.build_id }}
docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-status-job:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-status-job:latest
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-device-status-job:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/[email protected] # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/device-status/values-prod.yaml"
propertyPath: "image.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update device status production image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Login to GCP
uses: google-github-actions/[email protected]
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK
uses: google-github-actions/[email protected]
- name: Update the corresponding k8s configmap(s)
run: |
cd src/device-status/
gcloud secrets versions access latest --secret="prod-env-device-status" > .env
kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-device-status-production | kubectl replace -f - -n production
### kafka cluster operator ###
kafka-cluster-operator:
name: build-push-deploy-kafka-cluster-operator
needs: [check]
if: needs.check.outputs.run_kafka_cluster_operator == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
- name: Login to K8S
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Deploy to K8S
run: |
kubectl apply -f k8s/namespaces/message-broker.yaml
bash k8s/kafka/operator/deployment-script.sh
### kafka cluster ###
kafka-cluster:
name: build-push-deploy-kafka-cluster
needs: [check]
if: ${{ (needs.check.outputs.run_kafka_cluster == 'true') || (needs.check.outputs.run_kafka_cluster_operator == 'true') }}
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
- name: Login to K8S
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Deploy to K8S
run: |
export BROKER0_IP=${{ secrets.PROD_BROKER0_IP }}
export BROKER1_IP=${{ secrets.PROD_BROKER1_IP }}
export BROKER2_IP=${{ secrets.PROD_BROKER2_IP }}
sed -i 's/$(BROKER0_IP)/'"$BROKER0_IP"'/g' k8s/kafka/clusters/kafka-cluster-prod.yaml
sed -i 's/$(BROKER1_IP)/'"$BROKER1_IP"'/g' k8s/kafka/clusters/kafka-cluster-prod.yaml
sed -i 's/$(BROKER2_IP)/'"$BROKER2_IP"'/g' k8s/kafka/clusters/kafka-cluster-prod.yaml
kubectl apply -n message-broker -f k8s/kafka/clusters/kafka-cluster-prod.yaml
kubectl rollout restart deployment kafka-cluster-entity-operator -n message-broker
kubectl rollout restart deployment strimzi-cluster-operator -n message-broker
### kafka topics ###
kafka-topics:
name: build-push-deploy-kafka-topics
needs: [check]
if: ${{ (needs.check.outputs.run_kafka_cluster == 'true') || (needs.check.outputs.run_kafka_topics == 'true') }}
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
- name: Login to K8S
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Deploy to K8S
run: |
kubectl apply -f k8s/kafka/topics/kafka-topics.yaml -n message-broker
### locate ###
locate:
name: build-push-deploy-locate
needs: [check, image-tag]
if: needs.check.outputs.run_locate == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
### run unit tests ###
- name: Login to GCR
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Login to K8S
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Build and Push Docker Image
run: |
cd src/locate/
docker build --target=production --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-locate-api:${{ needs.image-tag.outputs.build_id }} .
docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-locate-api:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-locate-api:latest
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-locate-api:${{ needs.image-tag.outputs.build_id }}
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-locate-api:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/[email protected] # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/locate/values-prod.yaml"
propertyPath: "image.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update locate production image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Login to GCP
uses: google-github-actions/[email protected]
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK
uses: google-github-actions/[email protected]
- name: Update the corresponding k8s configmap(s)
run: |
cd src/locate/
gcloud secrets versions access latest --secret="prod-env-locate-api" > .env
kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-locate-production | kubectl replace -f - -n production
### gp model ###
gp-model:
name: build-push-deploy-gp-model
needs: [check, image-tag]
if: needs.check.outputs.run_gp_model == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
- name: Login to GCR
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Login to K8S
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Build and push cron job
uses: docker/[email protected]
with:
push: true
context: src/gp-model
target: production
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-gp-model-job:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-gp-model-job:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/[email protected] # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/gp-model/values-prod.yaml"
propertyPath: "images.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update gp-model production image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Login to GCP
uses: google-github-actions/[email protected]
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK
uses: google-github-actions/[email protected]
- name: Update the corresponding k8s configmap(s)
run: |
cd src/gp-model/
gcloud secrets versions access latest --secret="prod-env-gp-model" > .env
kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-gp-model-production | kubectl replace -f - -n production
gcloud secrets versions access latest --secret="prod-key-gp-model-service-account" > google_application_credentials.json
kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json gp-model-job-config-files | kubectl replace -f - -n production
### exceddences ###
exceedance:
name: build-push-deploy-exceedances
needs: [check, image-tag]
if: needs.check.outputs.run_exceedances == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
### run unit tests ###
- name: Login to GCR
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Login to K8S
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Build and push AirQo Docker image
uses: docker/[email protected]
with:
push: true
context: src/exceedances
target: airqo-production
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-exceedance-job:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-exceedance-job:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/[email protected] # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/exceedance/values-prod-airqo.yaml"
propertyPath: "image.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update AirQo exceedance production image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Build and push KCCA Docker image
uses: docker/[email protected]
with:
push: true
context: src/exceedances
target: kcca-production
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/kcca-exceedance-job:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/kcca-exceedance-job:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/[email protected] # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/exceedance/values-prod-kcca.yaml"
propertyPath: "image.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update KCCA exceedance production image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Login to GCP
uses: google-github-actions/[email protected]
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK
uses: google-github-actions/[email protected]
- name: Update the corresponding k8s configmap(s)
run: |
cd src/exceedances/
gcloud secrets versions access latest --secret="prod-env-exceedances" > .env
kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-exceedance-production | kubectl replace -f - -n production
### meta-data ###
meta-data:
name: build-push-deploy-meta-data
needs: [check, image-tag]
if: needs.check.outputs.run_meta_data == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
### run unit tests ###
- name: Login to GCR
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Login to K8S
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Build and push API
uses: docker/[email protected]
with:
push: true
context: src/meta-data
target: api
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-meta-data-api:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-meta-data-api:latest
- name: Build and push sites consumer
uses: docker/[email protected]
with:
push: true
context: src/meta-data
target: sites-consumer
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-meta-data-sites-consumer:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-meta-data-sites-consumer:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/[email protected] # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/meta-data/values-prod.yaml"
propertyPath: "images.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update meta-data production image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Login to GCP
uses: google-github-actions/[email protected]
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK
uses: google-github-actions/[email protected]
- name: Update the corresponding k8s configmap(s)
run: |
cd src/meta-data/
gcloud secrets versions access latest --secret="prod-env-metadata" > .env
kubectl create configmap --dry-run=client -o yaml --from-env-file=.env prod-meta-data-api-config | kubectl replace -f - -n production
gcloud secrets versions access latest --secret="prod-key-metadata-service-account" > google_application_credentials.json
kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json prod-meta-data-api-config-files | kubectl replace -f - -n production
### view ###
view:
name: build-push-deploy-view
needs: [check, image-tag]
if: needs.check.outputs.run_view == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
- name: Set up QEMU
uses: docker/[email protected]
- name: Set up Docker Buildx
uses: docker/[email protected]
- name: Login to GCR
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Login to K8S
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Build and push API
uses: docker/[email protected]
with:
push: true
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-view-api:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-view-api:latest
target: api
context: src/view
- name: Update corresponding helm values file(with retry)
uses: Wandalen/[email protected] # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/view/values-prod.yaml"
propertyPath: "images.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update view production image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Login to GCP
uses: google-github-actions/[email protected]
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK
uses: google-github-actions/[email protected]
- name: Update the corresponding k8s configmap(s)
run: |
cd src/view/
gcloud secrets versions access latest --secret="prod-env-view-api" > api.env
kubectl create configmap --dry-run=client -o yaml --from-env-file=api.env prod-view-api-config | kubectl replace -f - -n production
gcloud secrets versions access latest --secret="prod-key-view-api-google-credentials" > google_application_credentials.json
kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json view-api-config-files | kubectl replace -f - -n production
#### Calibrate ######
calibrate:
name: build-push-deploy-calibrate
needs: [check, image-tag]
if: needs.check.outputs.run_calibrate == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
- name: Set up QEMU
uses: docker/[email protected]
- name: Set up Docker Buildx
uses: docker/[email protected]
- name: Login to GCR
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Build and push calibrate API
uses: docker/[email protected]
with:
push: true
context: src/calibrate
target: production
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-calibrate-api:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-calibrate-api:latest
- name: Build and push calibrate pickle file
uses: docker/[email protected]
with:
push: true
context: src/calibrate
target: pickle-file
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-calibrate-pickle-file:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-calibrate-pickle-file:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/[email protected] # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/calibrate/values-prod.yaml"
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update calibrate production tags to ${{ needs.image-tag.outputs.build_id }}"
changes: |
{
"image.tag": "${{ needs.image-tag.outputs.build_id }}",
"initContainer.image.tag": "${{ needs.image-tag.outputs.build_id }}"
}
- name: Login to Main k8s cluster
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Login to GCP
uses: google-github-actions/[email protected]
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK
uses: google-github-actions/[email protected]
- name: Update the corresponding k8s configmap(s)
run: |
cd src/calibrate/
gcloud secrets versions access latest --secret="prod-env-calibrate-api" > .env
kubectl create configmap --dry-run=client -o yaml --from-env-file=.env prod-calibrate-api-config | kubectl replace -f - -n production
gcloud secrets versions access latest --secret="prod-key-calibrate-service-account" > google_application_credentials.json
kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json prod-calibrate-api-config-files | kubectl replace -f - -n production
### predict api and jobs ###
predict-service:
name: build-push-deploy-predict-service
needs: [check, image-tag]
if: needs.check.outputs.run_predict == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
- name: Set up QEMU
uses: docker/[email protected]
- name: Set up Docker Buildx
uses: docker/[email protected]
- name: Login to GCR
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Login to K8S
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Build and push API Docker Image
uses: docker/[email protected]
with:
push: true
context: src/predict/api
target: production
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-prediction-api:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-prediction-api:latest
- name: Build and push predict places image
uses: docker/build-push-action@v3
with:
push: true
context: src/predict/jobs/predict_places_air_quality
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-predict-places-air-quality:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-predict-places-air-quality:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/[email protected] # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/predict/values-prod.yaml"
propertyPath: "images.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update predict production image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Login to GCP
uses: google-github-actions/[email protected]
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK
uses: google-github-actions/[email protected]
- name: Update the corresponding k8s configmap(s)
run: |
cd src/predict/
echo "=============== Creating the required secret files ==============="
gcloud secrets versions access latest --secret="prod-env-predict-api" > api.env
gcloud secrets versions access latest --secret="prod-env-predict-predict-places-air-quality" > predict_places_air_quality.env
gcloud secrets versions access latest --secret="prod-key-predict-service-account" > google_application_credentials.json
echo "=============== Updating the respective .env files ==============="
kubectl create configmap --dry-run=client -o yaml --from-env-file=api.env env-predict-production | kubectl replace -f - -n production
kubectl create configmap --dry-run=client -o yaml --from-env-file=predict_places_air_quality.env env-predict-places-air-quality-production | kubectl replace -f - -n production
echo "=============== Updating the respective credentials files ==============="
kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json env-predict-production-files | kubectl replace -f - -n production
### incentives ###
incentives:
name: build-push-incentives
needs: [check, image-tag]
if: needs.check.outputs.run_incentives == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
- name: Login to GCR
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Login to K8S
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Build and Push Docker Image
uses: docker/[email protected]
with:
push: true
context: src/incentives/
target: production
tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-incentives-api:${{ needs.image-tag.outputs.build_id }}, ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-incentives-api:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/[email protected] # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/incentives/values-prod.yaml"
propertyPath: "image.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update incentives production image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Login to GCP
uses: google-github-actions/[email protected]
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK
uses: google-github-actions/[email protected]
- name: Update the corresponding k8s configmap(s)
run: |
cd src/incentives/
gcloud secrets versions access latest --secret="prod-env-incentives" > .env
kubectl create configmap --dry-run=client -o yaml --from-env-file=.env prod-incentives-api-config | kubectl replace -f - -n production
### apply nginx ###
nginx:
name: apply-nginx
needs: [check]
if: needs.check.outputs.run_nginx == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
- name: Login to K8S
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Apply nginx changes
run: kubectl apply -f k8s/nginx/production/
### spatial ###
spatial:
name: build-push-deploy-spatial
needs: [check, image-tag]
if: needs.check.outputs.run_spatial == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
- name: Login to GCR
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Login to K8S
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Build and Push Docker Image
run: |
cd src/spatial/
docker build --target=production --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-spatial-api:${{ needs.image-tag.outputs.build_id }} .
docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-spatial-api:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-spatial-api:latest
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-spatial-api:${{ needs.image-tag.outputs.build_id }}
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-spatial-api:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/[email protected] # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/spatial/values-prod.yaml"
propertyPath: "image.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update spatial production image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Login to GCP
uses: google-github-actions/[email protected]
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK
uses: google-github-actions/[email protected]
- name: Update the corresponding k8s configmap(s)
run: |
cd src/spatial/
gcloud secrets versions access latest --secret="prod-env-spatial" > .env
kubectl create configmap --dry-run=client -o yaml --from-env-file=.env prod-spatial-api-config | kubectl replace -f - -n production
gcloud secrets versions access latest --secret="prod-key-device-monitoring-service-account" > google_application_credentials.json
kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json prod-spatial-api-config-files | kubectl replace -f - -n production
### website ###
website:
name: build-push-deploy-website
needs: [check, image-tag]
if: needs.check.outputs.run_website == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/[email protected]
- name: Login to GCR
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Login to K8S
uses: azure/[email protected]
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
- name: Build and Push Docker Image
run: |
cd src/website/
docker build --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-website-api:${{ needs.image-tag.outputs.build_id }} .
docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-website-api:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-website-api:latest
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-website-api:${{ needs.image-tag.outputs.build_id }}
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-website-api:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/[email protected] # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/website/values-prod.yaml"
propertyPath: "image.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update website production image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Login to GCP
uses: google-github-actions/[email protected]
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK
uses: google-github-actions/[email protected]
- name: Update the corresponding k8s configmap(s)
run: |
cd src/website/
gcloud secrets versions access latest --secret="prod-env-website-backend" > .env
kubectl create configmap --dry-run=client -o yaml --from-env-file=.env prod-website-api-config | kubectl replace -f - -n production
gcloud secrets versions access latest --secret="prod-key-device-monitoring-service-account" > google_application_credentials.json
kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json prod-website-api-config-files | kubectl replace -f - -n production