refactor with search

.github/workflows/codecov.yml

@@ -38,11 +38,11 @@ jobs:
       REDIS_SERVER: ${{ secrets.REDIS_SERVER }}
       REDIS_PORT: ${{ secrets.REDIS_PORT }}
 
-  # setup for airflow and predict microservices
-  airflow-predict:
+  # setup for workflows and predict microservices
+  workflows-predict:
     strategy:
       matrix:
-        microservice: [airflow, predict]
+        microservice: [workflows, predict]
     runs-on: ubuntu-latest
     continue-on-error: true
     name: Code coverage for ${{ matrix.microservice }}
@@ -58,7 +58,7 @@ jobs:
         run: |
           cd src/${{ matrix.microservice }}
           pip install -r requirements.txt
-          if [ "${{ matrix.microservice }}" == "airflow" ]; then
+          if [ "${{ matrix.microservice }}" == "workflows" ]; then
             pytest --cov airqo_etl_utils/tests/ --cov-report=xml
           else
             pytest --cov api/tests/ --cov-report=xml

.github/workflows/deploy-apis-to-production.yml

@@ -51,7 +51,7 @@ jobs:
       run_kafka_cluster_operator: ${{ steps.check_files.outputs.run_kafka_cluster_operator }} # kafka cluster operator
       run_kafka_cluster: ${{ steps.check_files.outputs.run_kafka_cluster }} # kafka cluster
       run_kafka_topics: ${{ steps.check_files.outputs.run_kafka_topics }} # kafka topics
-      run_airflow: ${{ steps.check_files.outputs.run_airflow }} # airflow
+      run_workflows: ${{ steps.check_files.outputs.run_workflows }} # workflows
       run_incentives: ${{ steps.check_files.outputs.run_incentives }} # incentives
       run_kafka_connectors: ${{ steps.check_files.outputs.run_kafka_connectors }} # kafka connectors
       run_nginx: ${{ steps.check_files.outputs.run_nginx }} # nginx
@@ -90,7 +90,7 @@ jobs:
           echo "run_kafka_cluster_operator=false" >>$GITHUB_OUTPUT
           echo "run_kafka_cluster=false" >>$GITHUB_OUTPUT
           echo "run_kafka_topics=false" >>$GITHUB_OUTPUT
-          echo "run_airflow=false" >>$GITHUB_OUTPUT
+          echo "run_workflows=false" >>$GITHUB_OUTPUT
           echo "run_incentives=false" >>$GITHUB_OUTPUT
           echo "run_kafka_connectors=false" >>$GITHUB_OUTPUT
           echo "run_nginx=false" >>$GITHUB_OUTPUT
@@ -227,12 +227,12 @@ jobs:
               echo "run_kafka_connectors=true" >>$GITHUB_OUTPUT
             fi
 
-            if [[ $file == src/airflow/* ]]; then
-              echo "run_airflow=true" >>$GITHUB_OUTPUT
+            if [[ $file == src/workflows/* ]]; then
+              echo "run_workflows=true" >>$GITHUB_OUTPUT
             fi
 
-            if [[ $file == k8s/airflow/* ]]; then
-              echo "run_airflow=true" >>$GITHUB_OUTPUT
+            if [[ $file == k8s/workflows/* ]]; then
+              echo "run_workflows=true" >>$GITHUB_OUTPUT
             fi
 
             if [[ $file == src/incentives/* ]]; then
@@ -260,7 +260,7 @@ jobs:
               echo "run_device_monitoring=true" >>$GITHUB_OUTPUT
               echo "run_device_registry=true" >>$GITHUB_OUTPUT
               echo "run_auth_service=true" >>$GITHUB_OUTPUT
-              echo "run_airflow=true" >>$GITHUB_OUTPUT
+              echo "run_workflows=true" >>$GITHUB_OUTPUT
               echo "run_incentives=true" >>$GITHUB_OUTPUT
               echo "run_view=true" >>$GITHUB_OUTPUT
               echo "run_kafka_connectors=true" >>$GITHUB_OUTPUT
@@ -397,11 +397,11 @@ jobs:
           gcloud secrets versions access latest --secret="prod-key-auth-service-firebase-admin-sdk" > firebase_admin_sdk.json
           kubectl create configmap --dry-run=client -o yaml --from-file=firebase_admin_sdk.json prod-auth-api-config-files | kubectl replace -f - -n production
 
-  ### airflow ###
-  airflow:
-    name: build-push-deploy-airflow
+  ### workflows ###
+  workflows:
+    name: build-push-deploy-workflows
     needs: [check, image-tag]
-    if: needs.check.outputs.run_airflow == 'true'
+    if: needs.check.outputs.run_workflows == 'true'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -424,29 +424,29 @@ jobs:
         uses: docker/[email protected]
         with:
           push: true
-          context: src/airflow
+          context: src/workflows
           target: deployment
-          tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-apache-airflow:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-apache-airflow:latest
+          tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-apache-workflows:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-apache-workflows:latest
 
       - name: Build and push XCom
         uses: docker/[email protected]
         with:
           push: true
-          context: src/airflow
+          context: src/workflows
           target: xcom-setup
-          tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-apache-airflow-xcom:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-apache-airflow-xcom:latest
+          tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-apache-workflows-xcom:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-apache-workflows-xcom:latest
 
       - name: Update corresponding helm values file(with retry)
         uses: Wandalen/[email protected] # Retries action on fail
         with:
           action: fjogeleit/yaml-update-action@main # Action to retry
           with: |
-            valueFile: "k8s/airflow/values-prod.yaml"
+            valueFile: "k8s/workflows/values-prod.yaml"
             propertyPath: "images.tag"
             value: ${{ needs.image-tag.outputs.build_id }}
             branch: ${{ env.DEPLOY_BRANCH }}
             token: ${{ secrets.YAML_UPDATER_TOKEN }}
-            message: "Update airflow prod image tag to ${{ needs.image-tag.outputs.build_id }}"
+            message: "Update workflows prod image tag to ${{ needs.image-tag.outputs.build_id }}"
 
       - name: Login to GCP
         uses: google-github-actions/[email protected]
@@ -464,7 +464,7 @@ jobs:
 
       - name: Update the corresponding k8s configmap(s)
         run: |
-          cd src/airflow/
+          cd src/workflows/
           gcloud secrets versions access latest --secret="prod-env-airflow" > .env
           kubectl create configmap --dry-run=client -o yaml --from-env-file=.env airflow-config | kubectl replace -f - -n pipeline
 

.github/workflows/deploy-apis-to-staging.yml

@@ -51,7 +51,7 @@ jobs:
       run_kafka_cluster_operator: ${{ steps.check_files.outputs.run_kafka_cluster_operator }} # kafka cluster operator
       run_kafka_cluster: ${{ steps.check_files.outputs.run_kafka_cluster }} # kafka cluster
       run_kafka_topics: ${{ steps.check_files.outputs.run_kafka_topics }} # kafka topics
-      run_airflow: ${{ steps.check_files.outputs.run_airflow }} # airflow
+      run_workflows: ${{ steps.check_files.outputs.run_workflows }} # workflows
       run_incentives: ${{ steps.check_files.outputs.run_incentives }} # incentives
       run_kafka_connectors: ${{ steps.check_files.outputs.run_kafka_connectors }} # kafka connectors
       run_nginx: ${{ steps.check_files.outputs.run_nginx }} # nginx ingress
@@ -90,7 +90,7 @@ jobs:
           echo "run_kafka_cluster_operator=false" >>$GITHUB_OUTPUT
           echo "run_kafka_cluster=false" >>$GITHUB_OUTPUT
           echo "run_kafka_topics=false" >>$GITHUB_OUTPUT
-          echo "run_airflow=false" >>$GITHUB_OUTPUT
+          echo "run_workflows=false" >>$GITHUB_OUTPUT
           echo "run_incentives=false" >>$GITHUB_OUTPUT
           echo "run_kafka_connectors=false" >>$GITHUB_OUTPUT
           echo "run_nginx=false" >>$GITHUB_OUTPUT
@@ -227,12 +227,12 @@ jobs:
               echo "run_kafka_connectors=true" >>$GITHUB_OUTPUT
             fi
 
-            if [[ $file == src/airflow/* ]]; then
-              echo "run_airflow=true" >>$GITHUB_OUTPUT
+            if [[ $file == src/workflows/* ]]; then
+              echo "run_workflows=true" >>$GITHUB_OUTPUT
             fi
 
-            if [[ $file == k8s/airflow/* ]]; then
-              echo "run_airflow=true" >>$GITHUB_OUTPUT
+            if [[ $file == k8s/workflows/* ]]; then
+              echo "run_workflows=true" >>$GITHUB_OUTPUT
             fi
 
             if [[ $file == src/incentives/* ]]; then
@@ -261,7 +261,7 @@ jobs:
               echo "run_device_monitoring=true" >>$GITHUB_OUTPUT
               echo "run_device_registry=true" >>$GITHUB_OUTPUT
               echo "run_auth_service=true" >>$GITHUB_OUTPUT
-              echo "run_airflow=true" >>$GITHUB_OUTPUT
+              echo "run_workflows=true" >>$GITHUB_OUTPUT
               echo "run_incentives=true" >>$GITHUB_OUTPUT
               echo "run_view=true" >>$GITHUB_OUTPUT
               echo "run_kafka_connectors=true" >>$GITHUB_OUTPUT
@@ -398,11 +398,11 @@ jobs:
           gcloud secrets versions access latest --secret="sta-key-kafka-bigquery-connectors" > google_application_credentials.json
           kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json bigquery-connectors-files | kubectl replace -f - -n staging
 
-  ### airflow ###
-  airflow:
-    name: build-push-deploy-airflow
+  ### workflows ###
+  workflows:
+    name: build-push-deploy-workflows
     needs: [check, image-tag]
-    if: needs.check.outputs.run_airflow == 'true'
+    if: needs.check.outputs.run_workflows == 'true'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -425,15 +425,15 @@ jobs:
         uses: docker/[email protected]
         with:
           push: true
-          context: src/airflow
+          context: src/workflows
           target: deployment
           tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-apache-airflow:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-apache-airflow:latest
 
       - name: Build and push XCom
         uses: docker/[email protected]
         with:
           push: true
-          context: src/airflow
+          context: src/workflows
           target: xcom-setup
           tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-apache-airflow-xcom:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-apache-airflow-xcom:latest
 
@@ -442,12 +442,12 @@ jobs:
         with:
           action: fjogeleit/yaml-update-action@main # Action to retry
           with: |
-            valueFile: "k8s/airflow/values-stage.yaml"
+            valueFile: "k8s/workflows/values-stage.yaml"
             propertyPath: "images.tag"
             value: ${{ needs.image-tag.outputs.build_id }}
             branch: ${{ env.DEPLOY_BRANCH }}
             token: ${{ secrets.YAML_UPDATER_TOKEN }}
-            message: "Update airflow staging image tag to ${{ needs.image-tag.outputs.build_id }}"
+            message: "Update workflows staging image tag to ${{ needs.image-tag.outputs.build_id }}"
 
       - name: Login to GCP
         uses: google-github-actions/[email protected]
@@ -465,7 +465,7 @@ jobs:
 
       - name: Update the corresponding k8s configmap(s)
         run: |
-          cd src/airflow/
+          cd src/workflows/
           gcloud secrets versions access latest --secret="sta-env-airflow" > .env
           kubectl create configmap --dry-run=client -o yaml --from-env-file=.env airflow-config | kubectl replace -f - -n pipeline
 

contributing.md

@@ -8,7 +8,7 @@ In this guide you will get an overview of the contribution workflow from opening
 
 To get an overview of each microservice, read the respective microservice READMEs:
 
-- [Airflow](/src/airflow/README.md)
+- [Airflow](/src/workflows/README.md)
 - [Analytics](/src/analytics/README.md)
 - [Auth-service](/src/auth-service/README.md)
 - [Calibrate](/src/calibrate/readme.md)

src/airflow/README.md

@@ -76,10 +76,10 @@ python main.py --action=airqo_hourly_data --start-2022-01-01T10:00:00Z --end=202
 ```bash
 sh run.sh  
 ```
-### Running airflow only
+### Running workflows only
 
 ```bash
-sh run-airflow-only.sh  
+sh run-workflows-only.sh  
 ```
 
 **Note for Windows users:** There is a command in the sh files that requires to be modified / uncommented for windows 

src/airflow/dags/airflow_app.py

@@ -1,6 +1,6 @@
 from airflow.decorators import dag, task
 
-from airqo_etl_utils.airflow_custom_utils import AirflowUtils
+from airqo_etl_utils.workflows_custom_utils import AirflowUtils
 
 
 @dag(
@@ -13,7 +13,7 @@
 def aiflow_app_cleanup():
     @task()
     def delete_old_dag_runs(**kwargs):
-        from airqo_etl_utils.airflow_custom_utils import AirflowUtils
+        from airqo_etl_utils.workflows_custom_utils import AirflowUtils
 
         try:
             dag_run = kwargs.get("dag_run")

src/airflow/dags/airnow.py

@@ -1,6 +1,6 @@
 from airflow.decorators import dag, task
 
-from airqo_etl_utils.airflow_custom_utils import AirflowUtils
+from airqo_etl_utils.workflows_custom_utils import AirflowUtils
 
 from airqo_etl_utils.airnow_api import AirNowApi
 

src/airflow/dags/airqo_bam_measurements.py

@@ -1,6 +1,6 @@
 from airflow.decorators import dag, task
 
-from airqo_etl_utils.airflow_custom_utils import AirflowUtils
+from airqo_etl_utils.workflows_custom_utils import AirflowUtils
 
 
 @dag(

src/airflow/dags/airqo_measurements.py

@@ -1,6 +1,6 @@
 from airflow.decorators import dag, task
 
-from airqo_etl_utils.airflow_custom_utils import AirflowUtils
+from airqo_etl_utils.workflows_custom_utils import AirflowUtils
 from airqo_etl_utils.constants import Frequency
 from dag_docs import airqo_realtime_low_cost_measurements_doc
 from task_docs import (

src/airflow/dags/airqo_mobile_measurements.py

@@ -1,6 +1,6 @@
 from airflow.decorators import dag, task
 
-from airqo_etl_utils.airflow_custom_utils import AirflowUtils
+from airqo_etl_utils.workflows_custom_utils import AirflowUtils
 
 
 @dag(

src/airflow/dags/app_notifications.py

@@ -2,7 +2,7 @@
 
 from airflow.decorators import dag, task
 
-from airqo_etl_utils.airflow_custom_utils import AirflowUtils
+from airqo_etl_utils.workflows_custom_utils import AirflowUtils
 
 
 # Runs at 5, 6, 7 and 8 (Monday) to send

src/airflow/dags/daily_measurements.py

@@ -1,6 +1,6 @@
 from airflow.decorators import dag, task
 
-from airqo_etl_utils.airflow_custom_utils import AirflowUtils
+from airqo_etl_utils.workflows_custom_utils import AirflowUtils
 
 
 @dag(

src/airflow/dags/data_summary.py

@@ -1,6 +1,6 @@
 from airflow.decorators import dag, task
 
-from airqo_etl_utils.airflow_custom_utils import AirflowUtils
+from airqo_etl_utils.workflows_custom_utils import AirflowUtils
 
 
 @dag(

src/airflow/dags/data_warehouse.py

@@ -1,6 +1,6 @@
 from airflow.decorators import dag, task
 
-from airqo_etl_utils.airflow_custom_utils import AirflowUtils
+from airqo_etl_utils.workflows_custom_utils import AirflowUtils
 
 
 @dag(

src/airflow/dags/fault_detection_job.py

@@ -1,6 +1,6 @@
 from airflow.decorators import dag, task
 
-from airqo_etl_utils.airflow_custom_utils import AirflowUtils
+from airqo_etl_utils.workflows_custom_utils import AirflowUtils
 
 
 @dag(

src/airflow/dags/kcca_measurements.py

@@ -1,6 +1,6 @@
 from airflow.decorators import dag, task
 
-from airqo_etl_utils.airflow_custom_utils import AirflowUtils
+from airqo_etl_utils.workflows_custom_utils import AirflowUtils
 
 
 @dag(

src/airflow/dags/meta_data.py

@@ -1,6 +1,6 @@
 from airflow.decorators import dag, task
 
-from airqo_etl_utils.airflow_custom_utils import AirflowUtils
+from airqo_etl_utils.workflows_custom_utils import AirflowUtils
 
 
 @dag(

src/airflow/dags/ml_prediction_jobs.py

@@ -1,6 +1,6 @@
 from airflow.decorators import dag, task
 
-from airqo_etl_utils.airflow_custom_utils import AirflowUtils
+from airqo_etl_utils.workflows_custom_utils import AirflowUtils
 from airqo_etl_utils.bigquery_api import BigQueryApi
 from airqo_etl_utils.config import configuration
 from airqo_etl_utils.ml_utils import ForecastUtils, DecodingUtils

src/airflow/dags/ml_training_jobs.py

@@ -3,7 +3,7 @@
 from airflow.decorators import dag, task
 from dateutil.relativedelta import relativedelta
 
-from airqo_etl_utils.airflow_custom_utils import AirflowUtils
+from airqo_etl_utils.workflows_custom_utils import AirflowUtils
 from airqo_etl_utils.bigquery_api import BigQueryApi
 from airqo_etl_utils.config import configuration
 from airqo_etl_utils.date import date_to_str

src/airflow/dags/nasa_measurements.py

@@ -1,6 +1,6 @@
 from airflow.decorators import dag, task
 
-from airqo_etl_utils.airflow_custom_utils import AirflowUtils
+from airqo_etl_utils.workflows_custom_utils import AirflowUtils
 
 
 @dag(

src/airflow/dags/urban_better_measurements.py

@@ -1,6 +1,6 @@
 from airflow.decorators import dag, task
 
-from airqo_etl_utils.airflow_custom_utils import AirflowUtils
+from airqo_etl_utils.workflows_custom_utils import AirflowUtils
 
 
 @dag(