Merge pull request #4000 from airqo-platform/staging

move to production
airqo-platform · Dec 4, 2024 · be69bdb · be69bdb
2 parents 3d3f6c1 + 440a783
commit be69bdb
Show file tree

Hide file tree

Showing 10 changed files with 17 additions and 11 deletions.
diff --git a/k8s/analytics/values-prod.yaml b/k8s/analytics/values-prod.yaml
@@ -8,7 +8,7 @@ images:
     celeryWorker: eu.gcr.io/airqo-250220/airqo-analytics-celery-worker
     reportJob: eu.gcr.io/airqo-250220/airqo-analytics-report-job
     devicesSummaryJob: eu.gcr.io/airqo-250220/airqo-analytics-devices-summary-job
-  tag: prod-00731eb0-1733176039
+  tag: prod-3d3f6c14-1733315928
 api:
   name: airqo-analytics-api
   label: analytics-api

diff --git a/k8s/analytics/values-stage.yaml b/k8s/analytics/values-stage.yaml
@@ -8,7 +8,7 @@ images:
     celeryWorker: eu.gcr.io/airqo-250220/airqo-stage-analytics-celery-worker
     reportJob: eu.gcr.io/airqo-250220/airqo-stage-analytics-report-job
     devicesSummaryJob: eu.gcr.io/airqo-250220/airqo-stage-analytics-devices-summary-job
-  tag: stage-0c5e7596-1732625737
+  tag: stage-014ccd0f-1733315863
 api:
   name: airqo-stage-analytics-api
   label: sta-alytics-api

diff --git a/k8s/auth-service/values-prod.yaml b/k8s/auth-service/values-prod.yaml
@@ -6,7 +6,7 @@ app:
 replicaCount: 3
 image:
   repository: eu.gcr.io/airqo-250220/airqo-auth-api
-  tag: prod-d9febe23-1733311949
+  tag: prod-3d3f6c14-1733315928
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

diff --git a/k8s/exceedance/values-prod-airqo.yaml b/k8s/exceedance/values-prod-airqo.yaml
@@ -4,6 +4,6 @@ app:
   configmap: env-exceedance-production
 image:
   repository: eu.gcr.io/airqo-250220/airqo-exceedance-job
-  tag: prod-d9febe23-1733311949
+  tag: prod-3d3f6c14-1733315928
 nameOverride: ''
 fullnameOverride: ''
diff --git a/k8s/exceedance/values-prod-kcca.yaml b/k8s/exceedance/values-prod-kcca.yaml
@@ -4,6 +4,6 @@ app:
   configmap: env-exceedance-production
 image:
   repository: eu.gcr.io/airqo-250220/kcca-exceedance-job
-  tag: prod-d9febe23-1733311949
+  tag: prod-3d3f6c14-1733315928
 nameOverride: ''
 fullnameOverride: ''
diff --git a/k8s/predict/values-prod.yaml b/k8s/predict/values-prod.yaml
@@ -7,7 +7,7 @@ images:
     predictJob: eu.gcr.io/airqo-250220/airqo-predict-job
     trainJob: eu.gcr.io/airqo-250220/airqo-train-job
     predictPlaces: eu.gcr.io/airqo-250220/airqo-predict-places-air-quality
-  tag: prod-d9febe23-1733311949
+  tag: prod-3d3f6c14-1733315928
 api:
   name: airqo-prediction-api
   label: prediction-api

diff --git a/k8s/spatial/values-prod.yaml b/k8s/spatial/values-prod.yaml
@@ -6,7 +6,7 @@ app:
 replicaCount: 3
 image:
   repository: eu.gcr.io/airqo-250220/airqo-spatial-api
-  tag: prod-d9febe23-1733311949
+  tag: prod-3d3f6c14-1733315928
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

diff --git a/k8s/workflows/values-prod.yaml b/k8s/workflows/values-prod.yaml
@@ -10,7 +10,7 @@ images:
     initContainer: eu.gcr.io/airqo-250220/airqo-workflows-xcom
     redisContainer: eu.gcr.io/airqo-250220/airqo-redis
     containers: eu.gcr.io/airqo-250220/airqo-workflows
-  tag: prod-d9febe23-1733311949
+  tag: prod-3d3f6c14-1733315928
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

diff --git a/src/workflows/airqo_etl_utils/data_validator.py b/src/workflows/airqo_etl_utils/data_validator.py
@@ -151,9 +151,16 @@ def remove_outliers(data: pd.DataFrame) -> pd.DataFrame:
 
         validated_columns = list(chain.from_iterable(filtered_columns.values()))
         for col in validated_columns:
-            is_airqo_network = data["network"] == "airqo"
             mapped_name = configuration.AIRQO_DATA_COLUMN_NAME_MAPPING.get(col, None)
-            data.loc[is_airqo_network, col] = data.loc[is_airqo_network, col].apply(
+            if "network" in data.columns:
+                is_airqo_network = data["network"] == "airqo"
+                data.loc[is_airqo_network, col] = data.loc[is_airqo_network, col].apply(
+                    lambda x: DataValidationUtils.get_valid_value(
+                        column_name=mapped_name, row_value=x
+                    )
+                )
+        else:
+            data[col] = data[col].apply(
                 lambda x: DataValidationUtils.get_valid_value(
                     column_name=mapped_name, row_value=x
                 )

diff --git a/src/workflows/airqo_etl_utils/weather_data_utils.py b/src/workflows/airqo_etl_utils/weather_data_utils.py
@@ -133,7 +133,6 @@ def transform_raw_data(data: pd.DataFrame) -> pd.DataFrame:
         data["value"] = pd.to_numeric(data["value"], errors="coerce", downcast="float")
         data["time"] = pd.to_datetime(data["time"], errors="coerce")
         # TODO Clean this up.
-        data["network"] = "airqo"
         parameter_mappings = {
             "te": "temperature",
             "rh": "humidity",