Skip to content

Commit d9febe2

Browse files
BaalmartBaalmart
authored
Merge pull request #3994 from airqo-platform/staging
move to production
2 parents 03a25cf + 369cafd commit d9febe2

23 files changed

+817
-773
lines changed

‎k8s/auth-service/values-prod.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ app:
66
replicaCount: 3
77
image:
88
repository: eu.gcr.io/airqo-250220/airqo-auth-api
9-
tag: prod-da0db8f0-1733248410
9+
tag: prod-1ea1da62-1733249735
1010
nameOverride: ''
1111
fullnameOverride: ''
1212
podAnnotations: {}

‎k8s/device-registry/values-prod.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ app:
66
replicaCount: 3
77
image:
88
repository: eu.gcr.io/airqo-250220/airqo-device-registry-api
9-
tag: prod-da0db8f0-1733248410
9+
tag: prod-03a25cf1-1733249950
1010
nameOverride: ''
1111
fullnameOverride: ''
1212
podAnnotations: {}

‎k8s/exceedance/values-prod-airqo.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,6 @@ app:
44
configmap: env-exceedance-production
55
image:
66
repository: eu.gcr.io/airqo-250220/airqo-exceedance-job
7-
tag: prod-1ea1da62-1733249735
7+
tag: prod-03a25cf1-1733249950
88
nameOverride: ''
99
fullnameOverride: ''

‎k8s/exceedance/values-prod-kcca.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,6 @@ app:
44
configmap: env-exceedance-production
55
image:
66
repository: eu.gcr.io/airqo-250220/kcca-exceedance-job
7-
tag: prod-1ea1da62-1733249735
7+
tag: prod-03a25cf1-1733249950
88
nameOverride: ''
99
fullnameOverride: ''

‎k8s/predict/values-prod.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ images:
77
predictJob: eu.gcr.io/airqo-250220/airqo-predict-job
88
trainJob: eu.gcr.io/airqo-250220/airqo-train-job
99
predictPlaces: eu.gcr.io/airqo-250220/airqo-predict-places-air-quality
10-
tag: prod-da0db8f0-1733248410
10+
tag: prod-1ea1da62-1733249735
1111
api:
1212
name: airqo-prediction-api
1313
label: prediction-api

‎k8s/spatial/values-prod.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ app:
66
replicaCount: 3
77
image:
88
repository: eu.gcr.io/airqo-250220/airqo-spatial-api
9-
tag: prod-da0db8f0-1733248410
9+
tag: prod-1ea1da62-1733249735
1010
nameOverride: ''
1111
fullnameOverride: ''
1212
podAnnotations: {}

‎k8s/workflows/values-prod.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ images:
1010
initContainer: eu.gcr.io/airqo-250220/airqo-workflows-xcom
1111
redisContainer: eu.gcr.io/airqo-250220/airqo-redis
1212
containers: eu.gcr.io/airqo-250220/airqo-workflows
13-
tag: prod-da0db8f0-1733248410
13+
tag: prod-03a25cf1-1733249950
1414
nameOverride: ''
1515
fullnameOverride: ''
1616
podAnnotations: {}

‎k8s/workflows/values-stage.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ images:
1010
initContainer: eu.gcr.io/airqo-250220/airqo-stage-workflows-xcom
1111
redisContainer: eu.gcr.io/airqo-250220/airqo-stage-redis
1212
containers: eu.gcr.io/airqo-250220/airqo-stage-workflows
13-
tag: stage-7f60b036-1733224934
13+
tag: stage-f43cad79-1733305404
1414
nameOverride: ''
1515
fullnameOverride: ''
1616
podAnnotations: {}

‎src/workflows/airqo_etl_utils/airqo_api.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ def get_devices(
135135
device_category: DeviceCategory = DeviceCategory.NONE,
136136
) -> List[Dict[str, Any]]:
137137
"""
138-
Retrieve devices given a tenant and device category.
138+
Retrieve devices given a network and device category.
139139
140140
Args:
141141
- network (str): An Enum that represents site ownership.
@@ -198,7 +198,6 @@ def get_devices(
198198
"device_category": str(
199199
DeviceCategory.from_str(device.pop("category", None))
200200
),
201-
"network": device.get("network"),
202201
"device_manufacturer": device.get("network", "airqo"),
203202
**device,
204203
}

‎src/workflows/airqo_etl_utils/airqo_utils.py

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ def extract_uncalibrated_data(start_date_time, end_date_time) -> pd.DataFrame:
5959
null_cols=["pm2_5_calibrated_value"],
6060
start_date_time=start_date_time,
6161
end_date_time=end_date_time,
62-
tenant=Tenant.AIRQO,
62+
network=str(Tenant.AIRQO),
6363
)
6464

6565
return DataValidationUtils.remove_outliers(hourly_uncalibrated_data)
@@ -79,7 +79,7 @@ def extract_data_from_bigquery(
7979
table=table,
8080
start_date_time=start_date_time,
8181
end_date_time=end_date_time,
82-
tenant=Tenant.AIRQO,
82+
network=str(Tenant.AIRQO),
8383
)
8484

8585
return DataValidationUtils.remove_outliers(raw_data)
@@ -117,7 +117,10 @@ def remove_duplicates(data: pd.DataFrame) -> pd.DataFrame:
117117

118118
@staticmethod
119119
def extract_aggregated_raw_data(
120-
start_date_time: str, end_date_time: str, dynamic_query: bool = False
120+
start_date_time: str,
121+
end_date_time: str,
122+
network: str = None,
123+
dynamic_query: bool = False,
121124
) -> pd.DataFrame:
122125
"""
123126
Retrieves raw pm2.5 sensor data from bigquery and computes averages for the numeric columns grouped by device_number, device_id and site_id
@@ -128,9 +131,7 @@ def extract_aggregated_raw_data(
128131
start_date_time=start_date_time,
129132
end_date_time=end_date_time,
130133
table=bigquery_api.raw_measurements_table,
131-
network=str(
132-
Tenant.AIRQO
133-
), # TODO Replace tenant implementation with network implementation
134+
network=network,
134135
dynamic_query=dynamic_query,
135136
)
136137

@@ -757,19 +758,26 @@ def clean_low_cost_sensor_data(
757758
AirQoGxExpectations.from_pandas().pm2_5_low_cost_sensor_raw_data(
758759
data
759760
)
760-
761+
else:
762+
data["timestamp"] = pd.to_datetime(data["timestamp"])
761763
data.dropna(subset=["timestamp"], inplace=True)
762-
data["timestamp"] = pd.to_datetime(data["timestamp"])
764+
763765
data.drop_duplicates(
764766
subset=["timestamp", "device_id"], keep="first", inplace=True
765767
)
766768
# TODO Find an appropriate place to put this
767769
if device_category == DeviceCategory.LOW_COST:
768-
data["pm2_5_raw_value"] = data[["s1_pm2_5", "s2_pm2_5"]].mean(axis=1)
769-
data["pm2_5"] = data[["s1_pm2_5", "s2_pm2_5"]].mean(axis=1)
770-
data["pm10_raw_value"] = data[["s1_pm10", "s2_pm10"]].mean(axis=1)
771-
data["pm10"] = data[["s1_pm10", "s2_pm10"]].mean(axis=1)
770+
is_airqo_network = data["network"] == "airqo"
772771

772+
pm2_5_mean = data.loc[is_airqo_network, ["s1_pm2_5", "s2_pm2_5"]].mean(
773+
axis=1
774+
)
775+
pm10_mean = data.loc[is_airqo_network, ["s1_pm10", "s2_pm10"]].mean(axis=1)
776+
777+
data.loc[is_airqo_network, "pm2_5_raw_value"] = pm2_5_mean
778+
data.loc[is_airqo_network, "pm2_5"] = pm2_5_mean
779+
data.loc[is_airqo_network, "pm10_raw_value"] = pm10_mean
780+
data.loc[is_airqo_network, "pm10"] = pm10_mean
773781
return data
774782

775783
@staticmethod
@@ -1032,7 +1040,7 @@ def merge_aggregated_weather_data(
10321040
@staticmethod
10331041
def extract_devices_deployment_logs() -> pd.DataFrame:
10341042
airqo_api = AirQoApi()
1035-
devices = airqo_api.get_devices(tenant=Tenant.AIRQO)
1043+
devices = airqo_api.get_devices(network=str(Tenant.AIRQO))
10361044
devices_history = pd.DataFrame()
10371045
for device in devices:
10381046
try:

0 commit comments

Comments
 (0)