diff --git a/src/workflows/airqo_etl_utils/meta_data_utils.py b/src/workflows/airqo_etl_utils/meta_data_utils.py index c5cdd8691b..b54ea6a485 100644 --- a/src/workflows/airqo_etl_utils/meta_data_utils.py +++ b/src/workflows/airqo_etl_utils/meta_data_utils.py @@ -5,11 +5,12 @@ from .constants import Tenant from .data_validator import DataValidationUtils from .weather_data_utils import WeatherDataUtils +from datetime import datetime, timezone class MetaDataUtils: @staticmethod - def extract_devices_from_api(network: str = "all") -> pd.DataFrame: + def extract_devices_from_api() -> pd.DataFrame: devices = AirQoApi().get_devices() dataframe = pd.json_normalize(devices) dataframe = dataframe[ @@ -26,6 +27,7 @@ def extract_devices_from_api(network: str = "all") -> pd.DataFrame: ] ] dataframe["name"] = dataframe["device_id"] + dataframe["last_updated"] = datetime.now(timezone.utc) dataframe = DataValidationUtils.remove_outliers(dataframe) return dataframe diff --git a/src/workflows/airqo_etl_utils/schema/devices.json b/src/workflows/airqo_etl_utils/schema/devices.json index 44a7a76fc8..04f7312d0a 100644 --- a/src/workflows/airqo_etl_utils/schema/devices.json +++ b/src/workflows/airqo_etl_utils/schema/devices.json @@ -1,9 +1,4 @@ [ - { - "name": "tenant", - "type": "STRING", - "mode": "REQUIRED" - }, { "name": "network", "type": "STRING", @@ -19,16 +14,6 @@ "type": "FLOAT", "mode": "NULLABLE" }, - { - "name": "approximate_latitude", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "approximate_longitude", - "type": "FLOAT", - "mode": "NULLABLE" - }, { "name": "site_id", "type": "STRING",