From 1f67f9f8bbf1f8b0c37a357bc46529c91ee09916 Mon Sep 17 00:00:00 2001 From: Justin Terry Date: Mon, 19 Aug 2024 16:23:02 -0700 Subject: [PATCH 01/15] Initial data mart endpoint for net change --- app/routes/analysis/analysis.py | 38 ++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/app/routes/analysis/analysis.py b/app/routes/analysis/analysis.py index 4843542e6..69312e963 100644 --- a/app/routes/analysis/analysis.py +++ b/app/routes/analysis/analysis.py @@ -17,7 +17,7 @@ from ...settings.globals import GEOSTORE_SIZE_LIMIT_OTF from ...utils.geostore import get_geostore from .. import DATE_REGEX -from ..datasets.queries import _query_raster_lambda +from ..datasets.queries import _query_raster_lambda, _query_dataset_json router = APIRouter() @@ -92,6 +92,42 @@ async def zonal_statistics_post( ) + +@router.get( + "/datamart/net_tree_cover_change", + response_class=ORJSONResponse, + response_model=Response, + tags=["Analysis"], + deprecated=True, +) +async def zonal_statistics_get( + *, + iso: str = Query(..., title="ISO code"), + adm1: Optional[int] = Query(None, title="Admin level 1 ID"), + adm2: Optional[int] = Query(None, title="Admin level 2 ID"), +): + select_fields = "iso, adm1, adm2, stable, loss, gain, disturb, net, change, gfw_area__ha" + where_filter = f"iso = '${iso}'" + level = "iso" + + if adm1 is not None: + where_filter += f"AND adm1 = '${adm1}'" + level = "adm1" + + if adm2 is not None: + where_filter += f"AND adm1 = '${adm2}'" + level = "adm2" + + + results = await _query_dataset_json( + dataset=f"umd_{level}_net_tree_cover_change_from_height", + version="v202209", + sql=f"SELECT ${select_fields} FROM data WHERE ${where_filter}", + ) + + return ORJSONResponse(data=results) + + async def _zonal_statistics( geometry: Geometry, sum_layers: List[RasterLayer], From 63c6ab793d848ade1436422138081e4b6625ac31 Mon Sep 17 00:00:00 2001 From: Justin Terry Date: Fri, 23 Aug 2024 14:49:51 -0700 Subject: [PATCH 02/15] Add data mart endpoint --- app/main.py | 9 ++++ app/routes/analysis/analysis.py | 36 --------------- app/routes/datamart/__init__.py | 0 app/routes/datamart/globalforestwatch.py | 58 ++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 36 deletions(-) create mode 100644 app/routes/datamart/__init__.py create mode 100644 app/routes/datamart/globalforestwatch.py diff --git a/app/main.py b/app/main.py index ef1a5004a..28fec7e90 100644 --- a/app/main.py +++ b/app/main.py @@ -33,6 +33,7 @@ from .routes.geostore import geostore as geostore_top from .routes.jobs import job from .routes.tasks import task +from .routes.datamart import globalforestwatch ################ # LOGGING @@ -163,6 +164,12 @@ async def rve_error_handler( app.include_router(r, prefix="/analysis") +############### +# Data Mart API +############### + +app.include_router(globalforestwatch.router, prefix="/datamart/globalforestwatch") + ############### # JOB API ############### @@ -198,6 +205,7 @@ async def rve_error_handler( {"name": "Analysis", "description": analysis.__doc__}, {"name": "Job", "description": job.__doc__}, {"name": "Health", "description": health.__doc__}, + {"name": "Data Mart", "description": globalforestwatch.__doc__}, ] @@ -223,6 +231,7 @@ def custom_openapi(): {"name": "Task API", "tags": ["Tasks"]}, {"name": "Analysis API", "tags": ["Analysis"]}, {"name": "Health API", "tags": ["Health"]}, + {"name": "Data Mart API", "tags": ["Data Mart"]}, ] app.openapi_schema = openapi_schema diff --git a/app/routes/analysis/analysis.py b/app/routes/analysis/analysis.py index 69312e963..9d33cdb38 100644 --- a/app/routes/analysis/analysis.py +++ b/app/routes/analysis/analysis.py @@ -92,42 +92,6 @@ async def zonal_statistics_post( ) - -@router.get( - "/datamart/net_tree_cover_change", - response_class=ORJSONResponse, - response_model=Response, - tags=["Analysis"], - deprecated=True, -) -async def zonal_statistics_get( - *, - iso: str = Query(..., title="ISO code"), - adm1: Optional[int] = Query(None, title="Admin level 1 ID"), - adm2: Optional[int] = Query(None, title="Admin level 2 ID"), -): - select_fields = "iso, adm1, adm2, stable, loss, gain, disturb, net, change, gfw_area__ha" - where_filter = f"iso = '${iso}'" - level = "iso" - - if adm1 is not None: - where_filter += f"AND adm1 = '${adm1}'" - level = "adm1" - - if adm2 is not None: - where_filter += f"AND adm1 = '${adm2}'" - level = "adm2" - - - results = await _query_dataset_json( - dataset=f"umd_{level}_net_tree_cover_change_from_height", - version="v202209", - sql=f"SELECT ${select_fields} FROM data WHERE ${where_filter}", - ) - - return ORJSONResponse(data=results) - - async def _zonal_statistics( geometry: Geometry, sum_layers: List[RasterLayer], diff --git a/app/routes/datamart/__init__.py b/app/routes/datamart/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/app/routes/datamart/globalforestwatch.py b/app/routes/datamart/globalforestwatch.py new file mode 100644 index 000000000..f2451bc60 --- /dev/null +++ b/app/routes/datamart/globalforestwatch.py @@ -0,0 +1,58 @@ +"""APIs for Global Forest Watch data.""" + +from typing import Optional + +from fastapi import Query, APIRouter, HTTPException +from fastapi.logger import logger +from fastapi.responses import ORJSONResponse +from httpx import AsyncClient + +from app.models.pydantic.responses import Response + +router = APIRouter() + +PRODUCTION_SERVICE_URI = "https://data-api.globalforestwatch.org" +NET_CHANGE_VERSION = "v202209" + +@router.get( + "/v1/net_tree_cover_change", + response_class=ORJSONResponse, + response_model=Response, + tags=["Data Mart"], +) +async def net_tree_cover_change( + *, + iso: str = Query(..., title="ISO code"), + adm1: Optional[int] = Query(None, title="Admin level 1 ID"), + adm2: Optional[int] = Query(None, title="Admin level 2 ID"), +): + select_fields = "iso" + where_filter = f"iso = '{iso}'" + level = "adm0" + + if adm1 is not None: + where_filter += f"AND adm1 = '{adm1}'" + select_fields += ", adm1" + level = "adm1" + + if adm2 is not None: + where_filter += f"AND adm2 = '{adm2}'" + select_fields += ", adm2" + level = "adm2" + elif adm2 is not None: + raise HTTPException(400, "If query for adm2, you must also include an adm1 parameter.") + + select_fields += ", stable, loss, gain, disturb, net, change, gfw_area__ha" + + async with AsyncClient() as client: + sql = f"SELECT {select_fields} FROM data WHERE {where_filter}" + url = f"{PRODUCTION_SERVICE_URI}/dataset/umd_{level}_net_tree_cover_change_from_height/{NET_CHANGE_VERSION}/query/json?sql={sql}" + + response = await client.get(url) + if response.status_code != 200: + logger.error(f"API responded with status code {response.status_code}: {response.content}") + raise HTTPException(500, "Internal Server Error") + + results = response.json()["data"] + + return Response(data=results) From 106301b8dae5b2268315067aef62d83b361d1d87 Mon Sep 17 00:00:00 2001 From: Justin Terry Date: Mon, 26 Aug 2024 13:36:10 -0700 Subject: [PATCH 03/15] Force change --- app/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/main.py b/app/main.py index 28fec7e90..c7731ae6f 100644 --- a/app/main.py +++ b/app/main.py @@ -20,6 +20,7 @@ from .routes.analysis import analysis from .routes.assets import asset, assets from .routes.authentication import authentication +from .routes.datamart import globalforestwatch from .routes.datasets import asset as version_asset from .routes.datasets import ( dataset, @@ -33,7 +34,6 @@ from .routes.geostore import geostore as geostore_top from .routes.jobs import job from .routes.tasks import task -from .routes.datamart import globalforestwatch ################ # LOGGING @@ -202,7 +202,7 @@ async def rve_error_handler( {"name": "Download", "description": downloads.__doc__}, {"name": "Geostore", "description": geostore.__doc__}, {"name": "Tasks", "description": task.__doc__}, - {"name": "Analysis", "description": analysis.__doc__}, + {"name": "Analysis2", "description": analysis.__doc__}, {"name": "Job", "description": job.__doc__}, {"name": "Health", "description": health.__doc__}, {"name": "Data Mart", "description": globalforestwatch.__doc__}, From f3da5e9e3db276acd650dec65ec70e3988cde0f2 Mon Sep 17 00:00:00 2001 From: Gary Tempus Jr Date: Tue, 3 Sep 2024 17:49:35 -0400 Subject: [PATCH 04/15] :bug: fix: add missing api key header in proxied call --- app/routes/datamart/globalforestwatch.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/app/routes/datamart/globalforestwatch.py b/app/routes/datamart/globalforestwatch.py index f2451bc60..5409fd910 100644 --- a/app/routes/datamart/globalforestwatch.py +++ b/app/routes/datamart/globalforestwatch.py @@ -1,12 +1,13 @@ """APIs for Global Forest Watch data.""" - from typing import Optional -from fastapi import Query, APIRouter, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Query from fastapi.logger import logger +from fastapi.openapi.models import APIKey from fastapi.responses import ORJSONResponse from httpx import AsyncClient +from app.authentication.api_keys import get_api_key from app.models.pydantic.responses import Response router = APIRouter() @@ -14,6 +15,7 @@ PRODUCTION_SERVICE_URI = "https://data-api.globalforestwatch.org" NET_CHANGE_VERSION = "v202209" + @router.get( "/v1/net_tree_cover_change", response_class=ORJSONResponse, @@ -25,6 +27,7 @@ async def net_tree_cover_change( iso: str = Query(..., title="ISO code"), adm1: Optional[int] = Query(None, title="Admin level 1 ID"), adm2: Optional[int] = Query(None, title="Admin level 2 ID"), + api_key: APIKey = Depends(get_api_key), ): select_fields = "iso" where_filter = f"iso = '{iso}'" @@ -40,7 +43,9 @@ async def net_tree_cover_change( select_fields += ", adm2" level = "adm2" elif adm2 is not None: - raise HTTPException(400, "If query for adm2, you must also include an adm1 parameter.") + raise HTTPException( + 400, "If query for adm2, you must also include an adm1 parameter." + ) select_fields += ", stable, loss, gain, disturb, net, change, gfw_area__ha" @@ -48,9 +53,11 @@ async def net_tree_cover_change( sql = f"SELECT {select_fields} FROM data WHERE {where_filter}" url = f"{PRODUCTION_SERVICE_URI}/dataset/umd_{level}_net_tree_cover_change_from_height/{NET_CHANGE_VERSION}/query/json?sql={sql}" - response = await client.get(url) + response = await client.get(url, headers={"x-api-key": api_key}) if response.status_code != 200: - logger.error(f"API responded with status code {response.status_code}: {response.content}") + logger.error( + f"API responded with status code {response.status_code}: {response.content}" + ) raise HTTPException(500, "Internal Server Error") results = response.json()["data"] From 218f469a24650feade82de8eec7059150d3accb4 Mon Sep 17 00:00:00 2001 From: Gary Tempus Jr Date: Thu, 5 Sep 2024 19:31:53 -0400 Subject: [PATCH 05/15] feat: break out responsibilities and add more docs --- app/routes/datamart/globalforestwatch.py | 182 +++++++++++++++++------ 1 file changed, 140 insertions(+), 42 deletions(-) diff --git a/app/routes/datamart/globalforestwatch.py b/app/routes/datamart/globalforestwatch.py index 5409fd910..27e99630e 100644 --- a/app/routes/datamart/globalforestwatch.py +++ b/app/routes/datamart/globalforestwatch.py @@ -1,4 +1,5 @@ """APIs for Global Forest Watch data.""" +from enum import Enum from typing import Optional from fastapi import APIRouter, Depends, HTTPException, Query @@ -6,60 +7,157 @@ from fastapi.openapi.models import APIKey from fastapi.responses import ORJSONResponse from httpx import AsyncClient +from pydantic import Field, root_validator, ValidationError from app.authentication.api_keys import get_api_key -from app.models.pydantic.responses import Response +from app.models.pydantic.base import StrictBaseModel router = APIRouter() -PRODUCTION_SERVICE_URI = "https://data-api.globalforestwatch.org" -NET_CHANGE_VERSION = "v202209" + +class Gadm(str, Enum): + ISO = "iso" + ADM0 = "adm0" + ADM1 = "adm1" + ADM2 = "adm2" + + +class NetTreeCoverChangeRequest(StrictBaseModel): + iso: str = Field(..., description="ISO code of the country or region (e.g., 'BRA' for Brazil).") + adm1: Optional[int] = Field(None, description="Admin level 1 ID (e.g., a state or province).") + adm2: Optional[int] = Field(None, description="Admin level 2 ID (e.g., a municipality). ⚠️ **Must be provided with adm1.**") + + @root_validator + def check_adm1_adm2_dependency(cls, values): + """ + Validates that adm2 is only provided if adm1 is also present. + Raises a validation error if adm2 is given without adm1. + """ + print(values.keys()) + adm1, adm2 = values.get('adm1'), values.get('adm2') + if adm2 is not None and adm1 is None: + raise ValueError("If 'adm2' is provided, 'adm1' must also be present.") + return values + + def get_admin_level(self): + """ + Determines the appropriate level ('adm0', 'adm1', or 'adm2') based on the presence of adm1 and adm2. + """ + if self.adm2 is not None: + return Gadm.ADM2.value # Return the Enum value 'adm2' + if self.adm1 is not None: + return Gadm.ADM1.value # Return the Enum value 'adm1' + return Gadm.ADM0.value # Default to 'adm0' + + +class TreeCoverData(StrictBaseModel): + """ + Model representing individual tree cover change data from the API. + """ + iso: str = Field(..., description="ISO code of the country or region (e.g., 'BRA' for Brazil).") + adm1: Optional[int] = Field(None, description="Admin level 1 ID (e.g., a state or province).") + adm2: Optional[int] = Field(None, description="Admin level 2 ID (e.g., a municipality).") + stable: float = Field(..., description="The area of stable forest in hectares.") + loss: float = Field(..., description="The area of forest loss in hectares.") + gain: float = Field(..., description="The area of forest gain in hectares.") + disturb: float = Field(..., description="The area of forest disturbance in hectares.") + net: float = Field(..., description="The net change in forest cover in hectares (gain - loss).") + change: float = Field(..., description="The percentage change in forest cover.") + gfw_area__ha: float = Field(..., description="The total forest area in hectares.") + + +class NetTreeCoverChangeResponse(StrictBaseModel): + data: TreeCoverData = Field(..., description="A list of tree cover change data records.") + status: str = Field(..., description="Status of the request (e.g., 'success').") + + class Config: + schema_extra = { + "example": { + "data": + { + "iso": "BRA", + "stable": 413722809.3, + "loss": 36141245.77, + "gain": 8062324.946, + "disturb": 23421628.86, + "net": -28078920.83, + "change": -5.932759761810303, + "gfw_area__ha": 850036547.481532, + "adm1": 12, + "adm2": 34 + }, + "status": "success" + } + } + + +def build_sql_query(request): + select_fields = [Gadm.ISO.value] + where_conditions = [f"{Gadm.ISO.value} = '{request.iso}'"] + + append_field_and_condition(select_fields, where_conditions, Gadm.ADM1.value, request.adm1) + append_field_and_condition(select_fields, where_conditions, Gadm.ADM2.value, request.adm2) + + select_fields += ["stable", "loss", "gain", "disturb", "net", "change", "gfw_area__ha"] + + select_fields_str = ", ".join(select_fields) + where_filter_str = " AND ".join(where_conditions) + + sql = f"SELECT {select_fields_str} FROM data WHERE {where_filter_str}" + + return sql + +def append_field_and_condition(select_fields, where_conditions, field_name, field_value): + if field_value is not None: + select_fields.append(field_name) + where_conditions.append(f"{field_name} = '{field_value}'") + + +async def fetch_tree_cover_data(sql_query: str, level: str, api_key: str) -> TreeCoverData: + """ + Fetches tree cover data from the external API using the SQL query and level. + Handles the HTTP request, response status check, and data extraction. + """ + production_service_uri = "https://data-api.globalforestwatch.org" + net_change_version = "v202209" + url = f"{production_service_uri}/dataset/umd_{level}_net_tree_cover_change_from_height/{net_change_version}/query/json?sql={sql_query}" + + async with AsyncClient() as client: + response = await client.get(url, headers={"x-api-key": api_key}) + if response.status_code != 200: + logger.error(f"API responded with status code {response.status_code}: {response.content}") + raise Exception("Failed to fetch tree cover data.") + + # Parse and validate the response data into TreeCoverData models + response_data = response.json().get("data", [])[0] + return TreeCoverData(**response_data) @router.get( "/v1/net_tree_cover_change", response_class=ORJSONResponse, - response_model=Response, + response_model=NetTreeCoverChangeResponse, tags=["Data Mart"], + summary="Retrieve net tree cover change data", + description="This endpoint provides data on net tree cover change by querying the Global Forest Watch (GFW) database.", ) async def net_tree_cover_change( - *, - iso: str = Query(..., title="ISO code"), - adm1: Optional[int] = Query(None, title="Admin level 1 ID"), - adm2: Optional[int] = Query(None, title="Admin level 2 ID"), - api_key: APIKey = Depends(get_api_key), + iso: str = Query(..., description="ISO code of the country or region (e.g., 'BRA' for Brazil).", example="BRA"), + adm1: Optional[int] = Query(None, description="Admin level 1 ID (e.g., a state or province).", example="12"), + adm2: Optional[int] = Query(None, description="Admin level 2 ID (e.g., a municipality). ⚠️ **Must provide `adm1` also.**", example="34"), + api_key: APIKey = Depends(get_api_key) ): - select_fields = "iso" - where_filter = f"iso = '{iso}'" - level = "adm0" - - if adm1 is not None: - where_filter += f"AND adm1 = '{adm1}'" - select_fields += ", adm1" - level = "adm1" - - if adm2 is not None: - where_filter += f"AND adm2 = '{adm2}'" - select_fields += ", adm2" - level = "adm2" - elif adm2 is not None: - raise HTTPException( - 400, "If query for adm2, you must also include an adm1 parameter." - ) - - select_fields += ", stable, loss, gain, disturb, net, change, gfw_area__ha" - - async with AsyncClient() as client: - sql = f"SELECT {select_fields} FROM data WHERE {where_filter}" - url = f"{PRODUCTION_SERVICE_URI}/dataset/umd_{level}_net_tree_cover_change_from_height/{NET_CHANGE_VERSION}/query/json?sql={sql}" - - response = await client.get(url, headers={"x-api-key": api_key}) - if response.status_code != 200: - logger.error( - f"API responded with status code {response.status_code}: {response.content}" - ) - raise HTTPException(500, "Internal Server Error") - - results = response.json()["data"] + """ + Retrieves net tree cover change data. + """ + try: + request = NetTreeCoverChangeRequest(iso=iso, adm1=adm1, adm2=adm2) + sql_query: str = build_sql_query(request) + admin_level: str = request.get_admin_level() + tree_cover_data: TreeCoverData = await fetch_tree_cover_data(sql_query, admin_level, api_key) + except ValueError as e: + raise HTTPException(status_code=422, detail=str(e)) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) - return Response(data=results) + return NetTreeCoverChangeResponse(data=tree_cover_data, status="success") From 1b97c6ff31db030dfe08c208d3b9ffed43e8ee3a Mon Sep 17 00:00:00 2001 From: Gary Tempus Jr Date: Fri, 6 Sep 2024 12:05:03 -0400 Subject: [PATCH 06/15] :art: refactor: improve naming and scope of helper functions --- app/routes/datamart/globalforestwatch.py | 35 +++++++++++++++--------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/app/routes/datamart/globalforestwatch.py b/app/routes/datamart/globalforestwatch.py index 27e99630e..bc6c026e0 100644 --- a/app/routes/datamart/globalforestwatch.py +++ b/app/routes/datamart/globalforestwatch.py @@ -22,7 +22,7 @@ class Gadm(str, Enum): ADM2 = "adm2" -class NetTreeCoverChangeRequest(StrictBaseModel): +class GadmSpecification(StrictBaseModel): iso: str = Field(..., description="ISO code of the country or region (e.g., 'BRA' for Brazil).") adm1: Optional[int] = Field(None, description="Admin level 1 ID (e.g., a state or province).") adm2: Optional[int] = Field(None, description="Admin level 2 ID (e.g., a municipality). ⚠️ **Must be provided with adm1.**") @@ -33,13 +33,12 @@ def check_adm1_adm2_dependency(cls, values): Validates that adm2 is only provided if adm1 is also present. Raises a validation error if adm2 is given without adm1. """ - print(values.keys()) adm1, adm2 = values.get('adm1'), values.get('adm2') if adm2 is not None and adm1 is None: raise ValueError("If 'adm2' is provided, 'adm1' must also be present.") return values - def get_admin_level(self): + def get_specified_admin_level(self): """ Determines the appropriate level ('adm0', 'adm1', or 'adm2') based on the presence of adm1 and adm2. """ @@ -91,12 +90,12 @@ class Config: } -def build_sql_query(request): +def _build_sql_query(request): select_fields = [Gadm.ISO.value] where_conditions = [f"{Gadm.ISO.value} = '{request.iso}'"] - append_field_and_condition(select_fields, where_conditions, Gadm.ADM1.value, request.adm1) - append_field_and_condition(select_fields, where_conditions, Gadm.ADM2.value, request.adm2) + _append_field_and_condition(select_fields, where_conditions, Gadm.ADM1.value, request.adm1) + _append_field_and_condition(select_fields, where_conditions, Gadm.ADM2.value, request.adm2) select_fields += ["stable", "loss", "gain", "disturb", "net", "change", "gfw_area__ha"] @@ -107,23 +106,33 @@ def build_sql_query(request): return sql -def append_field_and_condition(select_fields, where_conditions, field_name, field_value): +def _append_field_and_condition(select_fields, where_conditions, field_name, field_value): if field_value is not None: select_fields.append(field_name) where_conditions.append(f"{field_name} = '{field_value}'") -async def fetch_tree_cover_data(sql_query: str, level: str, api_key: str) -> TreeCoverData: +async def _fetch_tree_cover_data(sql_query: str, level: str, api_key: str) -> TreeCoverData: """ Fetches tree cover data from the external API using the SQL query and level. Handles the HTTP request, response status check, and data extraction. + Adds a custom header for tracking the service name for NewRelic/AWS monitoring. """ production_service_uri = "https://data-api.globalforestwatch.org" net_change_version = "v202209" url = f"{production_service_uri}/dataset/umd_{level}_net_tree_cover_change_from_height/{net_change_version}/query/json?sql={sql_query}" + # Custom header for identifying the service for monitoring + service_name = "globalforestwatch-datamart" + async with AsyncClient() as client: - response = await client.get(url, headers={"x-api-key": api_key}) + # Add the 'x-api-key' and custom 'X-Service-Name' headers + headers = { + "x-api-key": api_key, + "x-service-name": service_name + } + response = await client.get(url, headers=headers) + if response.status_code != 200: logger.error(f"API responded with status code {response.status_code}: {response.content}") raise Exception("Failed to fetch tree cover data.") @@ -151,10 +160,10 @@ async def net_tree_cover_change( Retrieves net tree cover change data. """ try: - request = NetTreeCoverChangeRequest(iso=iso, adm1=adm1, adm2=adm2) - sql_query: str = build_sql_query(request) - admin_level: str = request.get_admin_level() - tree_cover_data: TreeCoverData = await fetch_tree_cover_data(sql_query, admin_level, api_key) + gadm_specifier = GadmSpecification(iso=iso, adm1=adm1, adm2=adm2) + sql_query: str = _build_sql_query(gadm_specifier) + admin_level: str = gadm_specifier.get_specified_admin_level() + tree_cover_data: TreeCoverData = await _fetch_tree_cover_data(sql_query, admin_level, api_key) except ValueError as e: raise HTTPException(status_code=422, detail=str(e)) except Exception as e: From e095b1cd3ae979487275bb6875bb8816af8787d1 Mon Sep 17 00:00:00 2001 From: Gary Tempus Jr Date: Fri, 6 Sep 2024 12:35:35 -0400 Subject: [PATCH 07/15] :memo: docs: adding the DataMart docstring --- app/routes/datamart/globalforestwatch.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/app/routes/datamart/globalforestwatch.py b/app/routes/datamart/globalforestwatch.py index bc6c026e0..517dca3ee 100644 --- a/app/routes/datamart/globalforestwatch.py +++ b/app/routes/datamart/globalforestwatch.py @@ -1,4 +1,25 @@ -"""APIs for Global Forest Watch data.""" +""" +Data Mart APIs for Global Forest Watch (GFW) backend consumption. + +These APIs provide granular, tailored data services specifically designed to meet the needs of the **Global Forest Watch (GFW)** backend infrastructure. +The endpoints abstract away the complexities of querying datasets related to net tree cover change, allowing the GFW backend to integrate and consume +data efficiently and reliably. + +### Intended Audience: +This API is not designed for general public use. It is purpose-built for internal use by the GFW backend services and systems. The consumers of this +API are expected to have an in-depth understanding of GFW's data models and query requirements. + +### Key Features: +- Tailored queries for retrieving net tree cover change data from the GFW database. +- Efficient data retrieval for ISO country codes and administrative regions. +- Abstracts the SQL query generation process to simplify integration with the backend. + +### Usage: +These endpoints are intended to be consumed programmatically by the GFW backend and are not optimized for external client-facing use. The data +retrieved is intended to support GFW's internal applications and services. + +Specifically, it supports the [Net change in tree cover](https://www.globalforestwatch.org/map/country/BRA/14/?mainMap=eyJzaG93QW5hbHlzaXMiOnRydWV9&map=eyJjZW50ZXIiOnsibGF0IjotMy42MjgwNjcwOTUyMDc3NDc2LCJsbmciOi01Mi40NzQ4OTk5OTk5OTczMzR9LCJ6b29tIjo2LjA1NTQ1ODQ3NjM4NDE1LCJjYW5Cb3VuZCI6ZmFsc2UsImRhdGFzZXRzIjpbeyJkYXRhc2V0IjoiTmV0LUNoYW5nZS1TVEFHSU5HIiwib3BhY2l0eSI6MSwidmlzaWJpbGl0eSI6dHJ1ZSwibGF5ZXJzIjpbImZvcmVzdC1uZXQtY2hhbmdlIl19LHsiZGF0YXNldCI6InBvbGl0aWNhbC1ib3VuZGFyaWVzIiwibGF5ZXJzIjpbImRpc3B1dGVkLXBvbGl0aWNhbC1ib3VuZGFyaWVzIiwicG9saXRpY2FsLWJvdW5kYXJpZXMiXSwib3BhY2l0eSI6MSwidmlzaWJpbGl0eSI6dHJ1ZX1dfQ%3D%3D&mapMenu=eyJtZW51U2VjdGlvbiI6ImRhdGFzZXRzIiwiZGF0YXNldENhdGVnb3J5IjoiZm9yZXN0Q2hhbmdlIn0%3D) widget +""" from enum import Enum from typing import Optional From 066f4095ca1c0a86b32a48cf054af48e0f34f0a1 Mon Sep 17 00:00:00 2001 From: Gary Tempus Jr Date: Fri, 6 Sep 2024 12:40:54 -0400 Subject: [PATCH 08/15] :memo: docs: update top-level doc string --- app/routes/datamart/globalforestwatch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/routes/datamart/globalforestwatch.py b/app/routes/datamart/globalforestwatch.py index 517dca3ee..bbb7ef02b 100644 --- a/app/routes/datamart/globalforestwatch.py +++ b/app/routes/datamart/globalforestwatch.py @@ -1,7 +1,7 @@ """ Data Mart APIs for Global Forest Watch (GFW) backend consumption. -These APIs provide granular, tailored data services specifically designed to meet the needs of the **Global Forest Watch (GFW)** backend infrastructure. +These APIs provide coarse-grained, tailored data services specifically designed to meet the needs of the **Global Forest Watch (GFW)** backend infrastructure. The endpoints abstract away the complexities of querying datasets related to net tree cover change, allowing the GFW backend to integrate and consume data efficiently and reliably. From bc40c96ffe41e5f8628123a3ec1485c2f63f9230 Mon Sep 17 00:00:00 2001 From: Gary Tempus Jr Date: Fri, 6 Sep 2024 12:57:01 -0400 Subject: [PATCH 09/15] :art: refactor: realign abstraction level of router function --- app/routes/datamart/globalforestwatch.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/app/routes/datamart/globalforestwatch.py b/app/routes/datamart/globalforestwatch.py index bbb7ef02b..ac4132111 100644 --- a/app/routes/datamart/globalforestwatch.py +++ b/app/routes/datamart/globalforestwatch.py @@ -31,6 +31,7 @@ from pydantic import Field, root_validator, ValidationError from app.authentication.api_keys import get_api_key +from app.models.orm.api_keys import ApiKey from app.models.pydantic.base import StrictBaseModel router = APIRouter() @@ -111,12 +112,12 @@ class Config: } -def _build_sql_query(request): +def _build_sql_query(gadm_specification: GadmSpecification): select_fields = [Gadm.ISO.value] - where_conditions = [f"{Gadm.ISO.value} = '{request.iso}'"] + where_conditions = [f"{Gadm.ISO.value} = '{gadm_specification.iso}'"] - _append_field_and_condition(select_fields, where_conditions, Gadm.ADM1.value, request.adm1) - _append_field_and_condition(select_fields, where_conditions, Gadm.ADM2.value, request.adm2) + _append_field_and_condition(select_fields, where_conditions, Gadm.ADM1.value, gadm_specification.adm1) + _append_field_and_condition(select_fields, where_conditions, Gadm.ADM2.value, gadm_specification.adm2) select_fields += ["stable", "loss", "gain", "disturb", "net", "change", "gfw_area__ha"] @@ -163,6 +164,12 @@ async def _fetch_tree_cover_data(sql_query: str, level: str, api_key: str) -> Tr return TreeCoverData(**response_data) +async def _get_tree_cover_data(gadm_specification: GadmSpecification, api_key: ApiKey) -> TreeCoverData: + sql_query = _build_sql_query(gadm_specification) + admin_level = gadm_specification.get_specified_admin_level() + return await _fetch_tree_cover_data(sql_query, admin_level, api_key) + + @router.get( "/v1/net_tree_cover_change", response_class=ORJSONResponse, @@ -182,9 +189,7 @@ async def net_tree_cover_change( """ try: gadm_specifier = GadmSpecification(iso=iso, adm1=adm1, adm2=adm2) - sql_query: str = _build_sql_query(gadm_specifier) - admin_level: str = gadm_specifier.get_specified_admin_level() - tree_cover_data: TreeCoverData = await _fetch_tree_cover_data(sql_query, admin_level, api_key) + tree_cover_data: TreeCoverData = await _get_tree_cover_data(gadm_specifier, api_key) except ValueError as e: raise HTTPException(status_code=422, detail=str(e)) except Exception as e: From 8dfe0bae2e478a670d41617ecd19a2b143f85b17 Mon Sep 17 00:00:00 2001 From: Gary Tempus Jr Date: Fri, 6 Sep 2024 13:01:25 -0400 Subject: [PATCH 10/15] :memo: docs: update NetTreeCoverChangeResponse doc --- app/routes/datamart/globalforestwatch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/routes/datamart/globalforestwatch.py b/app/routes/datamart/globalforestwatch.py index ac4132111..cc6a5305a 100644 --- a/app/routes/datamart/globalforestwatch.py +++ b/app/routes/datamart/globalforestwatch.py @@ -88,7 +88,7 @@ class TreeCoverData(StrictBaseModel): class NetTreeCoverChangeResponse(StrictBaseModel): - data: TreeCoverData = Field(..., description="A list of tree cover change data records.") + data: TreeCoverData = Field(..., description="A tree cover change data record.") status: str = Field(..., description="Status of the request (e.g., 'success').") class Config: From 984784e9b5e2f686cb3561062ee69c9edac4496c Mon Sep 17 00:00:00 2001 From: Gary Tempus Jr Date: Thu, 3 Oct 2024 18:23:08 -0400 Subject: [PATCH 11/15] :art: refactor(DataMart): remove app specific part of the path --- app/main.py | 8 ++++---- app/routes/datamart/analysis/__init__.py | 0 .../datamart/analysis/forest_change/__init__.py | 0 .../forest_change/tree_cover_change.py} | 17 +++++------------ 4 files changed, 9 insertions(+), 16 deletions(-) create mode 100644 app/routes/datamart/analysis/__init__.py create mode 100644 app/routes/datamart/analysis/forest_change/__init__.py rename app/routes/datamart/{globalforestwatch.py => analysis/forest_change/tree_cover_change.py} (91%) diff --git a/app/main.py b/app/main.py index c7731ae6f..961a2094b 100644 --- a/app/main.py +++ b/app/main.py @@ -20,7 +20,7 @@ from .routes.analysis import analysis from .routes.assets import asset, assets from .routes.authentication import authentication -from .routes.datamart import globalforestwatch +from app.routes.datamart.analysis.forest_change import tree_cover_change from .routes.datasets import asset as version_asset from .routes.datasets import ( dataset, @@ -168,7 +168,7 @@ async def rve_error_handler( # Data Mart API ############### -app.include_router(globalforestwatch.router, prefix="/datamart/globalforestwatch") +app.include_router(tree_cover_change.router, prefix="/datamart/analysis/forest_change/tree_cover_change") ############### # JOB API @@ -205,7 +205,7 @@ async def rve_error_handler( {"name": "Analysis2", "description": analysis.__doc__}, {"name": "Job", "description": job.__doc__}, {"name": "Health", "description": health.__doc__}, - {"name": "Data Mart", "description": globalforestwatch.__doc__}, + {"name": "Forest Change", "description": tree_cover_change.__doc__}, ] @@ -231,7 +231,7 @@ def custom_openapi(): {"name": "Task API", "tags": ["Tasks"]}, {"name": "Analysis API", "tags": ["Analysis"]}, {"name": "Health API", "tags": ["Health"]}, - {"name": "Data Mart API", "tags": ["Data Mart"]}, + {"name": "Data Mart API", "tags": ["Forest Change"]}, ] app.openapi_schema = openapi_schema diff --git a/app/routes/datamart/analysis/__init__.py b/app/routes/datamart/analysis/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/app/routes/datamart/analysis/forest_change/__init__.py b/app/routes/datamart/analysis/forest_change/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/app/routes/datamart/globalforestwatch.py b/app/routes/datamart/analysis/forest_change/tree_cover_change.py similarity index 91% rename from app/routes/datamart/globalforestwatch.py rename to app/routes/datamart/analysis/forest_change/tree_cover_change.py index cc6a5305a..7206b5c9c 100644 --- a/app/routes/datamart/globalforestwatch.py +++ b/app/routes/datamart/analysis/forest_change/tree_cover_change.py @@ -1,23 +1,16 @@ """ Data Mart APIs for Global Forest Watch (GFW) backend consumption. -These APIs provide coarse-grained, tailored data services specifically designed to meet the needs of the **Global Forest Watch (GFW)** backend infrastructure. -The endpoints abstract away the complexities of querying datasets related to net tree cover change, allowing the GFW backend to integrate and consume +These APIs provide coarse-grained, tailored data services specifically designed to meet the needs of WRI frontend applications. +The endpoints abstract away the complexities of querying datasets related to tree cover change, allowing applications to integrate and consume data efficiently and reliably. -### Intended Audience: -This API is not designed for general public use. It is purpose-built for internal use by the GFW backend services and systems. The consumers of this -API are expected to have an in-depth understanding of GFW's data models and query requirements. - ### Key Features: - Tailored queries for retrieving net tree cover change data from the GFW database. - Efficient data retrieval for ISO country codes and administrative regions. -- Abstracts the SQL query generation process to simplify integration with the backend. +- Abstracts the SQL query generation process to simplify integration with applications. ### Usage: -These endpoints are intended to be consumed programmatically by the GFW backend and are not optimized for external client-facing use. The data -retrieved is intended to support GFW's internal applications and services. - Specifically, it supports the [Net change in tree cover](https://www.globalforestwatch.org/map/country/BRA/14/?mainMap=eyJzaG93QW5hbHlzaXMiOnRydWV9&map=eyJjZW50ZXIiOnsibGF0IjotMy42MjgwNjcwOTUyMDc3NDc2LCJsbmciOi01Mi40NzQ4OTk5OTk5OTczMzR9LCJ6b29tIjo2LjA1NTQ1ODQ3NjM4NDE1LCJjYW5Cb3VuZCI6ZmFsc2UsImRhdGFzZXRzIjpbeyJkYXRhc2V0IjoiTmV0LUNoYW5nZS1TVEFHSU5HIiwib3BhY2l0eSI6MSwidmlzaWJpbGl0eSI6dHJ1ZSwibGF5ZXJzIjpbImZvcmVzdC1uZXQtY2hhbmdlIl19LHsiZGF0YXNldCI6InBvbGl0aWNhbC1ib3VuZGFyaWVzIiwibGF5ZXJzIjpbImRpc3B1dGVkLXBvbGl0aWNhbC1ib3VuZGFyaWVzIiwicG9saXRpY2FsLWJvdW5kYXJpZXMiXSwib3BhY2l0eSI6MSwidmlzaWJpbGl0eSI6dHJ1ZX1dfQ%3D%3D&mapMenu=eyJtZW51U2VjdGlvbiI6ImRhdGFzZXRzIiwiZGF0YXNldENhdGVnb3J5IjoiZm9yZXN0Q2hhbmdlIn0%3D) widget """ from enum import Enum @@ -159,7 +152,7 @@ async def _fetch_tree_cover_data(sql_query: str, level: str, api_key: str) -> Tr logger.error(f"API responded with status code {response.status_code}: {response.content}") raise Exception("Failed to fetch tree cover data.") - # Parse and validate the response data into TreeCoverData models + # Parse and validate the response data into a TreeCoverData model response_data = response.json().get("data", [])[0] return TreeCoverData(**response_data) @@ -174,7 +167,7 @@ async def _get_tree_cover_data(gadm_specification: GadmSpecification, api_key: A "/v1/net_tree_cover_change", response_class=ORJSONResponse, response_model=NetTreeCoverChangeResponse, - tags=["Data Mart"], + tags=["Forest Change"], summary="Retrieve net tree cover change data", description="This endpoint provides data on net tree cover change by querying the Global Forest Watch (GFW) database.", ) From e0e73208f7f9347b8d6ad267c81937293359e7d1 Mon Sep 17 00:00:00 2001 From: Gary Tempus Jr Date: Fri, 4 Oct 2024 13:32:22 -0400 Subject: [PATCH 12/15] :memo: docs(DataMart): update documentation structure --- app/main.py | 11 ++++--- app/routes/datamart/__init__.py | 27 +++++++++++++++ app/routes/datamart/analysis/__init__.py | 16 +++++++++ .../analysis/forest_change/__init__.py | 27 +++++++++++++++ .../forest_change/tree_cover_change.py | 33 +++++++------------ 5 files changed, 88 insertions(+), 26 deletions(-) diff --git a/app/main.py b/app/main.py index 961a2094b..aa5ac532d 100644 --- a/app/main.py +++ b/app/main.py @@ -13,6 +13,7 @@ from starlette.middleware.base import BaseHTTPMiddleware from app.errors import http_error_handler +from app.routes.datamart import data_mart_router, datamart_metadata_tags from .application import app from .middleware import no_cache_response_header, redirect_latest, set_db_mode @@ -20,7 +21,6 @@ from .routes.analysis import analysis from .routes.assets import asset, assets from .routes.authentication import authentication -from app.routes.datamart.analysis.forest_change import tree_cover_change from .routes.datasets import asset as version_asset from .routes.datasets import ( dataset, @@ -168,7 +168,7 @@ async def rve_error_handler( # Data Mart API ############### -app.include_router(tree_cover_change.router, prefix="/datamart/analysis/forest_change/tree_cover_change") +app.include_router(data_mart_router) ############### # JOB API @@ -202,12 +202,13 @@ async def rve_error_handler( {"name": "Download", "description": downloads.__doc__}, {"name": "Geostore", "description": geostore.__doc__}, {"name": "Tasks", "description": task.__doc__}, - {"name": "Analysis2", "description": analysis.__doc__}, + {"name": "Analysis", "description": analysis.__doc__}, {"name": "Job", "description": job.__doc__}, {"name": "Health", "description": health.__doc__}, - {"name": "Forest Change", "description": tree_cover_change.__doc__}, ] +tags_metadata.extend(datamart_metadata_tags) + def custom_openapi(): if app.openapi_schema: @@ -231,7 +232,7 @@ def custom_openapi(): {"name": "Task API", "tags": ["Tasks"]}, {"name": "Analysis API", "tags": ["Analysis"]}, {"name": "Health API", "tags": ["Health"]}, - {"name": "Data Mart API", "tags": ["Forest Change"]}, + {"name": "Data Mart API", "tags": ["Forest Change Analysis 📊"]}, ] app.openapi_schema = openapi_schema diff --git a/app/routes/datamart/__init__.py b/app/routes/datamart/__init__.py index e69de29bb..99a01b42f 100644 --- a/app/routes/datamart/__init__.py +++ b/app/routes/datamart/__init__.py @@ -0,0 +1,27 @@ +""" +Data Mart APIs for Global Forest Watch (GFW) backend consumption. + +These APIs provide coarse-grained, tailored data services specifically designed to meet the needs of WRI frontend applications. +The endpoints abstract away the complexities of querying datasets related to tree cover change, allowing applications to integrate and consume +data efficiently and reliably. + +### Key Features: +- Tailored queries for retrieving net tree cover change data from the GFW database. +- Efficient data retrieval for ISO country codes and administrative regions. +- Abstracts the SQL query generation process to simplify integration with applications. +""" +from fastapi import APIRouter + +from app.routes.datamart.analysis import analysis_router, datamart_analysis_metadata_tags + +datamart_metadata_tags = [ + {"name": "Data Mart", "description": __doc__}, +] + +datamart_metadata_tags.extend(datamart_analysis_metadata_tags) + +data_mart_router = APIRouter( + prefix="/datamart" +) + +data_mart_router.include_router(analysis_router) \ No newline at end of file diff --git a/app/routes/datamart/analysis/__init__.py b/app/routes/datamart/analysis/__init__.py index e69de29bb..d7592d01b 100644 --- a/app/routes/datamart/analysis/__init__.py +++ b/app/routes/datamart/analysis/__init__.py @@ -0,0 +1,16 @@ +from fastapi import APIRouter + +from app.routes.datamart.analysis.forest_change import forest_change_router, \ + datamart_analysis_forest_change_metadata_tags + +datamart_analysis_metadata_tags = [ + {"name": "Data Mart Analysis", "description": __doc__}, +] + +datamart_analysis_metadata_tags.extend(datamart_analysis_forest_change_metadata_tags) + +analysis_router = APIRouter( + prefix="/analysis" +) + +analysis_router.include_router(forest_change_router) \ No newline at end of file diff --git a/app/routes/datamart/analysis/forest_change/__init__.py b/app/routes/datamart/analysis/forest_change/__init__.py index e69de29bb..8e1188aff 100644 --- a/app/routes/datamart/analysis/forest_change/__init__.py +++ b/app/routes/datamart/analysis/forest_change/__init__.py @@ -0,0 +1,27 @@ +""" +Forest Change analysis tools! + +**Legend:** + +⚠️ = _Alerts_ + +🔥 = _Fires_ + +🌳 = _Tree Cover Change_ + +---- +""" +from fastapi import APIRouter + +from app.routes.datamart.analysis.forest_change.tree_cover_change import tree_cover_change_router + +datamart_analysis_forest_change_metadata_tags = [ + {"name": "Forest Change Analysis 📊", "description": __doc__}, +] + +forest_change_router = APIRouter( + prefix="/forest_change", + tags=["Forest Change Analysis 📊"] +) + +forest_change_router.include_router(tree_cover_change_router) \ No newline at end of file diff --git a/app/routes/datamart/analysis/forest_change/tree_cover_change.py b/app/routes/datamart/analysis/forest_change/tree_cover_change.py index 7206b5c9c..b07704a7a 100644 --- a/app/routes/datamart/analysis/forest_change/tree_cover_change.py +++ b/app/routes/datamart/analysis/forest_change/tree_cover_change.py @@ -1,18 +1,3 @@ -""" -Data Mart APIs for Global Forest Watch (GFW) backend consumption. - -These APIs provide coarse-grained, tailored data services specifically designed to meet the needs of WRI frontend applications. -The endpoints abstract away the complexities of querying datasets related to tree cover change, allowing applications to integrate and consume -data efficiently and reliably. - -### Key Features: -- Tailored queries for retrieving net tree cover change data from the GFW database. -- Efficient data retrieval for ISO country codes and administrative regions. -- Abstracts the SQL query generation process to simplify integration with applications. - -### Usage: -Specifically, it supports the [Net change in tree cover](https://www.globalforestwatch.org/map/country/BRA/14/?mainMap=eyJzaG93QW5hbHlzaXMiOnRydWV9&map=eyJjZW50ZXIiOnsibGF0IjotMy42MjgwNjcwOTUyMDc3NDc2LCJsbmciOi01Mi40NzQ4OTk5OTk5OTczMzR9LCJ6b29tIjo2LjA1NTQ1ODQ3NjM4NDE1LCJjYW5Cb3VuZCI6ZmFsc2UsImRhdGFzZXRzIjpbeyJkYXRhc2V0IjoiTmV0LUNoYW5nZS1TVEFHSU5HIiwib3BhY2l0eSI6MSwidmlzaWJpbGl0eSI6dHJ1ZSwibGF5ZXJzIjpbImZvcmVzdC1uZXQtY2hhbmdlIl19LHsiZGF0YXNldCI6InBvbGl0aWNhbC1ib3VuZGFyaWVzIiwibGF5ZXJzIjpbImRpc3B1dGVkLXBvbGl0aWNhbC1ib3VuZGFyaWVzIiwicG9saXRpY2FsLWJvdW5kYXJpZXMiXSwib3BhY2l0eSI6MSwidmlzaWJpbGl0eSI6dHJ1ZX1dfQ%3D%3D&mapMenu=eyJtZW51U2VjdGlvbiI6ImRhdGFzZXRzIiwiZGF0YXNldENhdGVnb3J5IjoiZm9yZXN0Q2hhbmdlIn0%3D) widget -""" from enum import Enum from typing import Optional @@ -27,7 +12,10 @@ from app.models.orm.api_keys import ApiKey from app.models.pydantic.base import StrictBaseModel -router = APIRouter() + +tree_cover_change_router = APIRouter( + prefix="/tree_cover_change" +) class Gadm(str, Enum): @@ -163,13 +151,16 @@ async def _get_tree_cover_data(gadm_specification: GadmSpecification, api_key: A return await _fetch_tree_cover_data(sql_query, admin_level, api_key) -@router.get( - "/v1/net_tree_cover_change", +@tree_cover_change_router.get( + "/net_tree_cover_change", response_class=ORJSONResponse, response_model=NetTreeCoverChangeResponse, - tags=["Forest Change"], - summary="Retrieve net tree cover change data", - description="This endpoint provides data on net tree cover change by querying the Global Forest Watch (GFW) database.", + summary="🌳 Net Tree Cover Change", + description=""" + Retrieve net tree cover change data. + This endpoint provides data on net tree cover change by querying the Global Forest Watch (GFW) database. + Specifically, it supports the [Net change in tree cover](https://www.globalforestwatch.org/map/country/BRA/14/?mainMap=eyJzaG93QW5hbHlzaXMiOnRydWV9&map=eyJjZW50ZXIiOnsibGF0IjotMy42MjgwNjcwOTUyMDc3NDc2LCJsbmciOi01Mi40NzQ4OTk5OTk5OTczMzR9LCJ6b29tIjo2LjA1NTQ1ODQ3NjM4NDE1LCJjYW5Cb3VuZCI6ZmFsc2UsImRhdGFzZXRzIjpbeyJkYXRhc2V0IjoiTmV0LUNoYW5nZS1TVEFHSU5HIiwib3BhY2l0eSI6MSwidmlzaWJpbGl0eSI6dHJ1ZSwibGF5ZXJzIjpbImZvcmVzdC1uZXQtY2hhbmdlIl19LHsiZGF0YXNldCI6InBvbGl0aWNhbC1ib3VuZGFyaWVzIiwibGF5ZXJzIjpbImRpc3B1dGVkLXBvbGl0aWNhbC1ib3VuZGFyaWVzIiwicG9saXRpY2FsLWJvdW5kYXJpZXMiXSwib3BhY2l0eSI6MSwidmlzaWJpbGl0eSI6dHJ1ZX1dfQ%3D%3D&mapMenu=eyJtZW51U2VjdGlvbiI6ImRhdGFzZXRzIiwiZGF0YXNldENhdGVnb3J5IjoiZm9yZXN0Q2hhbmdlIn0%3D) widget. + """, ) async def net_tree_cover_change( iso: str = Query(..., description="ISO code of the country or region (e.g., 'BRA' for Brazil).", example="BRA"), From b616d7ea469d69088d395b5a558138ea2824a783 Mon Sep 17 00:00:00 2001 From: Gary Tempus Jr Date: Fri, 4 Oct 2024 17:56:57 -0400 Subject: [PATCH 13/15] :white_check_mark: test(DataMart): write happy path tests for net_tree_cover_change --- tests_v2/unit/app/routes/datamart/__init__.py | 0 .../app/routes/datamart/analysis/__init__.py | 0 .../analysis/forest_change/__init__.py | 0 .../test_net_tree_cover_change.py | 181 ++++++++++++++++++ 4 files changed, 181 insertions(+) create mode 100644 tests_v2/unit/app/routes/datamart/__init__.py create mode 100644 tests_v2/unit/app/routes/datamart/analysis/__init__.py create mode 100644 tests_v2/unit/app/routes/datamart/analysis/forest_change/__init__.py create mode 100644 tests_v2/unit/app/routes/datamart/analysis/forest_change/test_net_tree_cover_change.py diff --git a/tests_v2/unit/app/routes/datamart/__init__.py b/tests_v2/unit/app/routes/datamart/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests_v2/unit/app/routes/datamart/analysis/__init__.py b/tests_v2/unit/app/routes/datamart/analysis/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests_v2/unit/app/routes/datamart/analysis/forest_change/__init__.py b/tests_v2/unit/app/routes/datamart/analysis/forest_change/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests_v2/unit/app/routes/datamart/analysis/forest_change/test_net_tree_cover_change.py b/tests_v2/unit/app/routes/datamart/analysis/forest_change/test_net_tree_cover_change.py new file mode 100644 index 000000000..63d3245fb --- /dev/null +++ b/tests_v2/unit/app/routes/datamart/analysis/forest_change/test_net_tree_cover_change.py @@ -0,0 +1,181 @@ +from unittest.mock import AsyncMock, patch, ANY +from fastapi import status +import pytest +from httpx import AsyncClient + +from app.routes.datamart.analysis.forest_change.tree_cover_change import TreeCoverData + +# Define the stubbed response for _fetch_tree_cover_data so the route is successful +stubbed_tree_cover_data = TreeCoverData( + iso="BRA", + adm1=12, + adm2=34, + stable=413722809.3, + loss=36141245.77, + gain=8062324.946, + disturb=23421628.86, + net=-28078920.83, + change=-5.932759761810303, + gfw_area__ha=850036547.481532 +) + + +# Common helper function to send the request +async def send_tree_cover_change_request(api_key, async_client: AsyncClient, params): + headers = {"x-api-key": api_key} + response = await async_client.get( + f"/datamart/analysis/forest_change/tree_cover_change/net_tree_cover_change", + params=params, + headers=headers, + follow_redirects=True, + ) + return response + + +@pytest.mark.asyncio +async def test_net_tree_cover_change_builds_succeeds( + apikey, async_client: AsyncClient +): + with patch( + "app.routes.datamart.analysis.forest_change.tree_cover_change._fetch_tree_cover_data", + AsyncMock(return_value=stubbed_tree_cover_data) + ): + api_key, payload = apikey + params = {"iso": "BRA"} + + response = await send_tree_cover_change_request(api_key, async_client, params) + + assert response.status_code == status.HTTP_200_OK, "Expected status code 200 OK" + + +@pytest.mark.asyncio +class TestSQLBuilding: + @pytest.mark.asyncio + async def test_net_tree_cover_change_builds_sql_with_iso(self, apikey, async_client: AsyncClient): + with patch( + "app.routes.datamart.analysis.forest_change.tree_cover_change._fetch_tree_cover_data", + AsyncMock(return_value=stubbed_tree_cover_data) + ) as mock_fetch: + api_key, payload = apikey + params = {"iso": "BRA"} + + await send_tree_cover_change_request(api_key, async_client, params) + + mock_fetch.assert_called_once_with( + "SELECT iso, stable, loss, gain, disturb, net, change, gfw_area__ha FROM data WHERE iso = 'BRA'", + # SQL query + ANY, # Ignore admin level + ANY # Ignore API Key + ) + + + @pytest.mark.asyncio + async def test_net_tree_cover_change_builds_sql_with_adm1(self, apikey, async_client: AsyncClient): + with patch( + "app.routes.datamart.analysis.forest_change.tree_cover_change._fetch_tree_cover_data", + AsyncMock(return_value=stubbed_tree_cover_data) + ) as mock_fetch: + api_key, payload = apikey + params = {"iso": "BRA", "adm1": 12} + + await send_tree_cover_change_request(api_key, async_client, params) + + mock_fetch.assert_called_once_with( + "SELECT iso, adm1, stable, loss, gain, disturb, net, change, gfw_area__ha FROM data WHERE iso = 'BRA' AND adm1 = '12'", + # SQL query + ANY, # Ignore admin level + ANY # Ignore API Key + ) + + + @pytest.mark.asyncio + async def test_net_tree_cover_change_builds_sql_with_adm2(self, apikey, async_client: AsyncClient): + with patch( + "app.routes.datamart.analysis.forest_change.tree_cover_change._fetch_tree_cover_data", + AsyncMock(return_value=stubbed_tree_cover_data) + ) as mock_fetch: + api_key, payload = apikey + params = {"iso": "BRA", "adm1": 12, "adm2": 34} + + await send_tree_cover_change_request(api_key, async_client, params) + + mock_fetch.assert_called_once_with( + "SELECT iso, adm1, adm2, stable, loss, gain, disturb, net, change, gfw_area__ha FROM data WHERE iso = 'BRA' AND adm1 = '12' AND adm2 = '34'", + # SQL query + ANY, # Ignore admin level + ANY # Ignore API Key + ) + + +@pytest.mark.asyncio +class TestAdminLevel: + @pytest.mark.asyncio + async def test_net_tree_cover_change_passes_iso(self, apikey, async_client): + api_key, payload = apikey + with patch( + "app.routes.datamart.analysis.forest_change.tree_cover_change._fetch_tree_cover_data", + AsyncMock(return_value=stubbed_tree_cover_data) + ) as mock_fetch: + params = {"iso": "BRA"} + + await send_tree_cover_change_request(api_key, async_client, params) + + mock_fetch.assert_called_once_with( + ANY, # Ignore SQL + 'adm0', # most precise adm level + ANY # Ignore API Key + ) + + @pytest.mark.asyncio + async def test_net_tree_cover_change_passes_adm1(self, apikey, async_client): + api_key, payload = apikey + with patch( + "app.routes.datamart.analysis.forest_change.tree_cover_change._fetch_tree_cover_data", + AsyncMock(return_value=stubbed_tree_cover_data) + ) as mock_fetch: + params = {"iso": "BRA", "adm1": "12"} + + await send_tree_cover_change_request(api_key, async_client, params) + + mock_fetch.assert_called_once_with( + ANY, # Ignore SQL + 'adm1', # most precise adm level + ANY # Ignore API Key + ) + + @pytest.mark.asyncio + async def test_net_tree_cover_change_passes_adm2(self, apikey, async_client): + api_key, payload = apikey + with patch( + "app.routes.datamart.analysis.forest_change.tree_cover_change._fetch_tree_cover_data", + AsyncMock(return_value=stubbed_tree_cover_data) + ) as mock_fetch: + params = {"iso": "BRA", "adm1": "12", "adm2": "34"} + + await send_tree_cover_change_request(api_key, async_client, params) + + mock_fetch.assert_called_once_with( + ANY, # Ignore SQL + 'adm2', # most precise adm level + ANY # Ignore API Key + ) + + +@pytest.mark.asyncio +async def test_net_tree_cover_change_passes_api_key( + apikey, async_client: AsyncClient +): + with patch( + "app.routes.datamart.analysis.forest_change.tree_cover_change._fetch_tree_cover_data", + AsyncMock(return_value=stubbed_tree_cover_data) + ) as mock_fetch: + api_key, payload = apikey + params = {"iso": "BRA"} + + await send_tree_cover_change_request(api_key, async_client, params) + + mock_fetch.assert_called_once_with( + ANY, # Ignore SQL + ANY, # Ignore admin level + api_key # api key + ) From 6405beefc231b2c330a338ce88aafc47f90fbb7b Mon Sep 17 00:00:00 2001 From: Gary Tempus Jr Date: Fri, 4 Oct 2024 18:47:03 -0400 Subject: [PATCH 14/15] :art: refactor: remove unneeded import --- app/routes/analysis/analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/routes/analysis/analysis.py b/app/routes/analysis/analysis.py index 9d33cdb38..4843542e6 100644 --- a/app/routes/analysis/analysis.py +++ b/app/routes/analysis/analysis.py @@ -17,7 +17,7 @@ from ...settings.globals import GEOSTORE_SIZE_LIMIT_OTF from ...utils.geostore import get_geostore from .. import DATE_REGEX -from ..datasets.queries import _query_raster_lambda, _query_dataset_json +from ..datasets.queries import _query_raster_lambda router = APIRouter() From 6a6d58ecf78af6943bfa572243b8326db165690c Mon Sep 17 00:00:00 2001 From: Gary Tempus Jr Date: Mon, 7 Oct 2024 18:29:52 -0400 Subject: [PATCH 15/15] :white_check_mark: test(DataMart): rename test --- .../analysis/forest_change/test_net_tree_cover_change.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests_v2/unit/app/routes/datamart/analysis/forest_change/test_net_tree_cover_change.py b/tests_v2/unit/app/routes/datamart/analysis/forest_change/test_net_tree_cover_change.py index 63d3245fb..802c51c3e 100644 --- a/tests_v2/unit/app/routes/datamart/analysis/forest_change/test_net_tree_cover_change.py +++ b/tests_v2/unit/app/routes/datamart/analysis/forest_change/test_net_tree_cover_change.py @@ -33,7 +33,7 @@ async def send_tree_cover_change_request(api_key, async_client: AsyncClient, par @pytest.mark.asyncio -async def test_net_tree_cover_change_builds_succeeds( +async def test_net_tree_cover_change_succeeds( apikey, async_client: AsyncClient ): with patch(