From 6366120f2fea368efaa7b1f2d350ebb5f8c1f688 Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Fri, 25 Oct 2024 22:26:56 +0300 Subject: [PATCH 01/14] dont use tilefactory for algos for ease of customization; make titiler endpoints consistent with tile cache service --- app/main.py | 8 +- app/routes/titiler/algorithms/alerts.py | 62 ++++++--------- app/routes/titiler/integrated_alerts.py | 92 ++++++++++++++++++++++ app/routes/titiler/readers.py | 3 +- app/routes/titiler/routes.py | 24 +----- app/routes/titiler/umd_glad_dist_alerts.py | 75 ++++++++++++++++++ tests/titiler/test_alerts_algo.py | 7 +- 7 files changed, 199 insertions(+), 72 deletions(-) create mode 100644 app/routes/titiler/integrated_alerts.py create mode 100644 app/routes/titiler/umd_glad_dist_alerts.py diff --git a/app/main.py b/app/main.py index c3a67f7e..25539586 100644 --- a/app/main.py +++ b/app/main.py @@ -42,6 +42,8 @@ from .routes import preview from .routes.titiler import routes as titiler_routes +from .routes.titiler.integrated_alerts import router as alerts_router +from .routes.titiler.umd_glad_dist_alerts import router as dist_alerts_router gunicorn_logger = logging.getLogger("gunicorn.error") logger.handlers = gunicorn_logger.handlers @@ -55,6 +57,8 @@ burned_areas_tiles.router, dynamic_vector_tiles.router, vector_tiles.router, + alerts_router, + dist_alerts_router, umd_tree_cover_loss_raster_tiles.router, umd_glad_landsat_alerts_raster_tiles.router, umd_glad_sentinel2_alerts_raster_tiles.router, @@ -79,10 +83,6 @@ app.include_router( titiler_routes.mosaic.router, prefix="/cog/mosaic", tags=["Mosaic Tiles"] ) -app.include_router( - titiler_routes.custom.router, prefix="/cog/custom", tags=["Custom Tiles"] -) - ##################### ## Middleware diff --git a/app/routes/titiler/algorithms/alerts.py b/app/routes/titiler/algorithms/alerts.py index e80e2642..103e816e 100644 --- a/app/routes/titiler/algorithms/alerts.py +++ b/app/routes/titiler/algorithms/alerts.py @@ -1,10 +1,8 @@ from collections import OrderedDict, namedtuple -from datetime import date +from typing import Optional import numpy as np -from dateutil.relativedelta import relativedelta from fastapi.logger import logger -from pydantic import Field from rio_tiler.models import ImageData from titiler.core.algorithm import BaseAlgorithm @@ -36,28 +34,10 @@ class Alerts(BaseAlgorithm): record_start_date: str = "2014-12-31" - today: date = date.today() - - # Parameters - default_start_date: str = (today - relativedelta(days=180)).strftime("%Y-%m-%d") - start_date: str = Field( - default_start_date, - description="start date of alert in YYYY-MM-DD format.", - ) - - default_end_date: str = today.strftime("%Y-%m-%d") - end_date: str = Field( - default_end_date, description="end date of alert in YYYY-MM-DD format." - ) - - alert_confidence: AlertConfidence = Field( - AlertConfidence.low, description="Alert confidence" - ) - - render_type: RenderType = Field( - RenderType.true_color, - description="Render true color or encoded pixels", - ) + start_date: Optional[str] = None + end_date: Optional[str] = None + alert_confidence: Optional[AlertConfidence] = None + render_type: Optional[RenderType] = RenderType.true_color # metadata input_nbands: int = 2 @@ -105,18 +85,27 @@ def create_mask(self): np.ndarray: A mask array pixels with no alert or alerts not meeting filter condition are masked. """ - start_mask = self.alert_date >= ( - np.datetime64(self.start_date) - np.datetime64(self.record_start_date) - ) - end_mask = self.alert_date <= ( - np.datetime64(self.end_date) - np.datetime64(self.record_start_date) - ) - confidence_mask = ( - self.data_alert_confidence - >= self.conf_colors[self.alert_confidence].confidence - ) - mask = ~self.no_data * start_mask * end_mask * confidence_mask + mask = ~self.no_data + + if self.alert_confidence: + confidence_mask = ( + self.data_alert_confidence + >= self.conf_colors[self.alert_confidence].confidence + ) + mask *= confidence_mask + + if self.start_date: + start_mask = self.alert_date >= ( + np.datetime64(self.start_date) - np.datetime64(self.record_start_date) + ) + mask *= start_mask + + if self.end_date: + end_mask = self.alert_date <= ( + np.datetime64(self.end_date) - np.datetime64(self.record_start_date) + ) + mask *= end_mask return mask @@ -131,7 +120,6 @@ def create_true_color_rgb(self): for properties in self.conf_colors.values(): confidence = properties.confidence colors = properties.colors - r[self.data_alert_confidence >= confidence] = colors.red g[self.data_alert_confidence >= confidence] = colors.green b[self.data_alert_confidence >= confidence] = colors.blue diff --git a/app/routes/titiler/integrated_alerts.py b/app/routes/titiler/integrated_alerts.py new file mode 100644 index 00000000..419c7a9b --- /dev/null +++ b/app/routes/titiler/integrated_alerts.py @@ -0,0 +1,92 @@ +import os +from typing import Optional, Tuple + +from aenum import Enum, extend_enum +from fastapi import APIRouter, Depends, Query, Response +from titiler.core.resources.enums import ImageType +from titiler.core.utils import render_image + +from ...crud.sync_db.tile_cache_assets import get_versions +from ...models.enumerators.tile_caches import TileCacheType +from ...models.enumerators.titiler import RenderType +from .. import DATE_REGEX, optional_implementation_dependency, raster_xyz +from .algorithms.integrated_alerts import IntegratedAlerts +from .readers import AlertsReader + +DATA_LAKE_BUCKET = os.environ.get("DATA_LAKE_BUCKET") + +router = APIRouter() + +dataset = "gfw_integrated_alerts" + + +class GfwIntegratdAlertsVersions(str, Enum): + """GFW Integrated Alerts versions. + + When using `latest` call will be redirected (307) to version tagged + as latest. + """ + + latest = "latest" + + +_versions = get_versions(dataset, TileCacheType.cog) +for _version in _versions: + extend_enum(GfwIntegratdAlertsVersions, _version, _version) + + +# will turn this on when we're ready to replace tile cache service +# @router.get( +# f"/{dataset}/{{version}}/dynamic/{{z}}/{{x}}/{{y}}.png", +# response_class=Response, +# tags=["Raster Tiles"], +# response_description="PNG Raster Tile", +# ) +@router.get( + f"/{dataset}/{{version}}/titiler/{{z}}/{{x}}/{{y}}.png", + response_class=Response, + tags=["Raster Tiles"], + response_description="PNG Raster Tile", +) +async def gfw_integrated_alerts_raster_tile( + *, + version: GfwIntegratdAlertsVersions, + xyz: Tuple[int, int, int] = Depends(raster_xyz), + start_date: Optional[str] = Query( + None, + regex=DATE_REGEX, + description="Only show alerts for given date and after", + ), + end_date: Optional[str] = Query( + None, regex=DATE_REGEX, description="Only show alerts until given date." + ), + render_type: RenderType = Query( + RenderType.encoded, description="Render true color or encoded tiles" + ), + alert_confidence: Optional[bool] = Query( + None, description="Show alerts with at least this confidence level" + ), + implementation: str = Depends(optional_implementation_dependency), +) -> Response: + """GFW Integrated Alerts raster tiles.""" + + bands = ["default", "intensity"] + folder: str = f"s3://{DATA_LAKE_BUCKET}/{dataset}/{version}/raster/epsg-4326/cog" + with AlertsReader(input=folder) as reader: + tile_x, tile_y, zoom = xyz + image_data = reader.tile(tile_x, tile_y, zoom, bands=bands) + + processed_image = IntegratedAlerts( + start_date=start_date, + end_date=end_date, + render_type=render_type, + alert_confidence=alert_confidence, + )(image_data) + + content, media_type = render_image( + processed_image, + output_format=ImageType("png"), + add_mask=False, + ) + + return Response(content, media_type=media_type) diff --git a/app/routes/titiler/readers.py b/app/routes/titiler/readers.py index 22660aee..542950f4 100644 --- a/app/routes/titiler/readers.py +++ b/app/routes/titiler/readers.py @@ -6,10 +6,9 @@ @attr.s -class IntegratedAlertsReader(MultiBandReader): +class AlertsReader(MultiBandReader): input: str = attr.ib() - # bands: Sequence[str] = attr.ib(init=False) tms: morecantile.TileMatrixSet = attr.ib( default=morecantile.tms.get("WebMercatorQuad") ) diff --git a/app/routes/titiler/routes.py b/app/routes/titiler/routes.py index b0eb659b..f6f42b5b 100644 --- a/app/routes/titiler/routes.py +++ b/app/routes/titiler/routes.py @@ -1,32 +1,10 @@ """Titiler Dynamic Raster tiles for Cloud Optimized Geotiffs (COG)""" -from typing import Callable - -from titiler.core.algorithm import Algorithms -from titiler.core.algorithm import algorithms as default_algorithms -from titiler.core.factory import AlgorithmFactory, MultiBandTilerFactory, TilerFactory +from titiler.core.factory import AlgorithmFactory, TilerFactory from titiler.extensions import cogValidateExtension, cogViewerExtension from titiler.mosaic.factory import MosaicTilerFactory from ...routes import cog_asset_dependency -from .algorithms.integrated_alerts import IntegratedAlerts -from .algorithms.dist_alerts import DISTAlerts -from .readers import IntegratedAlertsReader - - -algorithms: Algorithms = default_algorithms.register( - {"integrated_alerts": IntegratedAlerts, "dist_alerts": DISTAlerts} -) - -# Create a PostProcessParams dependency -PostProcessParams: Callable = algorithms.dependency - -custom = MultiBandTilerFactory( - router_prefix="/cog/custom", - process_dependency=PostProcessParams, - reader=IntegratedAlertsReader, - path_dependency=cog_asset_dependency, -) cog = TilerFactory( router_prefix="/cog/basic", diff --git a/app/routes/titiler/umd_glad_dist_alerts.py b/app/routes/titiler/umd_glad_dist_alerts.py new file mode 100644 index 00000000..abb01bb1 --- /dev/null +++ b/app/routes/titiler/umd_glad_dist_alerts.py @@ -0,0 +1,75 @@ +import os +from datetime import date +from typing import Optional, Tuple + +from dateutil.relativedelta import relativedelta +from fastapi import APIRouter, Depends, Query, Response +from titiler.core.resources.enums import ImageType +from titiler.core.utils import render_image + +from ...models.enumerators.titiler import AlertConfidence, RenderType +from .. import DATE_REGEX, optional_implementation_dependency, raster_xyz +from .algorithms.dist_alerts import DISTAlerts +from .readers import AlertsReader + +DATA_LAKE_BUCKET = os.environ.get("DATA_LAKE_BUCKET") + +router = APIRouter() + +# TODO: update to the actual dataset when ready +dataset = "dan_test" + +today = date.today() + + +@router.get( + f"/{dataset}/{{version}}/dynamic/{{z}}/{{x}}/{{y}}.png", + response_class=Response, + tags=["Raster Tiles"], + response_description="PNG Raster Tile", +) +async def glad_dist_alerts_raster_tile( + *, + version, + xyz: Tuple[int, int, int] = Depends(raster_xyz), + start_date: Optional[str] = Query( + (today - relativedelta(days=180)).strftime("%Y-%m-%d"), + regex=DATE_REGEX, + description="Only show alerts for given date and after", + ), + end_date: Optional[str] = Query( + today.strftime("%Y-%m-%d"), + regex=DATE_REGEX, + description="Only show alerts until given date.", + ), + render_type: RenderType = Query( + RenderType.true_color, description="Render true color or encoded tiles" + ), + alert_confidence: Optional[bool] = Query( + AlertConfidence.low, + description="Show alerts that are at least of this confidence level", + ), + implementation: str = Depends(optional_implementation_dependency), +) -> Response: + """UMD GLAD DIST alerts raster tiles.""" + + bands = ["default", "intensity"] + folder: str = f"s3://{DATA_LAKE_BUCKET}/{dataset}/{version}/raster/epsg-4326/cog" + with AlertsReader(input=folder) as reader: + tile_x, tile_y, zoom = xyz + image_data = reader.tile(tile_x, tile_y, zoom, bands=bands) + + processed_image = DISTAlerts( + start_date=start_date, + end_date=end_date, + render_type=render_type, + alert_confidence=alert_confidence, + )(image_data) + + content, media_type = render_image( + processed_image, + output_format=ImageType("png"), + add_mask=False, + ) + + return Response(content, media_type=media_type) diff --git a/tests/titiler/test_alerts_algo.py b/tests/titiler/test_alerts_algo.py index ed69dca6..72b90d75 100644 --- a/tests/titiler/test_alerts_algo.py +++ b/tests/titiler/test_alerts_algo.py @@ -29,12 +29,7 @@ def test_integrated_alerts_defaults(): """Test default values of the Alerts class.""" alerts = IntegratedAlerts() - assert alerts.start_date == (today - relativedelta(days=alert_period)).strftime( - "%Y-%m-%d" - ) - assert alerts.end_date == today.strftime("%Y-%m-%d") - assert alerts.alert_confidence == AlertConfidence.low - assert alerts.record_start_date == "2014-12-31" + assert alerts.render_type == RenderType.true_color def test_create_date_range_mask(): From d0e57d498b1f7b90ceb274dfebc35f6be29bfe10 Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Fri, 25 Oct 2024 22:32:07 +0300 Subject: [PATCH 02/14] make encoded the default render_type --- app/routes/titiler/umd_glad_dist_alerts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/routes/titiler/umd_glad_dist_alerts.py b/app/routes/titiler/umd_glad_dist_alerts.py index abb01bb1..2f3e4daf 100644 --- a/app/routes/titiler/umd_glad_dist_alerts.py +++ b/app/routes/titiler/umd_glad_dist_alerts.py @@ -43,7 +43,7 @@ async def glad_dist_alerts_raster_tile( description="Only show alerts until given date.", ), render_type: RenderType = Query( - RenderType.true_color, description="Render true color or encoded tiles" + RenderType.encoded, description="Render true color or encoded tiles" ), alert_confidence: Optional[bool] = Query( AlertConfidence.low, From d255c24244d97e5c468808f6a4799dc7da7fd368 Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Fri, 25 Oct 2024 23:05:49 +0300 Subject: [PATCH 03/14] don't use default alertconfidence enum; create separate alertconf enum for dist/integrated alerts as we surface in api doc --- app/models/enumerators/titiler.py | 5 +++++ app/routes/titiler/algorithms/alerts.py | 18 +++--------------- .../titiler/algorithms/integrated_alerts.py | 8 ++++---- app/routes/titiler/integrated_alerts.py | 7 ++++--- app/routes/titiler/umd_glad_dist_alerts.py | 2 +- tests/titiler/test_alerts_algo.py | 7 +++---- 6 files changed, 20 insertions(+), 27 deletions(-) diff --git a/app/models/enumerators/titiler.py b/app/models/enumerators/titiler.py index 545971c9..74d82fbf 100644 --- a/app/models/enumerators/titiler.py +++ b/app/models/enumerators/titiler.py @@ -4,6 +4,11 @@ class AlertConfidence(str, Enum): low = "low" high = "high" + + +class IntegratedAlertConfidence(str, Enum): + low = "low" + high = "high" highest = "highest" diff --git a/app/routes/titiler/algorithms/alerts.py b/app/routes/titiler/algorithms/alerts.py index 103e816e..ac0bc8fd 100644 --- a/app/routes/titiler/algorithms/alerts.py +++ b/app/routes/titiler/algorithms/alerts.py @@ -6,7 +6,7 @@ from rio_tiler.models import ImageData from titiler.core.algorithm import BaseAlgorithm -from app.models.enumerators.titiler import AlertConfidence, RenderType +from app.models.enumerators.titiler import RenderType Colors: namedtuple = namedtuple("Colors", ["red", "green", "blue"]) AlertConfig: namedtuple = namedtuple("AlertConfig", ["confidence", "colors"]) @@ -18,25 +18,13 @@ class Alerts(BaseAlgorithm): title: str = "Deforestation Alerts" description: str = "Decode and visualize alerts" - conf_colors: OrderedDict = OrderedDict( - { - AlertConfidence.low: AlertConfig( - confidence=2, colors=Colors(237, 164, 194) - ), - AlertConfidence.high: AlertConfig( - confidence=3, colors=Colors(220, 102, 153) - ), - AlertConfidence.highest: AlertConfig( - confidence=4, colors=Colors(201, 42, 109) - ), - } - ) + conf_colors: OrderedDict = None record_start_date: str = "2014-12-31" start_date: Optional[str] = None end_date: Optional[str] = None - alert_confidence: Optional[AlertConfidence] = None + alert_confidence: Optional[str] = None render_type: Optional[RenderType] = RenderType.true_color # metadata diff --git a/app/routes/titiler/algorithms/integrated_alerts.py b/app/routes/titiler/algorithms/integrated_alerts.py index ea13fa5a..8a36e816 100644 --- a/app/routes/titiler/algorithms/integrated_alerts.py +++ b/app/routes/titiler/algorithms/integrated_alerts.py @@ -2,7 +2,7 @@ import numpy as np -from app.models.enumerators.titiler import AlertConfidence +from app.models.enumerators.titiler import IntegratedAlertConfidence from .alerts import AlertConfig, Alerts, Colors @@ -13,13 +13,13 @@ class IntegratedAlerts(Alerts): conf_colors: OrderedDict = OrderedDict( { - AlertConfidence.low: AlertConfig( + IntegratedAlertConfidence.low: AlertConfig( confidence=2, colors=Colors(237, 164, 194) ), - AlertConfidence.high: AlertConfig( + IntegratedAlertConfidence.high: AlertConfig( confidence=3, colors=Colors(220, 102, 153) ), - AlertConfidence.highest: AlertConfig( + IntegratedAlertConfidence.highest: AlertConfig( confidence=4, colors=Colors(201, 42, 109) ), } diff --git a/app/routes/titiler/integrated_alerts.py b/app/routes/titiler/integrated_alerts.py index 419c7a9b..9866d1d4 100644 --- a/app/routes/titiler/integrated_alerts.py +++ b/app/routes/titiler/integrated_alerts.py @@ -8,7 +8,7 @@ from ...crud.sync_db.tile_cache_assets import get_versions from ...models.enumerators.tile_caches import TileCacheType -from ...models.enumerators.titiler import RenderType +from ...models.enumerators.titiler import IntegratedAlertConfidence, RenderType from .. import DATE_REGEX, optional_implementation_dependency, raster_xyz from .algorithms.integrated_alerts import IntegratedAlerts from .readers import AlertsReader @@ -63,8 +63,9 @@ async def gfw_integrated_alerts_raster_tile( render_type: RenderType = Query( RenderType.encoded, description="Render true color or encoded tiles" ), - alert_confidence: Optional[bool] = Query( - None, description="Show alerts with at least this confidence level" + alert_confidence: Optional[IntegratedAlertConfidence] = Query( + IntegratedAlertConfidence.low, + description="Show alerts with at least this confidence level", ), implementation: str = Depends(optional_implementation_dependency), ) -> Response: diff --git a/app/routes/titiler/umd_glad_dist_alerts.py b/app/routes/titiler/umd_glad_dist_alerts.py index 2f3e4daf..a740ae46 100644 --- a/app/routes/titiler/umd_glad_dist_alerts.py +++ b/app/routes/titiler/umd_glad_dist_alerts.py @@ -45,7 +45,7 @@ async def glad_dist_alerts_raster_tile( render_type: RenderType = Query( RenderType.encoded, description="Render true color or encoded tiles" ), - alert_confidence: Optional[bool] = Query( + alert_confidence: Optional[AlertConfidence] = Query( AlertConfidence.low, description="Show alerts that are at least of this confidence level", ), diff --git a/tests/titiler/test_alerts_algo.py b/tests/titiler/test_alerts_algo.py index 72b90d75..44acdd0d 100644 --- a/tests/titiler/test_alerts_algo.py +++ b/tests/titiler/test_alerts_algo.py @@ -2,10 +2,9 @@ import numpy as np import rasterio -from dateutil.relativedelta import relativedelta from rio_tiler.models import ImageData -from app.models.enumerators.titiler import AlertConfidence, RenderType +from app.models.enumerators.titiler import IntegratedAlertConfidence, RenderType from app.routes.titiler.algorithms.integrated_alerts import IntegratedAlerts from tests.conftest import DATE_CONF_TIF, INTENSITY_TIF @@ -50,7 +49,7 @@ def test_create_date_range_mask(): def test_create_confidence_mask(): """Test confidence filters are applied correctly.""" - alerts = IntegratedAlerts(alert_confidence=AlertConfidence.highest) + alerts = IntegratedAlerts(alert_confidence=IntegratedAlertConfidence.highest) alerts.start_date = alerts.record_start_date img = get_tile_data() @@ -63,7 +62,7 @@ def test_create_confidence_mask(): def test_mask_logic_with_nodata(): """Test that the mask properly handles no-data values.""" - alerts = IntegratedAlerts(alert_confidence=AlertConfidence.low) + alerts = IntegratedAlerts(alert_confidence=IntegratedAlertConfidence.low) img = get_tile_data() From 148ae0234bb9b4ba30af4c67b7e2e83f11e231ab Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Fri, 25 Oct 2024 23:37:58 +0300 Subject: [PATCH 04/14] update titiler test to point to new endpoint --- app/routes/titiler/integrated_alerts.py | 3 ++- tests/conftest.py | 12 ++++++++++-- tests/routes/test_cog_dynamic_tiles.py | 4 +--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/app/routes/titiler/integrated_alerts.py b/app/routes/titiler/integrated_alerts.py index 9866d1d4..e7583916 100644 --- a/app/routes/titiler/integrated_alerts.py +++ b/app/routes/titiler/integrated_alerts.py @@ -33,6 +33,7 @@ class GfwIntegratdAlertsVersions(str, Enum): _versions = get_versions(dataset, TileCacheType.cog) for _version in _versions: extend_enum(GfwIntegratdAlertsVersions, _version, _version) +# TODO: add version validation # will turn this on when we're ready to replace tile cache service @@ -50,7 +51,7 @@ class GfwIntegratdAlertsVersions(str, Enum): ) async def gfw_integrated_alerts_raster_tile( *, - version: GfwIntegratdAlertsVersions, + version, xyz: Tuple[int, int, int] = Depends(raster_xyz), start_date: Optional[str] = Query( None, diff --git a/tests/conftest.py b/tests/conftest.py index 81be8b5a..ca4259c6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -80,8 +80,16 @@ def prep_titiler_tifs(): "s3", endpoint_url=AWS_ENDPOINT_URI, ) - s3_client.upload_file(DATE_CONF_TIF, "gfw-data-lake-test", "default.tif") - s3_client.upload_file(DATE_CONF_TIF, "gfw-data-lake-test", "intensity.tif") + s3_client.upload_file( + DATE_CONF_TIF, + "gfw-data-lake-test", + "gfw_integrated_alerts/v20201012/raster/epsg-4326/cog/default.tif", + ) + s3_client.upload_file( + DATE_CONF_TIF, + "gfw-data-lake-test", + "gfw_integrated_alerts/v20201012/raster/epsg-4326/cog/intensity.tif", + ) s3_client.upload_file( COG_TIF, "gfw-data-lake-test", diff --git a/tests/routes/test_cog_dynamic_tiles.py b/tests/routes/test_cog_dynamic_tiles.py index e2ef01d6..9e131d1e 100644 --- a/tests/routes/test_cog_dynamic_tiles.py +++ b/tests/routes/test_cog_dynamic_tiles.py @@ -19,9 +19,7 @@ async def test_tile_for_data_api_dataset(client): @pytest.mark.asyncio async def test_tiling_with_custom_rendering(client): """Test Integrated Alerts Tile Rendering.""" - response = client.get( - "/cog/custom/tiles/WebMercatorQuad/14/5305/8879?url=s3://gfw-data-lake-test&bands=default&bands=intensity&algorithm=integrated_alerts&return_mask=False&format=png" - ) + response = client.get("/gfw_integrated_alerts/v20201012/titiler/14/5305/8879.png") response.status_code == 200 assert response.headers["content-type"] == "image/png" From 90825e2714096331ca38a7c17a778a4117d64f0a Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Mon, 28 Oct 2024 18:20:30 +0300 Subject: [PATCH 05/14] remove unused implementation query param from titiler endpoint --- app/routes/titiler/integrated_alerts.py | 3 +-- app/routes/titiler/umd_glad_dist_alerts.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/app/routes/titiler/integrated_alerts.py b/app/routes/titiler/integrated_alerts.py index e7583916..88f5d2e8 100644 --- a/app/routes/titiler/integrated_alerts.py +++ b/app/routes/titiler/integrated_alerts.py @@ -9,7 +9,7 @@ from ...crud.sync_db.tile_cache_assets import get_versions from ...models.enumerators.tile_caches import TileCacheType from ...models.enumerators.titiler import IntegratedAlertConfidence, RenderType -from .. import DATE_REGEX, optional_implementation_dependency, raster_xyz +from .. import DATE_REGEX, raster_xyz from .algorithms.integrated_alerts import IntegratedAlerts from .readers import AlertsReader @@ -68,7 +68,6 @@ async def gfw_integrated_alerts_raster_tile( IntegratedAlertConfidence.low, description="Show alerts with at least this confidence level", ), - implementation: str = Depends(optional_implementation_dependency), ) -> Response: """GFW Integrated Alerts raster tiles.""" diff --git a/app/routes/titiler/umd_glad_dist_alerts.py b/app/routes/titiler/umd_glad_dist_alerts.py index a740ae46..fa5d41e8 100644 --- a/app/routes/titiler/umd_glad_dist_alerts.py +++ b/app/routes/titiler/umd_glad_dist_alerts.py @@ -8,7 +8,7 @@ from titiler.core.utils import render_image from ...models.enumerators.titiler import AlertConfidence, RenderType -from .. import DATE_REGEX, optional_implementation_dependency, raster_xyz +from .. import DATE_REGEX, raster_xyz from .algorithms.dist_alerts import DISTAlerts from .readers import AlertsReader @@ -49,7 +49,6 @@ async def glad_dist_alerts_raster_tile( AlertConfidence.low, description="Show alerts that are at least of this confidence level", ), - implementation: str = Depends(optional_implementation_dependency), ) -> Response: """UMD GLAD DIST alerts raster tiles.""" From 0343d07b176f0f939d01bc765854d4a86df4471e Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Mon, 28 Oct 2024 18:24:44 +0300 Subject: [PATCH 06/14] add comment in the order of bands in tile --- app/routes/titiler/umd_glad_dist_alerts.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/routes/titiler/umd_glad_dist_alerts.py b/app/routes/titiler/umd_glad_dist_alerts.py index fa5d41e8..7e60166c 100644 --- a/app/routes/titiler/umd_glad_dist_alerts.py +++ b/app/routes/titiler/umd_glad_dist_alerts.py @@ -56,6 +56,9 @@ async def glad_dist_alerts_raster_tile( folder: str = f"s3://{DATA_LAKE_BUCKET}/{dataset}/{version}/raster/epsg-4326/cog" with AlertsReader(input=folder) as reader: tile_x, tile_y, zoom = xyz + + # NOTE: the bands in the output `image_data` array will be in the order of + # the input `bands` list image_data = reader.tile(tile_x, tile_y, zoom, bands=bands) processed_image = DISTAlerts( From 2401b992b10665728950711a27d0a02c53457099 Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Thu, 31 Oct 2024 21:03:48 +0300 Subject: [PATCH 07/14] consistent route file naming --- .../titiler/{integrated_alerts.py => gfw_integrated_alerts.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename app/routes/titiler/{integrated_alerts.py => gfw_integrated_alerts.py} (100%) diff --git a/app/routes/titiler/integrated_alerts.py b/app/routes/titiler/gfw_integrated_alerts.py similarity index 100% rename from app/routes/titiler/integrated_alerts.py rename to app/routes/titiler/gfw_integrated_alerts.py From 9463b2c79108d404744348b56a3df8995f836a26 Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Thu, 31 Oct 2024 21:04:20 +0300 Subject: [PATCH 08/14] use the version enum --- app/routes/titiler/gfw_integrated_alerts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/routes/titiler/gfw_integrated_alerts.py b/app/routes/titiler/gfw_integrated_alerts.py index 88f5d2e8..5fbb0872 100644 --- a/app/routes/titiler/gfw_integrated_alerts.py +++ b/app/routes/titiler/gfw_integrated_alerts.py @@ -51,7 +51,7 @@ class GfwIntegratdAlertsVersions(str, Enum): ) async def gfw_integrated_alerts_raster_tile( *, - version, + version: GfwIntegratdAlertsVersions, xyz: Tuple[int, int, int] = Depends(raster_xyz), start_date: Optional[str] = Query( None, From fa2aabb3de4d687b293fb1fa9c42570c46cd6130 Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Thu, 31 Oct 2024 21:04:43 +0300 Subject: [PATCH 09/14] use consistent import alias --- app/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/main.py b/app/main.py index 25539586..3a0d7c35 100644 --- a/app/main.py +++ b/app/main.py @@ -42,7 +42,7 @@ from .routes import preview from .routes.titiler import routes as titiler_routes -from .routes.titiler.integrated_alerts import router as alerts_router +from .routes.titiler.integrated_alerts import router as integrated_alerts_router from .routes.titiler.umd_glad_dist_alerts import router as dist_alerts_router gunicorn_logger = logging.getLogger("gunicorn.error") @@ -57,7 +57,7 @@ burned_areas_tiles.router, dynamic_vector_tiles.router, vector_tiles.router, - alerts_router, + integrated_alerts_router, dist_alerts_router, umd_tree_cover_loss_raster_tiles.router, umd_glad_landsat_alerts_raster_tiles.router, From bbe05f278f2e4f008a8fb0d953268790efe461b5 Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Thu, 31 Oct 2024 21:09:43 +0300 Subject: [PATCH 10/14] use color/conf and rendertype defaults --- app/routes/titiler/algorithms/alerts.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/app/routes/titiler/algorithms/alerts.py b/app/routes/titiler/algorithms/alerts.py index ac0bc8fd..f2490cdf 100644 --- a/app/routes/titiler/algorithms/alerts.py +++ b/app/routes/titiler/algorithms/alerts.py @@ -6,7 +6,7 @@ from rio_tiler.models import ImageData from titiler.core.algorithm import BaseAlgorithm -from app.models.enumerators.titiler import RenderType +from app.models.enumerators.titiler import IntegratedAlertConfidence, RenderType Colors: namedtuple = namedtuple("Colors", ["red", "green", "blue"]) AlertConfig: namedtuple = namedtuple("AlertConfig", ["confidence", "colors"]) @@ -18,14 +18,26 @@ class Alerts(BaseAlgorithm): title: str = "Deforestation Alerts" description: str = "Decode and visualize alerts" - conf_colors: OrderedDict = None + conf_colors: OrderedDict = OrderedDict( + { + IntegratedAlertConfidence.low: AlertConfig( + confidence=2, colors=Colors(237, 164, 194) + ), + IntegratedAlertConfidence.high: AlertConfig( + confidence=3, colors=Colors(220, 102, 153) + ), + IntegratedAlertConfidence.highest: AlertConfig( + confidence=4, colors=Colors(201, 42, 109) + ), + } + ) record_start_date: str = "2014-12-31" start_date: Optional[str] = None end_date: Optional[str] = None alert_confidence: Optional[str] = None - render_type: Optional[RenderType] = RenderType.true_color + render_type: RenderType = RenderType.true_color # metadata input_nbands: int = 2 From 736e98c1fd4cc527f8b6289e079708e127147efa Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Thu, 31 Oct 2024 21:18:27 +0300 Subject: [PATCH 11/14] fix import --- app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index 3a0d7c35..823713a1 100644 --- a/app/main.py +++ b/app/main.py @@ -42,7 +42,7 @@ from .routes import preview from .routes.titiler import routes as titiler_routes -from .routes.titiler.integrated_alerts import router as integrated_alerts_router +from .routes.titiler.gfw_integrated_alerts import router as integrated_alerts_router from .routes.titiler.umd_glad_dist_alerts import router as dist_alerts_router gunicorn_logger = logging.getLogger("gunicorn.error") From 406e29ab6308ef369b3f83225fea13b83e35f654 Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Fri, 1 Nov 2024 12:22:28 +0300 Subject: [PATCH 12/14] test test asset --- app/routes/titiler/gfw_integrated_alerts.py | 1 - tests/fixtures/session_start.sql | 7 +++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/app/routes/titiler/gfw_integrated_alerts.py b/app/routes/titiler/gfw_integrated_alerts.py index 5fbb0872..1765999d 100644 --- a/app/routes/titiler/gfw_integrated_alerts.py +++ b/app/routes/titiler/gfw_integrated_alerts.py @@ -33,7 +33,6 @@ class GfwIntegratdAlertsVersions(str, Enum): _versions = get_versions(dataset, TileCacheType.cog) for _version in _versions: extend_enum(GfwIntegratdAlertsVersions, _version, _version) -# TODO: add version validation # will turn this on when we're ready to replace tile cache service diff --git a/tests/fixtures/session_start.sql b/tests/fixtures/session_start.sql index aa5b2942..4652d449 100644 --- a/tests/fixtures/session_start.sql +++ b/tests/fixtures/session_start.sql @@ -233,6 +233,13 @@ INSERT INTO public.assets (dataset, version, asset_type, creation_options, metad INSERT INTO public.assets (dataset, version, asset_type, creation_options, metadata, fields, asset_id, status, asset_uri, is_managed, is_default) VALUES ('umd_glad_landsat_alerts', 'v20210101', 'COG', '{"implementation": "default", "source_asset_id": "3ac4028e-798d-4854-9b5e-6a9771ed06ed", "resampling":"mode", "blocksize": 256}', '{}', '[]', '821f0211-e439-4302-93bb-0925099df65d', 'saved', 'my_uri13', true, false); +INSERT INTO public.datasets (dataset) VALUES ('gfw_integrated_alerts'); +INSERT INTO public.versions (dataset, version, is_latest, status) + VALUES ('gfw_integrated_alerts', 'v20201012', true, 'saved'); + +INSERT INTO public.assets (dataset, version, asset_type, creation_options, metadata, fields, asset_id, status, asset_uri, is_managed, is_default) + VALUES ('gfw_integrated_alerts', 'v20201012', 'COG', '{"implementation": "default", "source_asset_id": "3ac4028e-798d-4854-9b5e-6a9771ed06dd", "resampling":"mode", "blocksize": 256}', '{}', '[]', '821f0211-e439-4302-93bb-0925099df66d', 'saved', 'my_uri14', true, false); + CREATE SCHEMA umd_modis_burned_areas; CREATE TABLE umd_modis_burned_areas.v202003 (alert__date date, gfw_area__ha numeric, geom_wm geometry); From 9d53a4341e7e303a5b74bc0031fcfc05751ec943 Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Fri, 1 Nov 2024 12:29:47 +0300 Subject: [PATCH 13/14] update latest versions in test --- tests/routes/test_helpers.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/routes/test_helpers.py b/tests/routes/test_helpers.py index 5f3c60e5..9c3ef397 100644 --- a/tests/routes/test_helpers.py +++ b/tests/routes/test_helpers.py @@ -1,7 +1,5 @@ def test_viirs_vector_tile_server(client): - """ - Basic test to check if empty data api response as expected - """ + """Basic test to check if empty data api response as expected.""" response = client.get("/_latest") api_data = response.json() @@ -15,6 +13,7 @@ def test_viirs_vector_tile_server(client): {"dataset": "umd_tree_cover_loss", "version": "v1.8"}, {"dataset": "wdpa_protected_areas", "version": "v201912"}, {"dataset": "wur_radd_alerts", "version": "v20201214"}, + {"dataset": "gfw_integrated_alerts", "version": "v20201012"}, ], "status": "success", } From a7ca04f73f7454daca12b2b72ff8eb6cfd65ad93 Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Fri, 1 Nov 2024 12:43:42 +0300 Subject: [PATCH 14/14] fix list order --- tests/routes/test_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/routes/test_helpers.py b/tests/routes/test_helpers.py index 9c3ef397..b61887f2 100644 --- a/tests/routes/test_helpers.py +++ b/tests/routes/test_helpers.py @@ -7,13 +7,13 @@ def test_viirs_vector_tile_server(client): assert api_data == { "data": [ + {"dataset": "gfw_integrated_alerts", "version": "v20201012"}, {"dataset": "nasa_viirs_fire_alerts", "version": "v202003"}, {"dataset": "umd_glad_landsat_alerts", "version": "v20210101"}, {"dataset": "umd_modis_burned_areas", "version": "v202003"}, {"dataset": "umd_tree_cover_loss", "version": "v1.8"}, {"dataset": "wdpa_protected_areas", "version": "v201912"}, {"dataset": "wur_radd_alerts", "version": "v20201214"}, - {"dataset": "gfw_integrated_alerts", "version": "v20201012"}, ], "status": "success", }