From 26a67e1266c7fc1ba190bdc5406cbfa2f6855edc Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Tue, 24 Jan 2023 15:22:44 -0500 Subject: [PATCH] Support collection-level vector tiles (#147) * Fetch all configs when iterating over collections Rather then fetch 1 render config at a time on the /collections endpoint, fetch all at once and preserve the Dict for the request duration. * Allow POST CORS requests in dev env * Vector tile support * Add default msft:region attribute to collections * Upgrade to postgres 14 and pgstac 0.6.13 Prod services operate on pg14 * Fix tests and setup The API now uses table_service.get_entities and there is an Azurite bug that prevents an empty string for "all records", so it was switched to a specific PartitionKey filter string. * Add logging for pbf requests * Deployment * Add logging and debug code Analyze relative performance of different calls in the VT endpoint chain. * Fix Exceptions * Changelog --- CHANGELOG.md | 4 + deployment/helm/deploy-values.template.yaml | 1 + .../templates/deployment.yaml | 2 + .../planetary-computer-tiler/values.yaml | 1 + docker-compose.dev.yml | 1 + nginx/etc/nginx/conf.d/default.conf | 3 + pc-tiler.dev.env | 1 + pccommon/pccommon/cli.py | 2 + pccommon/pccommon/config/__init__.py | 11 +- pccommon/pccommon/config/collections.py | 120 ++++++- pccommon/pccommon/constants.py | 1 + pccommon/pccommon/tables.py | 16 +- pccommon/setup.py | 2 +- .../tests/data-files/collection_config.json | 300 ------------------ pcstac/pcstac/client.py | 42 ++- pcstac/pcstac/tiles.py | 14 + pcstac/setup.py | 2 +- pcstac/tests/resources/test_collection.py | 10 + pctiler/pctiler/config.py | 4 + pctiler/pctiler/endpoints/vector_tiles.py | 127 ++++++++ pctiler/pctiler/errors.py | 38 +++ pctiler/pctiler/main.py | 8 +- pctiler/pctiler/reader_vector_tile.py | 69 ++++ pgstac/Dockerfile | 2 +- 24 files changed, 446 insertions(+), 335 deletions(-) create mode 100644 pctiler/pctiler/endpoints/vector_tiles.py create mode 100644 pctiler/pctiler/reader_vector_tile.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 668d61e7..71357674 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- New endpoints under `/vector` that server collection level Mapbox Vector Tiles (MVT) [#147](https://github.com/microsoft/planetary-computer-apis/pull/147) + ## [2022.4.0] ### Changed diff --git a/deployment/helm/deploy-values.template.yaml b/deployment/helm/deploy-values.template.yaml index 8257e1b8..bf50966c 100644 --- a/deployment/helm/deploy-values.template.yaml +++ b/deployment/helm/deploy-values.template.yaml @@ -76,6 +76,7 @@ tiler: # PCT sas needs to be accessed through api management pc_sdk_sas_url: https://pct-sas-westeurope-staging-apim.azure-api.net/sas/token pc_sdk_subscription_key: "{{ tf.pc_sdk_subscription_key }}" + vectortile_sa_base_url: https://pcvectortiles.blob.core.windows.net storage: account_name: "{{ tf.storage_account_name }}" diff --git a/deployment/helm/published/planetary-computer-tiler/templates/deployment.yaml b/deployment/helm/published/planetary-computer-tiler/templates/deployment.yaml index adde4b1b..f5016cb6 100644 --- a/deployment/helm/published/planetary-computer-tiler/templates/deployment.yaml +++ b/deployment/helm/published/planetary-computer-tiler/templates/deployment.yaml @@ -77,6 +77,8 @@ spec: value: "{{ .Values.tiler.stac_api_href}}" - name: PC_SDK_SAS_URL value: "{{ .Values.tiler.pc_sdk_sas_url}}" + - name: VECTORTILE_SA_BASE_URL + value: "{{ .Values.tiler.vectortile_sa_base_url}}" - name: PC_SDK_SUBSCRIPTION_KEY value: "{{ .Values.tiler.pc_sdk_subscription_key}}" - name: DEFAULT_MAX_ITEMS_PER_TILE diff --git a/deployment/helm/published/planetary-computer-tiler/values.yaml b/deployment/helm/published/planetary-computer-tiler/values.yaml index 8f86c816..07d0278d 100644 --- a/deployment/helm/published/planetary-computer-tiler/values.yaml +++ b/deployment/helm/published/planetary-computer-tiler/values.yaml @@ -36,6 +36,7 @@ tiler: stac_api_href: "" pc_sdk_sas_url: "" pc_sdk_subscription_key: "" + vectortile_sa_base_url: "" default_max_items_per_tile: 5 host: "0.0.0.0" diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index f79c61c1..e0db5c03 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -75,6 +75,7 @@ services: - FF_VRT="yes" - STAC_API_URL=http://stac:8081 - STAC_API_HREF=http://localhost:8080/stac/ + - VECTORTILE_SA_BASE_URL=http://example.com - PCAPIS_DEBUG=TRUE # titiler.pgstac diff --git a/nginx/etc/nginx/conf.d/default.conf b/nginx/etc/nginx/conf.d/default.conf index e4d5f830..bb09c24e 100644 --- a/nginx/etc/nginx/conf.d/default.conf +++ b/nginx/etc/nginx/conf.d/default.conf @@ -35,6 +35,9 @@ server { proxy_buffer_size "16k"; proxy_connect_timeout 120; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'X-PC-Request-Entity,DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; + proxy_pass http://tiler-upstream/; } diff --git a/pc-tiler.dev.env b/pc-tiler.dev.env index 89375ae9..fa5e9153 100644 --- a/pc-tiler.dev.env +++ b/pc-tiler.dev.env @@ -20,6 +20,7 @@ DB_MIN_CONN_SIZE=1 DB_MAX_CONN_SIZE=1 WEB_CONCURRENCY=1 DEFAULT_MAX_ITEMS_PER_TILE=5 +VECTORTILE_SA_BASE_URL=https://pcvectortiles.blob.core.windows.net # Azure Storage PCAPIS_COLLECTION_CONFIG__ACCOUNT_URL=http://azurite:10002/devstoreaccount1 diff --git a/pccommon/pccommon/cli.py b/pccommon/pccommon/cli.py index c2d65f62..ca9977ec 100644 --- a/pccommon/pccommon/cli.py +++ b/pccommon/pccommon/cli.py @@ -66,6 +66,7 @@ def dump(sas: str, account: str, table: str, type: str, **kwargs: Any) -> int: else: for (_, collection_id, col_config) in col_config_table.get_all(): assert collection_id + assert col_config result[collection_id] = col_config.dict() elif type == "container": @@ -80,6 +81,7 @@ def dump(sas: str, account: str, table: str, type: str, **kwargs: Any) -> int: result[f"{con_account}/{id}"] = con_config.dict() else: for (storage_account, container, con_config) in con_config_table.get_all(): + assert con_config result[f"{storage_account}/{container}"] = con_config.dict() else: print(f"Unknown type: {type}") diff --git a/pccommon/pccommon/config/__init__.py b/pccommon/pccommon/config/__init__.py index 483ac1c4..ed35b4cf 100644 --- a/pccommon/pccommon/config/__init__.py +++ b/pccommon/pccommon/config/__init__.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Dict, Optional from pccommon.config.collections import CollectionConfig, DefaultRenderConfig from pccommon.config.core import PCAPIsConfig @@ -16,3 +16,12 @@ def get_collection_config(collection_id: str) -> Optional[CollectionConfig]: def get_render_config(collection_id: str) -> Optional[DefaultRenderConfig]: return map_opt(lambda c: c.render_config, get_collection_config(collection_id)) + + +def get_all_render_configs() -> Dict[str, DefaultRenderConfig]: + return { + id: coll.render_config + for id, coll in get_apis_config() + .get_collection_config_table() + .get_all_configs() + } diff --git a/pccommon/pccommon/config/collections.py b/pccommon/pccommon/config/collections.py index 8b6783e4..2a158501 100644 --- a/pccommon/pccommon/config/collections.py +++ b/pccommon/pccommon/config/collections.py @@ -1,13 +1,51 @@ -from typing import Any, Dict, List, Optional +from enum import Enum +from typing import Any, Dict, List, Optional, Tuple import orjson from humps import camelize -from pydantic import BaseModel +from pydantic import BaseModel, Field from pccommon.tables import ModelTableService from pccommon.utils import get_param_str, orjson_dumps +class RenderOptionType(str, Enum): + def __str__(self) -> str: + return self.value + + raster_tile = "raster-tile" + vt_polygon = "vt-polygon" + vt_line = "vt-line" + + +class CamelModel(BaseModel): + class Config: + alias_generator = camelize + allow_population_by_field_name = True + json_loads = orjson.loads + json_dumps = orjson_dumps + + +class VectorTileset(CamelModel): + """ + Defines a static vector tileset for a collection. Used primarily to generate + tilejson metadata for the collection-level vector tile assets. + + + id: + The id of the vector tileset. This should match the prefix of the blob + path where the associated vector tiles are stored. Will also be used in + a URL. + """ + + id: str + name: Optional[str] = None + maxzoom: Optional[int] = Field(13, ge=0, le=24) + minzoom: Optional[int] = Field(0, ge=0, le=24) + center: Optional[List[float]] = None + bounds: Optional[List[float]] = None + + class DefaultRenderConfig(BaseModel): """ A class used to represent information convenient for accessing @@ -18,6 +56,12 @@ class DefaultRenderConfig(BaseModel): most convenient renderings for human consumption and preview. For example, if a TIF asset can be viewed as an RGB approximating normal human vision, parameters will likely encode this rendering. + + vector_tilesets: + TileJSON metadata defining static vector tilesets generated for this + collection. These are used to generate VT routes included as + collection-level assets in the STAC metadata as well as resolve paths to + the VT storage account and container to proxy actual pbf files. """ render_params: Dict[str, Any] @@ -30,6 +74,7 @@ class DefaultRenderConfig(BaseModel): mosaic_preview_coords: Optional[List[float]] = None requires_token: bool = False max_items_per_tile: Optional[int] = None + vector_tilesets: Optional[List[VectorTileset]] = None hidden: bool = False # Hide from API def get_full_render_qs(self, collection: str, item: Optional[str] = None) -> str: @@ -63,9 +108,22 @@ def get_render_params(self) -> str: if "format" in self.render_params: return default_params - # Encforce PNG rendering when otherwise unspecified + # Enforce PNG rendering when otherwise unspecified return default_params + "&format=png" + def get_vector_tileset(self, tileset_id: str) -> Optional[VectorTileset]: + """ + Get a tileset by id. + """ + tilesets = self.vector_tilesets or [] + matches = [tileset for tileset in tilesets if tileset.id == tileset_id] + + return matches[0] if matches else None + + @property + def has_vector_tiles(self) -> bool: + return bool(self.vector_tilesets) + @property def should_add_collection_links(self) -> bool: # TODO: has_mosaic flag is legacy from now-deprecated @@ -84,14 +142,6 @@ class Config: json_dumps = orjson_dumps -class CamelModel(BaseModel): - class Config: - alias_generator = camelize - allow_population_by_field_name = True - json_loads = orjson.loads - json_dumps = orjson_dumps - - class Mosaics(CamelModel): """ A single predefined CQL2-JSON query representing a named mosaic. @@ -124,7 +174,7 @@ class LegendConfig(CamelModel): `none` (note, `none` is a string literal). labels: List of string labels, ideally fewer than 3 items. Will be flex - spaced-between under the lagend image. + spaced-between under the legend image. trim_start: The number of items to trim from the start of the legend definition. Used if there are values important for rendering (e.g. nodata) that @@ -133,7 +183,7 @@ class LegendConfig(CamelModel): Same as trim_start, but for the end of the legend definition. scale_factor: A factor to multiply interval legend labels by. Useful for scaled - reasters whose colormap definitions map to unscaled values, effectively + rasters whose colormap definitions map to unscaled values, effectively showing legend labels as scaled values. """ @@ -144,6 +194,34 @@ class LegendConfig(CamelModel): scale_factor: Optional[float] +class VectorTileOptions(CamelModel): + """ + Defines a set of vector tile render options for a collection. + + Attributes + ---------- + tilejson_key: + The key in the collection-level assets which contains the tilejson URL. + source_layer: + The source layer name to render from the associated vector tiles. + fill_color: + The fill color for polygons. + stroke_color: + The stroke color for lines. + stroke_width: + The stroke width for lines. + filter: + MapBox Filter Expression to filter vector features by. + """ + + tilejson_key: str + source_layer: str + fill_color: Optional[str] + stroke_color: Optional[str] + stroke_width: Optional[int] + filter: Optional[List[Any]] + + class RenderOptionCondition(CamelModel): """ Defines a property/value condition for a render config to be enabled @@ -172,10 +250,15 @@ class RenderOptions(CamelModel): description: A longer description of the render option that can be used to explain its content. + type: + The type of render option, defaults to raster-tile. options: - A URL query-string encoded string of TiTiler rendering options. See - "Query Parameters": + A URL query-string encoded string of TiTiler rendering options. Valid + only for `raster-tile` types. See "Query Parameters": https://developmentseed.org/titiler/endpoints/cog/#description + vector_options: + Options for rendering vector tiles. Valid only for `vt-polygon` and + `vt-line` types. min_zoom: Zoom level at which to start rendering the layer. legend: @@ -187,7 +270,9 @@ class RenderOptions(CamelModel): name: str description: Optional[str] = None - options: str + type: Optional[RenderOptionType] = Field(default=RenderOptionType.raster_tile) + options: Optional[str] + vector_options: Optional[VectorTileOptions] = None min_zoom: int legend: Optional[LegendConfig] = None conditions: Optional[List[RenderOptionCondition]] = None @@ -258,3 +343,6 @@ def get_config(self, collection_id: str) -> Optional[CollectionConfig]: def set_config(self, collection_id: str, config: CollectionConfig) -> None: self.upsert("", collection_id, config) + + def get_all_configs(self) -> List[Tuple[Optional[str], CollectionConfig]]: + return [(config[1], config[2]) for config in self.get_all()] diff --git a/pccommon/pccommon/constants.py b/pccommon/pccommon/constants.py index 57c4db1c..8929cbf9 100644 --- a/pccommon/pccommon/constants.py +++ b/pccommon/pccommon/constants.py @@ -4,6 +4,7 @@ DEFAULT_CONTAINER_CONFIG_TABLE_NAME = "containerconfig" DEFAULT_IP_EXCEPTION_CONFIG_TABLE_NAME = "ipexceptionlist" +DEFAULT_COLLECTION_REGION = "westeurope" DEFAULT_TTL = 600 # 10 minutes DEFAULT_IP_EXCEPTIONS_TTL = 43200 # 12 hours diff --git a/pccommon/pccommon/tables.py b/pccommon/pccommon/tables.py index 33f789a9..20993306 100644 --- a/pccommon/pccommon/tables.py +++ b/pccommon/pccommon/tables.py @@ -61,6 +61,9 @@ def __init__( self._cache: Cache = TTLCache(maxsize=1024, ttl=ttl or DEFAULT_TTL) self._cache_lock: Lock = Lock() + def _get_cache(self) -> Cache: + return self._cache + def _ensure_table_client(self) -> None: if not self._table_client: raise TableError("Table client not initialized. Use as a context manager.") @@ -189,7 +192,11 @@ def update(self, partition_key: str, row_key: str, entity: M) -> None: } ) - @cachedmethod(cache=lambda self: self._cache, lock=lambda self: self._cache_lock) + @cachedmethod( + cache=lambda self: self._get_cache(), + lock=lambda self: self._cache_lock, + key=lambda _, partition_key, row_key: f"get_{partition_key}_{row_key}", + ) def get(self, partition_key: str, row_key: str) -> Optional[M]: with self as table_client: try: @@ -201,9 +208,14 @@ def get(self, partition_key: str, row_key: str) -> Optional[M]: except ResourceNotFoundError: return None + @cachedmethod( + cache=lambda self: self._get_cache(), + lock=lambda self: self._cache_lock, + key=lambda _: "getall", + ) def get_all(self) -> Iterable[Tuple[Optional[str], Optional[str], M]]: with self as table_client: - for entity in table_client.query_entities(""): + for entity in table_client.query_entities("PartitionKey eq ''"): partition_key, row_key = entity.get("PartitionKey"), entity.get( "RowKey" ) diff --git a/pccommon/setup.py b/pccommon/setup.py index b2a3e3da..5c106f43 100644 --- a/pccommon/setup.py +++ b/pccommon/setup.py @@ -12,7 +12,7 @@ "azure-identity==1.7.1", "azure-data-tables==12.4.0", "azure-storage-blob==12.12.0", - "pydantic==1.9.0", + "pydantic>=1.9, <2.0.0", "cachetools==5.0.0", "types-cachetools==4.2.9", "pyhumps==3.5.3", diff --git a/pccommon/tests/data-files/collection_config.json b/pccommon/tests/data-files/collection_config.json index 701a45d8..a669edcc 100644 --- a/pccommon/tests/data-files/collection_config.json +++ b/pccommon/tests/data-files/collection_config.json @@ -1221,306 +1221,6 @@ "description": "Available water storage estimate (aws) in standard layer 5 (100-150 cm depth), expressed in mm. the volume of plant available water that the soil can store in this layer based on all map unit components (weighted average). null values are presented where data are incomplete or not available.", "options": "assets=aws100_150&rescale=0.0,287.42&colormap_name=cividis", "minZoom": 4 - }, - { - "name": "Available water storage estimate, aws150_999 (mm)", - "description": "Available water storage estimate (aws) in standard layer 6 (150 cm to the reported depth of the soil profile), expressed in mm. the volume of plant available water that the soil can store in this layer based on all map unit components (weighted average). null values are presented where data are incomplete or not available.", - "options": "assets=aws150_999&rescale=0.0,600.1&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Available water storage estimate, aws20_50 (mm)", - "description": "Available water storage estimate (aws) in standard layer 3 (20-50 cm depth), expressed in mm. the volume of plant available water that the soil can store in this layer based on all map unit components (weighted average). null values are presented where data are incomplete or not available.", - "options": "assets=aws20_50&rescale=0.0,180.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Available water storage estimate, aws50_100 (mm)", - "description": "Available water storage estimate (aws) in standard layer 3 (50-100 cm depth), expressed in mm. the volume of plant available water that the soil can store in this layer based on all map unit components (weighted average). null values are presented where data are incomplete or not available.", - "options": "assets=aws50_100&rescale=0.0,300.55&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Available water storage estimate, aws5_20 (mm)", - "description": "Available water storage estimate (aws) in standard layer 2 (5-20 cm depth), expressed in mm. the volume of plant available water that the soil can store in this layer based on all map unit components (weighted average). null values are presented where data are incomplete or not available.", - "options": "assets=aws5_20&rescale=0.0,90.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "National commodity crop productivity index for corn, nccpi3corn", - "description": "National commodity crop productivity index for corn (weighted average) for major earthy components. values range from .01 (low productivity) to .99 (high productivity). earthy components are those soil series or higher level taxa components that can support crop growth (dobos et al., 2012). major components are those soil components where the majorcompflag = 'yes' (ssurgo component table). null values are presented where data are incomplete or not available.", - "options": "assets=nccpi3corn&rescale=0.0,0.991&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "National commodity crop productivity index for cotton, nccpi3cot", - "description": "National commodity crop productivity index for cotton (weighted average) for major earthy components. values range from .01 (low productivity) to .99 (high productivity). earthy components are those soil series or higher level taxa components that can support crop growth (dobos et al., 2012). major components are those soil components where the majorcompflag = 'yes' (ssurgo component table). null values are presented where data are incomplete or not available.", - "options": "assets=nccpi3cot&rescale=0.0,0.901&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "National commodity crop productivity index for small grains, nccpi3sg", - "description": "National commodity crop productivity index for small grains (weighted average) for major earthy components. values range from .01 (low productivity) to .99 (high productivity). earthy components are those soil series or higher level taxa components that can support crop growth (dobos et al., 2012). major components are those soil components where the majorcompflag = 'yes' (ssurgo component table). null values are presented where data are incomplete or not available.", - "options": "assets=nccpi3sg&rescale=0.0,0.983&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "National commodity crop productivity index for soybeans, nccpi3soy", - "description": "National commodity crop productivity index for soybeans (weighted average) for major earthy components. values range from .01 (low productivity) to .99 (high productivity). earthy components are those soil series or higher level taxa components that can support crop growth (dobos et al., 2012). major components are those soil components where the majorcompflag = 'yes' (ssurgo component table). null values are presented where data are incomplete or not available.", - "options": "assets=nccpi3soy&rescale=0.0,0.981&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "National commodity crop productivity index that has the highest value among corn and soybeans, small grains, or cotton, nccpi3all", - "description": "National commodity crop productivity index that has the highest value among corn and soybeans, small grains, or cotton (weighted average) for major earthy components. values range from .01 (low productivity) to .99 (high productivity). earthy components are those soil series or higher level taxa components that can support crop growth (dobos et al., 2012). major components are those soil components where the majorcompflag = 'yes' (ssurgo component table). null values are presented where data are incomplete or not available.", - "options": "assets=nccpi3all&rescale=0.0,0.991&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Potential wetland soil landscapes, pwsl1pomu", - "description": "Potential wetland soil landscapes (pwsl) is expressed as the percentage of the map unit that meets the pwsl criteria. the hydric rating (soil component variable \u201chydricrating\u201d) is an indicator of wet soils. for version 1 (pwsl1), those soil components that meet the following criteria are tagged as pwsl and their comppct_r values are summed for each map unit. soil components with hydricrating = 'yes' are considered pwsl. soil components with hydricrating = \u201cno\u201d are not pwsl. soil components with hydricrating = 'unranked' are tested using other attributes, and will be considered pwsl if any of the following conditions are met: drainagecl = 'poorly drained' or 'very poorly drained' or the localphase or the otherph data fields contain any of the phrases \"drained\" or \"undrained\" or \"channeled\" or \"protected\" or \"ponded\" or \"flooded\". if these criteria do not determine the pwsl for a component and hydricrating = 'unranked', then the map unit will be classified as pwsl if the map unit name contains any of the phrases \"drained\" or \"undrained\" or \"channeled\" or \"protected\" or \"ponded\" or \"flooded\". for version 1 (pwsl1), waterbodies are identified as \"999\" when map unit names match a list of terms that identify water or intermittent water or map units have a sum of the comppct_r for \"water\" that is 80% or greater. null values are presented where data are incomplete or not available.", - "options": "assets=pwsl1pomu&rescale=1,999&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Root zone depth is the depth within the soil profile that commodity crop, rootznemc", - "description": "Root zone depth is the depth within the soil profile that commodity crop (cc) roots can effectively extract water and nutrients for growth. root zone depth influences soil productivity significantly. soil component horizon criteria for root-limiting depth include: presence of hard bedrock, soft bedrock, a fragipan, a duripan, sulfuric material, a dense layer, a layer having a ph of less than 3.5, or a layer having an electrical conductivity of more than 12 within the component soil profile. if no root-restricting zone is identified, a depth of 150 cm is used to approximate the root zone depth (dobos et al., 2012). root zone depth is computed for all map unit major earthy components (weighted average). earthy components are those soil series or higher level taxa components that can support crop growth (dobos et al., 2012). major components are those soil components where the majorcompflag = 'yes' (ssurgo component table). null values are presented where data are incomplete or not available.", - "options": "assets=rootznemc&rescale=0,150&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Root zone, rootznaws (mm)", - "description": "Root zone (commodity crop) available water storage estimate (rzaws) , expressed in mm, is the volume of plant available water that the soil can store within the root zone based on all map unit earthy major components (weighted average). earthy components are those soil series or higher level taxa components that can support crop growth (dobos et al., 2012). major components are those soil components where the majorcompflag = 'yes' (ssurgo component table). null values are presented where data are incomplete or not available.", - "options": "assets=rootznaws&rescale=0,900&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Soil organic carbon stock estimate, soc0_100 (g/m2)", - "description": "Soil organic carbon stock estimate (soc) in standard zone 4 (0-100 cm depth). the concentration of organic carbon present in the soil expressed in grams c per square meter to a depth of 100 cm. null values are presented where data are incomplete or not available.", - "options": "assets=soc0_100&rescale=0.0,802441.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Soil organic carbon stock estimate, soc0_150 (g/m2)", - "description": "Soil organic carbon stock estimate (soc) in standard zone 5 (0-150 cm depth). the concentration of organic carbon present in the soil expressed in grams c per square meter to a depth of 150 cm. null values are presented where data are incomplete or not available.", - "options": "assets=soc0_150&rescale=0.0,1178783.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Soil organic carbon stock estimate, soc0_20 (g/m2)", - "description": "Soil organic carbon stock estimate (soc) in standard zone 2 (0-20 cm depth). the concentration of organic carbon present in the soil expressed in grams c per square meter to a depth of 20 cm. null values are presented where data are incomplete or not available.", - "options": "assets=soc0_20&rescale=0.0,160518.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Soil organic carbon stock estimate, soc0_30 (g/m2)", - "description": "Soil organic carbon stock estimate (soc) in standard zone 3 (0-30 cm depth). the concentration of organic carbon present in the soil expressed in grams c per square meter to a depth of 30 cm. null values are presented where data are incomplete or not available.", - "options": "assets=soc0_30&rescale=0.0,240770.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Soil organic carbon stock estimate, soc0_5 (g/m2)", - "description": "Soil organic carbon stock estimate (soc) in standard layer 1 or standard zone 1 (0-5 cm depth). the concentration of organic carbon present in the soil expressed in grams c per square meter to a depth of 5 cm. null values are presented where data are incomplete or not available.", - "options": "assets=soc0_5&rescale=0.0,40130.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Soil organic carbon stock estimate, soc0_999 (g/m2)", - "description": "Soil organic carbon stock estimate (soc) in total soil profile (0 cm to the reported depth of the soil profile). the concentration of organic carbon present in the soil expressed in grams c per square meter for the total reported soil profile depth. null values are presented where data are incomplete or not available.", - "options": "assets=soc0_999&rescale=0.0,1182344.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Soil organic carbon stock estimate, soc100_150 (g/m2)", - "description": "Soil organic carbon stock estimate (soc) in standard layer 5 (100-150 cm depth). the concentration of organic carbon present in the soil expressed in grams c per square meter for the 100-150 cm layer. null values are presented where data are incomplete or not available.", - "options": "assets=soc100_150&rescale=0.0,376342.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Soil organic carbon stock estimate, soc150_999 (g/m2)", - "description": "Soil organic carbon stock estimate (soc) in standard layer 6 (150 cm to the reported depth of the soil profile). the concentration of organic carbon present in the soil expressed in grams c per square meter for the 150 cm and greater depth layer. null values are presented where data are incomplete or not available.", - "options": "assets=soc150_999&rescale=0.0,126247.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Soil organic carbon stock estimate, soc20_50 (g/m2)", - "description": "Soil organic carbon stock estimate (soc) in standard layer 3 (20-50 cm depth). the concentration of organic carbon present in the soil expressed in grams c per square meter for the 20-50 cm layer. null values are presented where data are incomplete or not available.", - "options": "assets=soc20_50&rescale=0.0,240743.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Soil organic carbon stock estimate, soc50_100 (g/m2)", - "description": "Soil organic carbon stock estimate (soc) in standard layer 4 (50-100 cm depth). the concentration of organic carbon present in the soil expressed in grams c per square meter for the 50-100 cm layer. null values are presented where data are incomplete or not available.", - "options": "assets=soc50_100&rescale=0.0,401179.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Soil organic carbon stock estimate, soc5_20 (g/m2)", - "description": "Soil organic carbon stock estimate (soc) in standard layer 2 (5-20 cm depth). the concentration of organic carbon present in the soil expressed in grams c per square meter for the 5-20 cm layer. null values are presented where data are incomplete or not available.", - "options": "assets=soc5_20&rescale=0.0,120389.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "The national commodity crop productivity index map unit percent earthy is the map unit summed comppct_r for major earthy comps. earthy comps are those soil series or higher level taxa comps that can support crop growth, pctearthmc", - "description": "The national commodity crop productivity index map unit percent earthy is the map unit summed comppct_r for major earthy components. earthy components are those soil series or higher level taxa components that can support crop growth (dobos et al., 2012). major components are those soil components where the majorcompflag = 'yes' (ssurgo component table). useful metadata information. null values are presented where data are incomplete or not available.", - "options": "assets=pctearthmc&rescale=0,100&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "The sum of the comppct_r, musumcpct", - "description": "The sum of the comppct_r (ssurgo component table) values for all listed components in the map unit. useful metadata information. null values are presented where data are incomplete or not available.", - "options": "assets=musumcpct&rescale=0,110&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "The sum of the comppct_r, musumcpcta", - "description": "The sum of the comppct_r (ssurgo component table) values used in the available water storage calculation for the map unit. useful metadata information. null values are presented where data are incomplete or not available.", - "options": "assets=musumcpcta&rescale=1,110&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "The sum of the comppct_r, musumcpcts", - "description": "The sum of the comppct_r (ssurgo component table) values used in the soil organic carbon calculation for the map unit. useful metadata information. null values are presented where data are incomplete or not available.", - "options": "assets=musumcpcts&rescale=1,110&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Thickness of soil comps used in std layer 1 or std zone 1, tk0_5a (cm)", - "description": "Thickness of soil components used in standard layer 1 or standard zone 1 (0-5 cm) expressed in cm (weighted average) for the available water storage calculation. null values are presented where data are incomplete or not available.", - "options": "assets=tk0_5a&rescale=0.05,5.5&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Thickness of soil comps used in std layer 1 or std zone 1, tk0_5s (cm)", - "description": "Thickness of soil components used in standard layer 1 or standard zone 1 (0-5 cm) expressed in cm (weighted average) for the soil organic carbon calculation. null values are presented where data are incomplete or not available.", - "options": "assets=tk0_5s&rescale=0.0,6.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Thickness of soil comps used in std layer 2, tk5_20a (cm)", - "description": "Thickness of soil components used in standard layer 2 (5-20 cm) expressed in cm (weighted average) for the available water storage calculation. null values are presented where data are incomplete or not available.", - "options": "assets=tk5_20a&rescale=0.03,16.5&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Thickness of soil comps used in std layer 2, tk5_20s (cm)", - "description": "Thickness of soil components used in standard layer 2 (5-20 cm) expressed in cm (weighted average) for the soil organic carbon calculation. null values are presented where data are incomplete or not available.", - "options": "assets=tk5_20s&rescale=0.0,17.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Thickness of soil comps used in std layer 3, tk20_50a (cm)", - "description": "Thickness of soil components used in standard layer 3 (20-50 cm) expressed in cm (weighted average) for the available water storage calculation. null values are presented where data are incomplete or not available.", - "options": "assets=tk20_50a&rescale=0.06,37.65&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Thickness of soil comps used in std layer 3, tk20_50s (cm)", - "description": "Thickness of soil components used in standard layer 3 (20-50 cm) expressed in cm (weighted average) for the soil organic carbon calculation. null values are presented where data are incomplete or not available.", - "options": "assets=tk20_50s&rescale=0.0,38.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Thickness of soil comps used in std layer 4, tk50_100a (cm)", - "description": "Thickness of soil components used in standard layer 4 (50-100 cm) expressed in cm (weighted average) for the available water storage calculation. null values are presented where data are incomplete or not available.", - "options": "assets=tk50_100a&rescale=0.03,55.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Thickness of soil comps used in std layer 4, tk50_100s (cm)", - "description": "Thickness of soil components used in standard layer 4 (50-100 cm) expressed in cm (weighted average) for the soil organic carbon calculation. null values are presented where data are incomplete or not available.", - "options": "assets=tk50_100s&rescale=0.0,55.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Thickness of soil comps used in std layer 5, tk100_150a (cm)", - "description": "Thickness of soil components used in standard layer 5 (100-150 cm) expressed in cm (weighted average) for the available water storage calculation. null values are presented where data are incomplete or not available.", - "options": "assets=tk100_150a&rescale=0.04,55.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Thickness of soil comps used in std layer 5, tk100_150s (cm)", - "description": "Thickness of soil components used in standard layer 5 (100-150 cm) expressed in cm (weighted average) for the soil organic carbon calculation. null values are presented where data are incomplete or not available.", - "options": "assets=tk100_150s&rescale=0.0,55.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Thickness of soil comps used in std layer 6, tk150_999a (cm)", - "description": "Thickness of soil components used in standard layer 6 (150 cm to the reported depth of the soil profile) expressed in cm (weighted average) for the available water storage calculation. null values are presented where data are incomplete or not available.", - "options": "assets=tk150_999a&rescale=0.0,307.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Thickness of soil comps used in std layer 6, tk150_999s (cm)", - "description": "Thickness of soil components used in standard layer 6 (150 cm to the reported depth of the soil profile) expressed in cm (weighted average) for the soil organic carbon calculation. null values are presented where data are incomplete or not available.", - "options": "assets=tk150_999s&rescale=0.0,307.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Thickness of soil comps used in std zone 2, tk0_20a (cm)", - "description": "Thickness of soil components used in standard zone 2 (0-20 cm) expressed in cm (weighted average) for the available water storage calculation. null values are presented where data are incomplete or not available.", - "options": "assets=tk0_20a&rescale=0.08,22.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Thickness of soil comps used in std zone 2, tk0_20s (cm)", - "description": "Thickness of soil components used in standard zone 2 (0-20 cm) expressed in cm (weighted average) for the soil organic carbon calculation. null values are presented where data are incomplete or not available.", - "options": "assets=tk0_20s&rescale=0.0,22.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Thickness of soil comps used in std zone 3, tk0_30a (cm)", - "description": "Thickness of soil components used in standard zone 3 (0-30 cm) expressed in cm (weighted average) for the available water storage calculation. null values are presented where data are incomplete or not available.", - "options": "assets=tk0_30a&rescale=0.08,33.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Thickness of soil comps used in std zone 3, tk0_30s (cm)", - "description": "Thickness of soil components used in standard zone 3 (0-30 cm) expressed in cm (weighted average) for the soil organic carbon calculation. null values are presented where data are incomplete or not available.", - "options": "assets=tk0_30s&rescale=0.0,33.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Thickness of soil comps used in std zone 4, tk0_100a (cm)", - "description": "Thickness of soil components used in standard zone 4 (0-100 cm) expressed in cm (weighted average) for the available water storage calculation. null values are presented where data are incomplete or not available.", - "options": "assets=tk0_100a&rescale=0.08,110.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Thickness of soil comps used in std zone 4, tk0_100s (cm)", - "description": "Thickness of soil components used in standard zone 4 (0-100 cm) expressed in cm (weighted average) for the soil organic carbon calculation. null values are presented where data are incomplete or not available.", - "options": "assets=tk0_100s&rescale=0.0,110.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Thickness of soil comps used in std zone 5, tk0_150a (cm)", - "description": "Thickness of soil components used in standard zone 5 (0-150 cm) expressed in cm (weighted average) for the available water storage calculation. null values are presented where data are incomplete or not available.", - "options": "assets=tk0_150a&rescale=0.08,165.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Thickness of soil comps used in std zone 5, tk0_150s (cm)", - "description": "Thickness of soil components used in standard zone 5 (0-150 cm) expressed in cm (weighted average) for the soil organic carbon calculation. null values are presented where data are incomplete or not available.", - "options": "assets=tk0_150s&rescale=0.0,165.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Thickness of soil comps used in total soil profile, tk0_999a (cm)", - "description": "Thickness of soil components used in total soil profile (0 cm to the reported depth of the soil profile) expressed in cm (weighted average) for the available water storage calculation. null values are presented where data are incomplete or not available.", - "options": "assets=tk0_999a&rescale=0.08,457.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Thickness of soil comps used in total soil profile, tk0_999s (cm)", - "description": "Thickness of soil components used in total soil profile (0 cm to the reported depth of the soil profile) expressed in cm (weighted average) for the soil organic carbon calculation. null values are presented where data are incomplete or not available.", - "options": "assets=tk0_999s&rescale=0.0,457.0&colormap_name=cividis", - "minZoom": 4 - }, - { - "name": "Zone for commodity crops that is \u2264 6 inches, droughty", - "description": "Zone for commodity crops that is less than or equal to 6 inches (152 mm) expressed as \"1\" for a drought vulnerable soil landscape map unit or \"0\" for a non-droughty soil landscape map unit or null for miscellaneous areas (includes water bodies) or where data were not available. it is computed as a weighted average for major earthy components. earthy components are those soil series or higher level taxa components that can support crop growth (dobos et al., 2012). major components are those soil components where the majorcompflag = 'yes'", - "options": "assets=droughty&rescale=0,1&colormap_name=cividis", - "minZoom": 4 } ], "defaultLocation": { diff --git a/pcstac/pcstac/client.py b/pcstac/pcstac/client.py index a7ca2bc0..1f1c965a 100644 --- a/pcstac/pcstac/client.py +++ b/pcstac/pcstac/client.py @@ -15,7 +15,9 @@ LandingPage, ) -from pccommon.config import get_render_config +from pccommon.config import get_all_render_configs, get_render_config +from pccommon.config.collections import DefaultRenderConfig +from pccommon.constants import DEFAULT_COLLECTION_REGION from pccommon.logging import get_custom_dimensions from pccommon.redis import back_pressure, cached_result, rate_limit from pcstac.config import API_DESCRIPTION, API_LANDING_PAGE_ID, API_TITLE, get_settings @@ -54,16 +56,26 @@ def conformance_classes(self) -> List[str]: return sorted(list(set(base_conformance_classes))) - def inject_collection_links( - self, collection: Collection, request: Request + def inject_collection_extras( + self, + collection: Collection, + request: Request, + render_config: Optional[DefaultRenderConfig] = None, ) -> Collection: - """Add extra/non-mandatory links to a Collection""" + """Add extra/non-mandatory links, assets, and properties to a Collection""" + collection_id = collection.get("id", "") - render_config = get_render_config(collection_id) - if render_config and render_config.should_add_collection_links: - TileInfo(collection_id, render_config, request).inject_collection( - collection - ) + config = render_config or get_render_config(collection_id) + if config: + tile_info = TileInfo(collection_id, config, request) + if config.should_add_collection_links: + tile_info.inject_collection(collection) + + if config.has_vector_tiles: + tile_info.inject_collection_vectortile_assets(collection) + + if "msft:region" not in collection: + collection["msft:region"] = DEFAULT_COLLECTION_REGION collection.get("links", []).append( { @@ -110,15 +122,21 @@ async def all_collections(self, **kwargs: Any) -> Collections: async def _fetch() -> Collections: collections = await _super.all_collections(**kwargs) + render_configs = get_all_render_configs() modified_collections = [] for col in collections.get("collections", []): collection_id = col.get("id", "") - render_config = get_render_config(collection_id) + render_config = render_configs.get( + collection_id, + DefaultRenderConfig( + create_links=False, minzoom=0, render_params={} + ), + ) if render_config and render_config.hidden: pass else: modified_collections.append( - self.inject_collection_links(col, _request) + self.inject_collection_extras(col, _request, render_config) ) collections["collections"] = modified_collections return collections @@ -158,7 +176,7 @@ async def _fetch() -> Collection: result = await _super.get_collection(collection_id, **kwargs) except NotFoundError: raise NotFoundError(f"No collection with id '{collection_id}' found!") - return self.inject_collection_links(result, _request) + return self.inject_collection_extras(result, _request, render_config) cache_key = f"{CACHE_KEY_COLLECTION}:{collection_id}" return await cached_result(_fetch, cache_key, kwargs["request"]) diff --git a/pcstac/pcstac/tiles.py b/pcstac/pcstac/tiles.py index fd9560d1..683d4b61 100644 --- a/pcstac/pcstac/tiles.py +++ b/pcstac/pcstac/tiles.py @@ -45,6 +45,20 @@ def inject_collection(self, collection: Collection) -> None: assets["tilejson"] = self._get_collection_tilejson_asset() collection["assets"] = assets # assets not a required property. + def inject_collection_vectortile_assets(self, collection: Collection) -> None: + """Inject vector tile assets to a collection""" + assets = collection.get("assets", {}) + + for tileset in self.render_config.vector_tilesets or []: + tile_path = f"vector/collections/{self.collection_id}/tilesets/{tileset.id}/tilejson.json" # noqa + assets[tileset.id] = { + "title": tileset.name, + "href": urljoin(self.tiler_href, tile_path), + "type": pystac.MediaType.JSON, + "roles": ["tiles"], + } + collection["assets"] = assets + def inject_item(self, item: Item) -> None: """Inject rendering links to an item""" item_id = item.get("id", "") diff --git a/pcstac/setup.py b/pcstac/setup.py index 4add1cf9..6626faa0 100644 --- a/pcstac/setup.py +++ b/pcstac/setup.py @@ -10,7 +10,7 @@ "stac-fastapi.types @ git+https://github.com/stac-utils/stac-fastapi/@25879afe94296eb82b94b523bfa2871b686e035a#egg=stac-fastapi.types&subdirectory=stac_fastapi/types", "pccommon", # Required due to some imports related to pypgstac CLI usage in startup script - "pypgstac[psycopg]==0.6.9", + "pypgstac[psycopg]==0.6.13", "pystac==1.*", ] diff --git a/pcstac/tests/resources/test_collection.py b/pcstac/tests/resources/test_collection.py index fdfee203..e1cefbc3 100644 --- a/pcstac/tests/resources/test_collection.py +++ b/pcstac/tests/resources/test_collection.py @@ -23,3 +23,13 @@ async def test_collection_not_found(app_client): resp = await app_client.get("/collections/does-not-exist") print(json.dumps(resp.json(), indent=2)) assert resp.status_code == 404 + + +@pytest.mark.asyncio +async def test_all_collections_have_msft_regions(app_client): + """Test that all collections have msft:region""" + resp = await app_client.get("/collections") + assert resp.status_code == 200 + collections = resp.json()["collections"] + for collection in collections: + assert "msft:region" in collection diff --git a/pctiler/pctiler/config.py b/pctiler/pctiler/config.py index ce22ec02..d278758e 100644 --- a/pctiler/pctiler/config.py +++ b/pctiler/pctiler/config.py @@ -13,6 +13,7 @@ DEFAULT_MAX_ITEMS_PER_TILE_ENV_VAR = "DEFAULT_MAX_ITEMS_PER_TILE" REQUEST_TIMEOUT_ENV_VAR = "REQUEST_TIMEOUT" +VECTORTILE_SA_BASE_URL_ENV_VAR = "VECTORTILE_SA_BASE_URL" @dataclass @@ -36,6 +37,9 @@ class Settings(BaseSettings): item_endpoint_prefix: str = "/item" mosaic_endpoint_prefix: str = "/mosaic" legend_endpoint_prefix: str = "/legend" + vector_tile_endpoint_prefix: str = "/vector" + vector_tile_sa_base_url: str = Field(env=VECTORTILE_SA_BASE_URL_ENV_VAR) + debug: bool = os.getenv("TILER_DEBUG", "False").lower() == "true" api_version: str = "1.0" default_max_items_per_tile: int = Field( diff --git a/pctiler/pctiler/endpoints/vector_tiles.py b/pctiler/pctiler/endpoints/vector_tiles.py new file mode 100644 index 00000000..0a82db26 --- /dev/null +++ b/pctiler/pctiler/endpoints/vector_tiles.py @@ -0,0 +1,127 @@ +import logging + +from fastapi import APIRouter, HTTPException, Path +from fastapi.responses import Response +from starlette.requests import Request +from titiler.core.models.mapbox import TileJSON + +from pccommon.config import get_render_config +from pccommon.config.collections import VectorTileset +from pctiler.config import get_settings +from pctiler.errors import VectorTileError, VectorTileNotFoundError +from pctiler.reader_vector_tile import VectorTileReader + +logger = logging.getLogger(__name__) +settings = get_settings() + +vector_tile_router = APIRouter() + + +@vector_tile_router.get( + "/collections/{collection_id}/tilesets/{tileset_id}/tilejson.json", + response_model=TileJSON, + response_model_exclude_none=True, +) +async def get_tilejson( + request: Request, + collection_id: str = Path(description="STAC Collection ID"), + tileset_id: str = Path( + description="Registered tileset ID, see Collection metadata for valid values." + ), +) -> TileJSON: + """Get the tilejson for a given tileset.""" + + tileset = _get_tileset_config(collection_id, tileset_id) + + tile_url = str( + request.url_for( + "get_tile", + collection_id=collection_id, + tileset_id=tileset_id, + z="{z}", + x="{x}", + y="{y}", + ) + ) + + name = ( + tileset.name or f"Planetary Computer: {collection_id} {tileset_id} vector tiles" + ) + + tilejson = { + "tiles": [tile_url], + "name": name, + "maxzoom": tileset.maxzoom, + "minzoom": tileset.minzoom, + } + + if tileset.bounds: + tilejson["bounds"] = tileset.bounds + if tileset.center: + tilejson["center"] = tileset.center + + return tilejson + + +@vector_tile_router.get( + "/collections/{collection_id}/tilesets/{tileset_id}/tiles/{z}/{x}/{y}", + response_class=Response, +) +async def get_tile( + request: Request, + collection_id: str = Path(description="STAC Collection ID"), + tileset_id: str = Path( + description="Registered tileset ID, see Collection metadata for valid values." + ), + z: int = Path(description="Zoom"), + x: int = Path(description="Tile column"), + y: int = Path(description="Tile row"), +) -> Response: + """Get a vector tile for a given tileset.""" + tileset = _get_tileset_config(collection_id, tileset_id) + + reader = VectorTileReader(collection_id, tileset, request) + + try: + pbf = reader.get_tile(z, x, y) + except Exception as e: + logger.exception(e) + raise VectorTileError( + collection=collection_id, + tileset_id=tileset_id, + z=z, + x=x, + y=y, + ) + + if not pbf: + raise VectorTileNotFoundError( + collection=collection_id, tileset_id=tileset_id, z=z, x=x, y=y + ) + + return Response( + content=pbf, + media_type="application/x-protobuf", + headers={"content-encoding": "gzip"}, + ) + + +def _get_tileset_config(collection_id: str, tileset_id: str) -> VectorTileset: + """Get the render configuration for a given collection.""" + config = get_render_config(collection_id) + + if not config: + raise HTTPException( + status_code=404, + detail=f"Collection {collection_id} has no registered tilesets", + ) + + tileset = config.get_vector_tileset(tileset_id) + + if not tileset: + raise HTTPException( + status_code=404, + detail=f"Tileset {tileset_id} not found for collection {collection_id}", + ) + + return tileset diff --git a/pctiler/pctiler/errors.py b/pctiler/pctiler/errors.py index 4f774593..8ec409bd 100644 --- a/pctiler/pctiler/errors.py +++ b/pctiler/pctiler/errors.py @@ -7,3 +7,41 @@ class TilerError(Exception, ABC): @abstractmethod def to_http(self) -> HTTPException: pass + + +class VectorTileError(HTTPException): + def __init__( + self, + collection: str, + tileset_id: str, + z: int, + x: int, + y: int, + ) -> None: + + super().__init__( + status_code=500, + detail=( + f"Error loading tile {z}/{x}/{y} for tileset: '{tileset_id}' in " + f"collection: '{collection}'" + ), + ) + + +class VectorTileNotFoundError(HTTPException): + def __init__( + self, + collection: str, + tileset_id: str, + z: int, + x: int, + y: int, + ) -> None: + + super().__init__( + status_code=404, + detail=( + f"Tile {z}/{x}/{y} not found for tileset " + f"{tileset_id} in collection {collection}" + ), + ) diff --git a/pctiler/pctiler/main.py b/pctiler/pctiler/main.py index bb9cbdf7..f120389b 100755 --- a/pctiler/pctiler/main.py +++ b/pctiler/pctiler/main.py @@ -26,7 +26,7 @@ ) from pccommon.openapi import fixup_schema from pctiler.config import get_settings -from pctiler.endpoints import health, item, legend, pg_mosaic +from pctiler.endpoints import health, item, legend, pg_mosaic, vector_tiles # Initialize logging init_logging(ServiceName.TILER) @@ -63,6 +63,12 @@ tags=["Legend endpoints"], ) +app.include_router( + vector_tiles.vector_tile_router, + prefix=settings.vector_tile_endpoint_prefix, + tags=["Collection vector tile endpoints"], +) + app.include_router(health.health_router, tags=["Liveliness/Readiness"]) app.add_middleware(RequestTracingMiddleware, service_name=ServiceName.TILER) diff --git a/pctiler/pctiler/reader_vector_tile.py b/pctiler/pctiler/reader_vector_tile.py new file mode 100644 index 00000000..26dd2e2d --- /dev/null +++ b/pctiler/pctiler/reader_vector_tile.py @@ -0,0 +1,69 @@ +import logging +import time +from typing import Optional + +import planetary_computer as pc +import requests +from starlette.requests import Request + +from pccommon.config.collections import VectorTileset +from pccommon.logging import get_custom_dimensions +from pctiler.config import get_settings + +settings = get_settings() + +logger = logging.getLogger(__name__) + + +class VectorTileReader: + """ + Load a vector tile from a storage account container and return + """ + + def __init__(self, collection: str, tileset: VectorTileset, request: Request): + self.request = request + self.tileset = tileset + self.collection = collection + + def get_tile(self, z: int, x: int, y: int) -> Optional[bytes]: + """ + Get a vector tile from a storage account container + """ + blob_url = self._blob_url_for_tile(z, x, y) + + ts = time.perf_counter() + response = requests.get(blob_url, stream=True) + logger.info( + "Perf: PBF upsteam load time", + extra=get_custom_dimensions( + {"duration": f"{time.perf_counter() - ts:0.4f}"}, self.request + ), + ) + + if response.status_code == 404: + return None + + if response.status_code != 200: + raise Exception(f"Error loading tile {blob_url}") + + ts = time.perf_counter() + b = response.raw.read() + logger.info( + "Perf: PBF raw bytes read", + extra=get_custom_dimensions( + {"duration": f"{time.perf_counter() - ts:0.4f}"}, self.request + ), + ) + return b + + def _blob_url_for_tile(self, z: int, x: int, y: int) -> str: + """ + Get the URL to the storage account container and blob + """ + tile_url = ( + f"{settings.vector_tile_sa_base_url}" + f"/{self.collection}" + f"/{self.tileset.id}/{z}/{x}/{y}.pbf" + ) + + return pc.sign_url(tile_url) diff --git a/pgstac/Dockerfile b/pgstac/Dockerfile index b738da05..b518d44a 100644 --- a/pgstac/Dockerfile +++ b/pgstac/Dockerfile @@ -1,4 +1,4 @@ -FROM postgres:13 as pg +FROM postgres:14 as pg ENV POSTGIS_MAJOR 3