Support collection-level vector tiles (#147)

* Fetch all configs when iterating over collections

Rather then fetch 1 render config at a time on the /collections
endpoint, fetch all at once and preserve the Dict for the request
duration.

* Allow POST CORS requests in dev env

* Vector tile support

* Add default msft:region attribute to collections

* Upgrade to postgres 14 and pgstac 0.6.13

Prod services operate on pg14

* Fix tests and setup

The API now uses table_service.get_entities and there is an Azurite bug
that prevents an empty string for "all records", so it was switched to a
specific PartitionKey filter string.

* Add logging for pbf requests

* Deployment

* Add logging and debug code

Analyze relative performance of different calls in the VT endpoint
chain.

* Fix Exceptions

* Changelog

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+
+- New endpoints under `/vector` that server collection level Mapbox Vector Tiles (MVT) [#147](https://github.com/microsoft/planetary-computer-apis/pull/147)
+
 ## [2022.4.0]
 
 ### Changed

deployment/helm/deploy-values.template.yaml

@@ -76,6 +76,7 @@ tiler:
   # PCT sas needs to be accessed through api management
   pc_sdk_sas_url: https://pct-sas-westeurope-staging-apim.azure-api.net/sas/token
   pc_sdk_subscription_key: "{{ tf.pc_sdk_subscription_key }}"
+  vectortile_sa_base_url: https://pcvectortiles.blob.core.windows.net
 
 storage:
   account_name: "{{ tf.storage_account_name }}"

deployment/helm/published/planetary-computer-tiler/templates/deployment.yaml

@@ -77,6 +77,8 @@ spec:
               value: "{{ .Values.tiler.stac_api_href}}"
             - name: PC_SDK_SAS_URL
               value: "{{ .Values.tiler.pc_sdk_sas_url}}"
+            - name: VECTORTILE_SA_BASE_URL
+              value: "{{ .Values.tiler.vectortile_sa_base_url}}"
             - name: PC_SDK_SUBSCRIPTION_KEY
               value: "{{ .Values.tiler.pc_sdk_subscription_key}}"
             - name: DEFAULT_MAX_ITEMS_PER_TILE

deployment/helm/published/planetary-computer-tiler/values.yaml

@@ -36,6 +36,7 @@ tiler:
   stac_api_href: ""
   pc_sdk_sas_url: ""
   pc_sdk_subscription_key: ""
+  vectortile_sa_base_url: ""
 
   default_max_items_per_tile: 5
   host: "0.0.0.0"

docker-compose.dev.yml

@@ -75,6 +75,7 @@ services:
       - FF_VRT="yes"
       - STAC_API_URL=http://stac:8081
       - STAC_API_HREF=http://localhost:8080/stac/
+      - VECTORTILE_SA_BASE_URL=http://example.com
       - PCAPIS_DEBUG=TRUE
 
       # titiler.pgstac

nginx/etc/nginx/conf.d/default.conf

@@ -35,6 +35,9 @@ server {
         proxy_buffer_size "16k";
         proxy_connect_timeout  120;
 
+        add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+        add_header 'Access-Control-Allow-Headers' 'X-PC-Request-Entity,DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
+
         proxy_pass http://tiler-upstream/;
     }
 

pc-tiler.dev.env

@@ -20,6 +20,7 @@ DB_MIN_CONN_SIZE=1
 DB_MAX_CONN_SIZE=1
 WEB_CONCURRENCY=1
 DEFAULT_MAX_ITEMS_PER_TILE=5
+VECTORTILE_SA_BASE_URL=https://pcvectortiles.blob.core.windows.net
 
 # Azure Storage
 PCAPIS_COLLECTION_CONFIG__ACCOUNT_URL=http://azurite:10002/devstoreaccount1

pccommon/pccommon/cli.py

@@ -66,6 +66,7 @@ def dump(sas: str, account: str, table: str, type: str, **kwargs: Any) -> int:
         else:
             for (_, collection_id, col_config) in col_config_table.get_all():
                 assert collection_id
+                assert col_config
                 result[collection_id] = col_config.dict()
 
     elif type == "container":
@@ -80,6 +81,7 @@ def dump(sas: str, account: str, table: str, type: str, **kwargs: Any) -> int:
             result[f"{con_account}/{id}"] = con_config.dict()
         else:
             for (storage_account, container, con_config) in con_config_table.get_all():
+                assert con_config
                 result[f"{storage_account}/{container}"] = con_config.dict()
     else:
         print(f"Unknown type: {type}")

pccommon/pccommon/config/__init__.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import Dict, Optional
 
 from pccommon.config.collections import CollectionConfig, DefaultRenderConfig
 from pccommon.config.core import PCAPIsConfig
@@ -16,3 +16,12 @@ def get_collection_config(collection_id: str) -> Optional[CollectionConfig]:
 
 def get_render_config(collection_id: str) -> Optional[DefaultRenderConfig]:
     return map_opt(lambda c: c.render_config, get_collection_config(collection_id))
+
+
+def get_all_render_configs() -> Dict[str, DefaultRenderConfig]:
+    return {
+        id: coll.render_config
+        for id, coll in get_apis_config()
+        .get_collection_config_table()
+        .get_all_configs()
+    }

pccommon/pccommon/config/collections.py

@@ -1,13 +1,51 @@
-from typing import Any, Dict, List, Optional
+from enum import Enum
+from typing import Any, Dict, List, Optional, Tuple
 
 import orjson
 from humps import camelize
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 from pccommon.tables import ModelTableService
 from pccommon.utils import get_param_str, orjson_dumps
 
 
+class RenderOptionType(str, Enum):
+    def __str__(self) -> str:
+        return self.value
+
+    raster_tile = "raster-tile"
+    vt_polygon = "vt-polygon"
+    vt_line = "vt-line"
+
+
+class CamelModel(BaseModel):
+    class Config:
+        alias_generator = camelize
+        allow_population_by_field_name = True
+        json_loads = orjson.loads
+        json_dumps = orjson_dumps
+
+
+class VectorTileset(CamelModel):
+    """
+    Defines a static vector tileset for a collection. Used primarily to generate
+    tilejson metadata for the collection-level vector tile assets.
+
+
+    id:
+        The id of the vector tileset. This should match the prefix of the blob
+        path where the associated vector tiles are stored. Will also be used in
+        a URL.
+    """
+
+    id: str
+    name: Optional[str] = None
+    maxzoom: Optional[int] = Field(13, ge=0, le=24)
+    minzoom: Optional[int] = Field(0, ge=0, le=24)
+    center: Optional[List[float]] = None
+    bounds: Optional[List[float]] = None
+
+
 class DefaultRenderConfig(BaseModel):
     """
     A class used to represent information convenient for accessing
@@ -18,6 +56,12 @@ class DefaultRenderConfig(BaseModel):
     most convenient renderings for human consumption and preview.
     For example, if a TIF asset can be viewed as an RGB approximating
     normal human vision, parameters will likely encode this rendering.
+
+    vector_tilesets:
+        TileJSON metadata defining static vector tilesets generated for this
+        collection. These are used to generate VT routes included as
+        collection-level assets in the STAC metadata as well as resolve paths to
+        the VT storage account and container to proxy actual pbf files.
     """
 
     render_params: Dict[str, Any]
@@ -30,6 +74,7 @@ class DefaultRenderConfig(BaseModel):
     mosaic_preview_coords: Optional[List[float]] = None
     requires_token: bool = False
     max_items_per_tile: Optional[int] = None
+    vector_tilesets: Optional[List[VectorTileset]] = None
     hidden: bool = False  # Hide from API
 
     def get_full_render_qs(self, collection: str, item: Optional[str] = None) -> str:
@@ -63,9 +108,22 @@ def get_render_params(self) -> str:
         if "format" in self.render_params:
             return default_params
 
-        # Encforce PNG rendering when otherwise unspecified
+        # Enforce PNG rendering when otherwise unspecified
         return default_params + "&format=png"
 
+    def get_vector_tileset(self, tileset_id: str) -> Optional[VectorTileset]:
+        """
+        Get a tileset by id.
+        """
+        tilesets = self.vector_tilesets or []
+        matches = [tileset for tileset in tilesets if tileset.id == tileset_id]
+
+        return matches[0] if matches else None
+
+    @property
+    def has_vector_tiles(self) -> bool:
+        return bool(self.vector_tilesets)
+
     @property
     def should_add_collection_links(self) -> bool:
         # TODO: has_mosaic flag is legacy from now-deprecated
@@ -84,14 +142,6 @@ class Config:
         json_dumps = orjson_dumps
 
 
-class CamelModel(BaseModel):
-    class Config:
-        alias_generator = camelize
-        allow_population_by_field_name = True
-        json_loads = orjson.loads
-        json_dumps = orjson_dumps
-
-
 class Mosaics(CamelModel):
     """
     A single predefined CQL2-JSON query representing a named mosaic.
@@ -124,7 +174,7 @@ class LegendConfig(CamelModel):
         `none` (note, `none` is a string literal).
     labels:
         List of string labels, ideally fewer than 3 items. Will be flex
-        spaced-between under the lagend image.
+        spaced-between under the legend image.
     trim_start:
         The number of items to trim from the start of the legend definition.
         Used if there are values important for rendering (e.g. nodata) that
@@ -133,7 +183,7 @@ class LegendConfig(CamelModel):
         Same as trim_start, but for the end of the legend definition.
     scale_factor:
         A factor to multiply interval legend labels by. Useful for scaled
-        reasters whose colormap definitions map to unscaled values, effectively
+        rasters whose colormap definitions map to unscaled values, effectively
         showing legend labels as scaled values.
     """
 
@@ -144,6 +194,34 @@ class LegendConfig(CamelModel):
     scale_factor: Optional[float]
 
 
+class VectorTileOptions(CamelModel):
+    """
+    Defines a set of vector tile render options for a collection.
+
+    Attributes
+    ----------
+    tilejson_key:
+        The key in the collection-level assets which contains the tilejson URL.
+    source_layer:
+        The source layer name to render from the associated vector tiles.
+    fill_color:
+        The fill color for polygons.
+    stroke_color:
+        The stroke color for lines.
+    stroke_width:
+        The stroke width for lines.
+    filter:
+        MapBox Filter Expression to filter vector features by.
+    """
+
+    tilejson_key: str
+    source_layer: str
+    fill_color: Optional[str]
+    stroke_color: Optional[str]
+    stroke_width: Optional[int]
+    filter: Optional[List[Any]]
+
+
 class RenderOptionCondition(CamelModel):
     """
     Defines a property/value condition for a render config to be enabled
@@ -172,10 +250,15 @@ class RenderOptions(CamelModel):
     description:
         A longer description of the render option that can be used to explain
         its content.
+    type:
+        The type of render option, defaults to raster-tile.
     options:
-        A URL query-string encoded string of TiTiler rendering options. See
-        "Query Parameters":
+        A URL query-string encoded string of TiTiler rendering options. Valid
+        only for `raster-tile` types.  See "Query Parameters":
         https://developmentseed.org/titiler/endpoints/cog/#description
+    vector_options:
+        Options for rendering vector tiles. Valid only for `vt-polygon` and
+        `vt-line` types.
     min_zoom:
         Zoom level at which to start rendering the layer.
     legend:
@@ -187,7 +270,9 @@ class RenderOptions(CamelModel):
 
     name: str
     description: Optional[str] = None
-    options: str
+    type: Optional[RenderOptionType] = Field(default=RenderOptionType.raster_tile)
+    options: Optional[str]
+    vector_options: Optional[VectorTileOptions] = None
     min_zoom: int
     legend: Optional[LegendConfig] = None
     conditions: Optional[List[RenderOptionCondition]] = None
@@ -258,3 +343,6 @@ def get_config(self, collection_id: str) -> Optional[CollectionConfig]:
 
     def set_config(self, collection_id: str, config: CollectionConfig) -> None:
         self.upsert("", collection_id, config)
+
+    def get_all_configs(self) -> List[Tuple[Optional[str], CollectionConfig]]:
+        return [(config[1], config[2]) for config in self.get_all()]

pccommon/pccommon/constants.py

@@ -4,6 +4,7 @@
 DEFAULT_CONTAINER_CONFIG_TABLE_NAME = "containerconfig"
 DEFAULT_IP_EXCEPTION_CONFIG_TABLE_NAME = "ipexceptionlist"
 
+DEFAULT_COLLECTION_REGION = "westeurope"
 DEFAULT_TTL = 600  # 10 minutes
 DEFAULT_IP_EXCEPTIONS_TTL = 43200  # 12 hours
 

pccommon/pccommon/tables.py

@@ -61,6 +61,9 @@ def __init__(
         self._cache: Cache = TTLCache(maxsize=1024, ttl=ttl or DEFAULT_TTL)
         self._cache_lock: Lock = Lock()
 
+    def _get_cache(self) -> Cache:
+        return self._cache
+
     def _ensure_table_client(self) -> None:
         if not self._table_client:
             raise TableError("Table client not initialized. Use as a context manager.")
@@ -189,7 +192,11 @@ def update(self, partition_key: str, row_key: str, entity: M) -> None:
                 }
             )
 
-    @cachedmethod(cache=lambda self: self._cache, lock=lambda self: self._cache_lock)
+    @cachedmethod(
+        cache=lambda self: self._get_cache(),
+        lock=lambda self: self._cache_lock,
+        key=lambda _, partition_key, row_key: f"get_{partition_key}_{row_key}",
+    )
     def get(self, partition_key: str, row_key: str) -> Optional[M]:
         with self as table_client:
             try:
@@ -201,9 +208,14 @@ def get(self, partition_key: str, row_key: str) -> Optional[M]:
             except ResourceNotFoundError:
                 return None
 
+    @cachedmethod(
+        cache=lambda self: self._get_cache(),
+        lock=lambda self: self._cache_lock,
+        key=lambda _: "getall",
+    )
     def get_all(self) -> Iterable[Tuple[Optional[str], Optional[str], M]]:
         with self as table_client:
-            for entity in table_client.query_entities(""):
+            for entity in table_client.query_entities("PartitionKey eq ''"):
                 partition_key, row_key = entity.get("PartitionKey"), entity.get(
                     "RowKey"
                 )

pccommon/setup.py

@@ -12,7 +12,7 @@
     "azure-identity==1.7.1",
     "azure-data-tables==12.4.0",
     "azure-storage-blob==12.12.0",
-    "pydantic==1.9.0",
+    "pydantic>=1.9, <2.0.0",
     "cachetools==5.0.0",
     "types-cachetools==4.2.9",
     "pyhumps==3.5.3",