diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 0e6dce31..b15494c4 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -1,4 +1,4 @@ -name: stac-fastapi-elasticsearch +name: sfeos on: push: diff --git a/CHANGELOG.md b/CHANGELOG.md index 7931b027..c918a12c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,11 +13,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed - Updated stac-fastapi libraries to v3.0.0a1 [#265](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/265) +- Updated stac-fastapi libraries to v3.0.0a3 [#269](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/269) ### Fixed - API sort extension tests [#264](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/264) - Basic auth permission fix for checking route path instead of absolute path [#266](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/266) +- Remove deprecated filter_fields property, return all properties as default [#269](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/269) ## [v3.0.0a1] diff --git a/stac_fastapi/core/setup.py b/stac_fastapi/core/setup.py index c4071651..90e5b141 100644 --- a/stac_fastapi/core/setup.py +++ b/stac_fastapi/core/setup.py @@ -10,9 +10,9 @@ "attrs>=23.2.0", "pydantic[dotenv]", "stac_pydantic>=3", - "stac-fastapi.types==3.0.0a1", - "stac-fastapi.api==3.0.0a1", - "stac-fastapi.extensions==3.0.0a1", + "stac-fastapi.types==3.0.0a3", + "stac-fastapi.api==3.0.0a3", + "stac-fastapi.extensions==3.0.0a3", "orjson", "overrides", "geojson-pydantic", diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index 984b2e0a..57f7c816 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -9,7 +9,6 @@ import attr import orjson -import stac_pydantic from fastapi import HTTPException, Request from overrides import overrides from pydantic import ValidationError @@ -25,19 +24,16 @@ from stac_fastapi.core.models.links import PagingLinks from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer from stac_fastapi.core.session import Session +from stac_fastapi.core.utilities import filter_fields +from stac_fastapi.extensions.core.filter.client import AsyncBaseFiltersClient from stac_fastapi.extensions.third_party.bulk_transactions import ( BaseBulkTransactionsClient, BulkTransactionMethod, Items, ) from stac_fastapi.types import stac as stac_types -from stac_fastapi.types.config import Settings from stac_fastapi.types.conformance import BASE_CONFORMANCE_CLASSES -from stac_fastapi.types.core import ( - AsyncBaseCoreClient, - AsyncBaseFiltersClient, - AsyncBaseTransactionsClient, -) +from stac_fastapi.types.core import AsyncBaseCoreClient, AsyncBaseTransactionsClient from stac_fastapi.types.extension import ApiExtension from stac_fastapi.types.requests import get_base_url from stac_fastapi.types.rfc3339 import DateTimeType @@ -491,34 +487,26 @@ async def get_search( base_args["intersects"] = orjson.loads(unquote_plus(intersects)) if sortby: - sort_param = [] - for sort in sortby: - sort_param.append( - { - "field": sort[1:], - "direction": "desc" if sort[0] == "-" else "asc", - } - ) - base_args["sortby"] = sort_param + base_args["sortby"] = [ + {"field": sort[1:], "direction": "desc" if sort[0] == "-" else "asc"} + for sort in sortby + ] if filter: - if filter_lang == "cql2-json": - base_args["filter-lang"] = "cql2-json" - base_args["filter"] = orjson.loads(unquote_plus(filter)) - else: - base_args["filter-lang"] = "cql2-json" - base_args["filter"] = orjson.loads(to_cql2(parse_cql2_text(filter))) + base_args["filter-lang"] = "cql2-json" + base_args["filter"] = orjson.loads( + unquote_plus(filter) + if filter_lang == "cql2-json" + else to_cql2(parse_cql2_text(filter)) + ) if fields: - includes = set() - excludes = set() + includes, excludes = set(), set() for field in fields: if field[0] == "-": excludes.add(field[1:]) - elif field[0] == "+": - includes.add(field[1:]) else: - includes.add(field) + includes.add(field[1:] if field[0] in "+ " else field) base_args["fields"] = {"include": includes, "exclude": excludes} # Do the request @@ -614,32 +602,22 @@ async def post_search( collection_ids=search_request.collections, ) + fields = ( + getattr(search_request, "fields", None) + if self.extension_is_enabled("FieldsExtension") + else None + ) + include: Set[str] = fields.include if fields and fields.include else set() + exclude: Set[str] = fields.exclude if fields and fields.exclude else set() + items = [ - self.item_serializer.db_to_stac(item, base_url=base_url) for item in items + filter_fields( + self.item_serializer.db_to_stac(item, base_url=base_url), + include, + exclude, + ) + for item in items ] - - if self.extension_is_enabled("FieldsExtension"): - if search_request.query is not None: - query_include: Set[str] = set( - [ - k if k in Settings.get().indexed_fields else f"properties.{k}" - for k in search_request.query.keys() - ] - ) - if not search_request.fields.include: - search_request.fields.include = query_include - else: - search_request.fields.include.union(query_include) - - filter_kwargs = search_request.fields.filter_fields - - items = [ - orjson.loads( - stac_pydantic.Item(**feat).json(**filter_kwargs, exclude_unset=True) - ) - for feat in items - ] - links = await PagingLinks(request=request, next=next_token).get_links() return stac_types.ItemCollection( diff --git a/stac_fastapi/core/stac_fastapi/core/extensions/fields.py b/stac_fastapi/core/stac_fastapi/core/extensions/fields.py new file mode 100644 index 00000000..e2b11816 --- /dev/null +++ b/stac_fastapi/core/stac_fastapi/core/extensions/fields.py @@ -0,0 +1,41 @@ +"""Fields extension.""" + +from typing import Optional, Set + +from pydantic import BaseModel, Field + +from stac_fastapi.extensions.core import FieldsExtension as FieldsExtensionBase +from stac_fastapi.extensions.core.fields import request + + +class PostFieldsExtension(request.PostFieldsExtension): + """PostFieldsExtension.""" + + # Set defaults if needed + # include : Optional[Set[str]] = Field( + # default_factory=lambda: { + # "id", + # "type", + # "stac_version", + # "geometry", + # "bbox", + # "links", + # "assets", + # "properties.datetime", + # "collection", + # } + # ) + include: Optional[Set[str]] = set() + exclude: Optional[Set[str]] = set() + + +class FieldsExtensionPostRequest(BaseModel): + """Additional fields and schema for the POST request.""" + + fields: Optional[PostFieldsExtension] = Field(PostFieldsExtension()) + + +class FieldsExtension(FieldsExtensionBase): + """Override the POST model.""" + + POST = FieldsExtensionPostRequest diff --git a/stac_fastapi/core/stac_fastapi/core/utilities.py b/stac_fastapi/core/stac_fastapi/core/utilities.py index faa4f6a9..d8c69529 100644 --- a/stac_fastapi/core/stac_fastapi/core/utilities.py +++ b/stac_fastapi/core/stac_fastapi/core/utilities.py @@ -3,7 +3,9 @@ This module contains functions for transforming geospatial coordinates, such as converting bounding boxes to polygon representations. """ -from typing import List +from typing import Any, Dict, List, Optional, Set, Union + +from stac_fastapi.types.stac import Item MAX_LIMIT = 10000 @@ -21,3 +23,113 @@ def bbox2polygon(b0: float, b1: float, b2: float, b3: float) -> List[List[List[f List[List[List[float]]]: A polygon represented as a list of lists of coordinates. """ return [[[b0, b1], [b2, b1], [b2, b3], [b0, b3], [b0, b1]]] + + +# copied from stac-fastapi-pgstac +# https://github.com/stac-utils/stac-fastapi-pgstac/blob/26f6d918eb933a90833f30e69e21ba3b4e8a7151/stac_fastapi/pgstac/utils.py#L10-L116 +def filter_fields( # noqa: C901 + item: Union[Item, Dict[str, Any]], + include: Optional[Set[str]] = None, + exclude: Optional[Set[str]] = None, +) -> Item: + """Preserve and remove fields as indicated by the fields extension include/exclude sets. + + Returns a shallow copy of the Item with the fields filtered. + + This will not perform a deep copy; values of the original item will be referenced + in the return item. + """ + if not include and not exclude: + return item + + # Build a shallow copy of included fields on an item, or a sub-tree of an item + def include_fields( + source: Dict[str, Any], fields: Optional[Set[str]] + ) -> Dict[str, Any]: + if not fields: + return source + + clean_item: Dict[str, Any] = {} + for key_path in fields or []: + key_path_parts = key_path.split(".") + key_root = key_path_parts[0] + if key_root in source: + if isinstance(source[key_root], dict) and len(key_path_parts) > 1: + # The root of this key path on the item is a dict, and the + # key path indicates a sub-key to be included. Walk the dict + # from the root key and get the full nested value to include. + value = include_fields( + source[key_root], fields={".".join(key_path_parts[1:])} + ) + + if isinstance(clean_item.get(key_root), dict): + # A previously specified key and sub-keys may have been included + # already, so do a deep merge update if the root key already exists. + dict_deep_update(clean_item[key_root], value) + else: + # The root key does not exist, so add it. Fields + # extension only allows nested referencing on dicts, so + # this won't overwrite anything. + clean_item[key_root] = value + else: + # The item value to include is not a dict, or, it is a dict but the + # key path is for the whole value, not a sub-key. Include the entire + # value in the cleaned item. + clean_item[key_root] = source[key_root] + else: + # The key, or root key of a multi-part key, is not present in the item, + # so it is ignored + pass + return clean_item + + # For an item built up for included fields, remove excluded fields. This + # modifies `source` in place. + def exclude_fields(source: Dict[str, Any], fields: Optional[Set[str]]) -> None: + for key_path in fields or []: + key_path_part = key_path.split(".") + key_root = key_path_part[0] + if key_root in source: + if isinstance(source[key_root], dict) and len(key_path_part) > 1: + # Walk the nested path of this key to remove the leaf-key + exclude_fields( + source[key_root], fields={".".join(key_path_part[1:])} + ) + # If, after removing the leaf-key, the root is now an empty + # dict, remove it entirely + if not source[key_root]: + del source[key_root] + else: + # The key's value is not a dict, or there is no sub-key to remove. The + # entire key can be removed from the source. + source.pop(key_root, None) + + # Coalesce incoming type to a dict + item = dict(item) + + clean_item = include_fields(item, include) + + # If, after including all the specified fields, there are no included properties, + # return just id and collection. + if not clean_item: + return Item({"id": item["id"], "collection": item["collection"]}) + + exclude_fields(clean_item, exclude) + + return Item(**clean_item) + + +def dict_deep_update(merge_to: Dict[str, Any], merge_from: Dict[str, Any]) -> None: + """Perform a deep update of two dicts. + + merge_to is updated in-place with the values from merge_from. + merge_from values take precedence over existing values in merge_to. + """ + for k, v in merge_from.items(): + if ( + k in merge_to + and isinstance(merge_to[k], dict) + and isinstance(merge_from[k], dict) + ): + dict_deep_update(merge_to[k], merge_from[k]) + else: + merge_to[k] = v diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index 6a5ee006..ebaf5f7b 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -12,6 +12,7 @@ TransactionsClient, ) from stac_fastapi.core.extensions import QueryExtension +from stac_fastapi.core.extensions.fields import FieldsExtension from stac_fastapi.core.session import Session from stac_fastapi.elasticsearch.config import ElasticsearchSettings from stac_fastapi.elasticsearch.database_logic import ( @@ -20,7 +21,6 @@ create_index_templates, ) from stac_fastapi.extensions.core import ( - FieldsExtension, FilterExtension, SortExtension, TokenPaginationExtension, diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py index d06b0f29..fbf43934 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py @@ -12,9 +12,9 @@ TransactionsClient, ) from stac_fastapi.core.extensions import QueryExtension +from stac_fastapi.core.extensions.fields import FieldsExtension from stac_fastapi.core.session import Session from stac_fastapi.extensions.core import ( - FieldsExtension, FilterExtension, SortExtension, TokenPaginationExtension, diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index 8fe09934..798be3f7 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -118,7 +118,10 @@ async def test_app_context_results(app_client, txn_client, ctx, load_test_data): @pytest.mark.asyncio async def test_app_fields_extension(app_client, ctx, txn_client): - resp = await app_client.get("/search", params={"collections": ["test-collection"]}) + resp = await app_client.get( + "/search", + params={"collections": ["test-collection"], "fields": "+properties.datetime"}, + ) assert resp.status_code == 200 resp_json = resp.json() assert list(resp_json["features"][0]["properties"]) == ["datetime"] @@ -132,11 +135,12 @@ async def test_app_fields_extension_query(app_client, ctx, txn_client): json={ "query": {"proj:epsg": {"gte": item["properties"]["proj:epsg"]}}, "collections": ["test-collection"], + "fields": {"include": ["properties.datetime", "properties.proj:epsg"]}, }, ) assert resp.status_code == 200 resp_json = resp.json() - assert list(resp_json["features"][0]["properties"]) == ["datetime", "proj:epsg"] + assert set(resp_json["features"][0]["properties"]) == set(["datetime", "proj:epsg"]) @pytest.mark.asyncio diff --git a/stac_fastapi/tests/resources/test_item.py b/stac_fastapi/tests/resources/test_item.py index a767f0a2..d84577d0 100644 --- a/stac_fastapi/tests/resources/test_item.py +++ b/stac_fastapi/tests/resources/test_item.py @@ -754,7 +754,11 @@ async def test_field_extension_post(app_client, ctx): "ids": [test_item["id"]], "fields": { "exclude": ["assets.B1"], - "include": ["properties.eo:cloud_cover", "properties.orientation"], + "include": [ + "properties.eo:cloud_cover", + "properties.orientation", + "assets", + ], }, } @@ -782,7 +786,7 @@ async def test_field_extension_exclude_and_include(app_client, ctx): resp = await app_client.post("/search", json=body) resp_json = resp.json() - assert "eo:cloud_cover" not in resp_json["features"][0]["properties"] + assert "properties" not in resp_json["features"][0] @pytest.mark.asyncio