From 19ebcf262ca257298fb511c58d7e072dec8d24d0 Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Fri, 23 Feb 2024 16:06:16 -0500 Subject: [PATCH 01/11] add assets to collections --- STACpopulator/populator_base.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/STACpopulator/populator_base.py b/STACpopulator/populator_base.py index 9e5fa53..6133eb6 100644 --- a/STACpopulator/populator_base.py +++ b/STACpopulator/populator_base.py @@ -5,7 +5,7 @@ import os from abc import ABC, abstractmethod from datetime import datetime -from typing import Any, MutableMapping, Optional, Type, Union +from typing import Any, Dict, MutableMapping, Optional, Type, Union import pystac from requests.sessions import Session @@ -135,6 +135,9 @@ def create_stac_collection(self) -> dict[str, Any]: ) self._collection_info["extent"] = pystac.Extent(sp_extent, tmp_extent) self._collection_info["summaries"] = pystac.Summaries({"needs_summaries_update": ["true"]}) + + if "assets" in self._collection_info: + self._collection_info["assets"] = self.__make_collection_assets() collection = pystac.Collection(**self._collection_info) collection.add_links(self._ingest_pipeline.links) @@ -142,6 +145,17 @@ def create_stac_collection(self) -> dict[str, Any]: self.publish_stac_collection(collection_data) return collection_data + def __make_collection_assets(self) -> Dict[pystac.Asset]: + """Creates collection level assets based on data read in from the configuration file. + + :return: Dictionary of pystac Asset objects + :rtype: Dict[pystac.Asset] + """ + pystac_assets = {} + for asset_name, asset_info in self._collection_info["assets"].items(): + pystac_assets[asset_name] = pystac.Asset(**asset_info) + return pystac_assets + def publish_stac_collection(self, collection_data: dict[str, Any]) -> None: post_stac_collection(self.stac_host, collection_data, self.update, session=self._session) From 5011beaa02c2da924b7d995fdb72fc09a58076a5 Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Fri, 23 Feb 2024 16:06:39 -0500 Subject: [PATCH 02/11] add CMIP6 implementation collection assets --- .../CMIP6_UofT/collection_config.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/STACpopulator/implementations/CMIP6_UofT/collection_config.yml b/STACpopulator/implementations/CMIP6_UofT/collection_config.yml index 0f43c78..6cd1f75 100644 --- a/STACpopulator/implementations/CMIP6_UofT/collection_config.yml +++ b/STACpopulator/implementations/CMIP6_UofT/collection_config.yml @@ -4,4 +4,17 @@ description: Coupled Model Intercomparison Project phase 6 keywords: ['CMIP', 'CMIP6', 'WCRP', 'Climate Change'] license: "CC-BY-4.0" spatialextent: [-180, -90, 180, 90] -temporalextent: ['1850-01-01', null] \ No newline at end of file +temporalextent: ['1850-01-01', null] + +assets: + homepage: + title : Project homepage + href : https://wcrp-cmip.org/cmip-phase-6-cmip6/ + description: CMIP6 project homepage + media_type: text/html + license: + title : License + href : https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/main/LICENSE-CC-BY + description: License for CMIP6 data + media_type: text/plain + From 5d86e2c15e8d000d8af3c01101a84d69a0c1caa3 Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Mon, 26 Feb 2024 15:40:03 -0500 Subject: [PATCH 03/11] bug fix on return type --- STACpopulator/populator_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/STACpopulator/populator_base.py b/STACpopulator/populator_base.py index 6133eb6..0e41e9d 100644 --- a/STACpopulator/populator_base.py +++ b/STACpopulator/populator_base.py @@ -145,7 +145,7 @@ def create_stac_collection(self) -> dict[str, Any]: self.publish_stac_collection(collection_data) return collection_data - def __make_collection_assets(self) -> Dict[pystac.Asset]: + def __make_collection_assets(self) -> Dict[str, pystac.Asset]: """Creates collection level assets based on data read in from the configuration file. :return: Dictionary of pystac Asset objects From 5601c5c513e8354d69d65381d1bd81bdb992dc33 Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Mon, 26 Feb 2024 15:40:56 -0500 Subject: [PATCH 04/11] updating UoT CMIP6 collection temporal extent --- STACpopulator/implementations/CMIP6_UofT/collection_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/STACpopulator/implementations/CMIP6_UofT/collection_config.yml b/STACpopulator/implementations/CMIP6_UofT/collection_config.yml index 6cd1f75..126270b 100644 --- a/STACpopulator/implementations/CMIP6_UofT/collection_config.yml +++ b/STACpopulator/implementations/CMIP6_UofT/collection_config.yml @@ -4,7 +4,7 @@ description: Coupled Model Intercomparison Project phase 6 keywords: ['CMIP', 'CMIP6', 'WCRP', 'Climate Change'] license: "CC-BY-4.0" spatialextent: [-180, -90, 180, 90] -temporalextent: ['1850-01-01', null] +temporalextent: ['1850-01-01', '2500-01-01'] assets: homepage: From b0259f54190c4c468d6dbc6098b77b8f894aef8c Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Mon, 26 Feb 2024 15:43:47 -0500 Subject: [PATCH 05/11] Updating CHANGES.md --- CHANGES.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 8cb305a..96b0e30 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,9 @@ ## [Unreleased](https://github.com/crim-ca/stac-populator) (latest) +- Adding ability to add collection level assets +- Adding collection assets to `CMIP6_UofT` +- Adding an end date to `CMIP6_UofT`'s temporal extent for better rendering in STAC Browser ## [0.6.0](https://github.com/crim-ca/stac-populator/tree/0.6.0) (2024-02-22) From 7aa26d188f747dcf27976bb5f8e521bf3f6b8d8f Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Mon, 26 Feb 2024 18:18:14 -0500 Subject: [PATCH 06/11] adding ability to add collection level links --- STACpopulator/populator_base.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/STACpopulator/populator_base.py b/STACpopulator/populator_base.py index 0e41e9d..f5ff453 100644 --- a/STACpopulator/populator_base.py +++ b/STACpopulator/populator_base.py @@ -5,7 +5,7 @@ import os from abc import ABC, abstractmethod from datetime import datetime -from typing import Any, Dict, MutableMapping, Optional, Type, Union +from typing import Any, Dict, List, MutableMapping, Optional, Type, Union import pystac from requests.sessions import Session @@ -136,15 +136,38 @@ def create_stac_collection(self) -> dict[str, Any]: self._collection_info["extent"] = pystac.Extent(sp_extent, tmp_extent) self._collection_info["summaries"] = pystac.Summaries({"needs_summaries_update": ["true"]}) + # Add any assets if provided in the config if "assets" in self._collection_info: self._collection_info["assets"] = self.__make_collection_assets() + + # Construct links if provided in the config + if "links" in self._collection_info: + collection_links = self.__make_collection_links() + # need to remove the links item other constructing a collection object fails + _ = self._collection_info.pop("links") + else: + collection_links = [] + collection = pystac.Collection(**self._collection_info) + if collection_links: + collection.add_links(collection_links) collection.add_links(self._ingest_pipeline.links) collection_data = collection.to_dict() self.publish_stac_collection(collection_data) return collection_data + def __make_collection_links(self) -> List[pystac.Link]: + """Create collection level links based on data read in from the configuration file. + + :return: List of pystac Link objects + :rtype: List[pystac.Link] + """ + links = [] + for link_name, link_info in self._collection_info["links"].items(): + links.append(pystac.Link(**link_info)) + return links + def __make_collection_assets(self) -> Dict[str, pystac.Asset]: """Creates collection level assets based on data read in from the configuration file. From a7daa7b1dd164754110d5a04f2adc0727cdf83d8 Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Mon, 26 Feb 2024 18:18:39 -0500 Subject: [PATCH 07/11] changing CMIP6_UofT assets to links --- .../implementations/CMIP6_UofT/collection_config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/STACpopulator/implementations/CMIP6_UofT/collection_config.yml b/STACpopulator/implementations/CMIP6_UofT/collection_config.yml index 126270b..bf8c151 100644 --- a/STACpopulator/implementations/CMIP6_UofT/collection_config.yml +++ b/STACpopulator/implementations/CMIP6_UofT/collection_config.yml @@ -6,15 +6,15 @@ license: "CC-BY-4.0" spatialextent: [-180, -90, 180, 90] temporalextent: ['1850-01-01', '2500-01-01'] -assets: +links: homepage: + rel: about title : Project homepage - href : https://wcrp-cmip.org/cmip-phase-6-cmip6/ - description: CMIP6 project homepage + target : https://wcrp-cmip.org/cmip-phase-6-cmip6/ media_type: text/html license: + rel: license title : License - href : https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/main/LICENSE-CC-BY - description: License for CMIP6 data + target : https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/main/LICENSE-CC-BY media_type: text/plain From 63beeae151e6003824682484f639b61e518e4146 Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Mon, 26 Feb 2024 18:19:18 -0500 Subject: [PATCH 08/11] updating CHANGED.md --- CHANGES.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 96b0e30..f3f8fee 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,8 @@ - Adding ability to add collection level assets -- Adding collection assets to `CMIP6_UofT` +- Adding ability to add collection level links +- Adding collection links to `CMIP6_UofT` - Adding an end date to `CMIP6_UofT`'s temporal extent for better rendering in STAC Browser ## [0.6.0](https://github.com/crim-ca/stac-populator/tree/0.6.0) (2024-02-22) From 269d66ad7a76affd09ffa46a7203f6c3817f57b7 Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Tue, 27 Feb 2024 11:07:56 -0500 Subject: [PATCH 09/11] refactoring logic for adding collection links and assets --- STACpopulator/populator_base.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/STACpopulator/populator_base.py b/STACpopulator/populator_base.py index f5ff453..97e9767 100644 --- a/STACpopulator/populator_base.py +++ b/STACpopulator/populator_base.py @@ -137,16 +137,10 @@ def create_stac_collection(self) -> dict[str, Any]: self._collection_info["summaries"] = pystac.Summaries({"needs_summaries_update": ["true"]}) # Add any assets if provided in the config - if "assets" in self._collection_info: - self._collection_info["assets"] = self.__make_collection_assets() + self._collection_info["assets"] = self.__make_collection_assets() - # Construct links if provided in the config - if "links" in self._collection_info: - collection_links = self.__make_collection_links() - # need to remove the links item other constructing a collection object fails - _ = self._collection_info.pop("links") - else: - collection_links = [] + # Construct links if provided in the config. This needs to be done before constructing a collection object. + collection_links = self.__make_collection_links() collection = pystac.Collection(**self._collection_info) @@ -164,7 +158,8 @@ def __make_collection_links(self) -> List[pystac.Link]: :rtype: List[pystac.Link] """ links = [] - for link_name, link_info in self._collection_info["links"].items(): + config_links = self._collection_info.pop("links", []) + for link_name, link_info in config_links.items(): links.append(pystac.Link(**link_info)) return links @@ -175,8 +170,9 @@ def __make_collection_assets(self) -> Dict[str, pystac.Asset]: :rtype: Dict[pystac.Asset] """ pystac_assets = {} - for asset_name, asset_info in self._collection_info["assets"].items(): - pystac_assets[asset_name] = pystac.Asset(**asset_info) + if "assets" in self._collection_info: + for asset_name, asset_info in self._collection_info["assets"].items(): + pystac_assets[asset_name] = pystac.Asset(**asset_info) return pystac_assets def publish_stac_collection(self, collection_data: dict[str, Any]) -> None: From 19412a5277fc80cc21709c8579d95647f8a31b5c Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Tue, 27 Feb 2024 11:20:38 -0500 Subject: [PATCH 10/11] bug fix in __make_collection_links --- STACpopulator/populator_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/STACpopulator/populator_base.py b/STACpopulator/populator_base.py index 97e9767..5ad5c20 100644 --- a/STACpopulator/populator_base.py +++ b/STACpopulator/populator_base.py @@ -158,7 +158,7 @@ def __make_collection_links(self) -> List[pystac.Link]: :rtype: List[pystac.Link] """ links = [] - config_links = self._collection_info.pop("links", []) + config_links = self._collection_info.pop("links", {}) for link_name, link_info in config_links.items(): links.append(pystac.Link(**link_info)) return links From f750fa516c09ecd55129723a783a64746cd9b73b Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Tue, 27 Feb 2024 22:14:40 -0500 Subject: [PATCH 11/11] specify collection assets as yaml list rather than dict --- .../implementations/CMIP6_UofT/collection_config.yml | 6 ++---- STACpopulator/populator_base.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/STACpopulator/implementations/CMIP6_UofT/collection_config.yml b/STACpopulator/implementations/CMIP6_UofT/collection_config.yml index bf8c151..9d3665f 100644 --- a/STACpopulator/implementations/CMIP6_UofT/collection_config.yml +++ b/STACpopulator/implementations/CMIP6_UofT/collection_config.yml @@ -7,13 +7,11 @@ spatialextent: [-180, -90, 180, 90] temporalextent: ['1850-01-01', '2500-01-01'] links: - homepage: - rel: about + - rel: about title : Project homepage target : https://wcrp-cmip.org/cmip-phase-6-cmip6/ media_type: text/html - license: - rel: license + - rel: license title : License target : https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/main/LICENSE-CC-BY media_type: text/plain diff --git a/STACpopulator/populator_base.py b/STACpopulator/populator_base.py index 5ad5c20..44341f3 100644 --- a/STACpopulator/populator_base.py +++ b/STACpopulator/populator_base.py @@ -159,7 +159,7 @@ def __make_collection_links(self) -> List[pystac.Link]: """ links = [] config_links = self._collection_info.pop("links", {}) - for link_name, link_info in config_links.items(): + for link_info in config_links: links.append(pystac.Link(**link_info)) return links