Merge remote-tracking branch 'origin/master' into devtools

Makefile

@@ -8,14 +8,18 @@ APP_VERSION ?= 0.1.0
 
 IMP_DIR := STACpopulator/implementations
 STAC_HOST ?= http://localhost:8880/stac
+# CATALOG = https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/birdhouse/testdata/xclim/cmip6/catalog.html
+CATALOG = https://daccs.cs.toronto.edu/twitcher/ows/proxy/thredds/catalog/datasets/CMIP6/catalog.html
+# CATALOG = https://daccs.cs.toronto.edu/twitcher/ows/proxy/thredds/catalog/datasets/CMIP6/CMIP/NOAA-GFDL/catalog.html
+# CATALOG = https://daccs.cs.toronto.edu/twitcher/ows/proxy/thredds/catalog/datasets/CMIP6/CMIP/AS-RCEC/catalog.html
 
 ## -- Testing targets -------------------------------------------------------------------------------------------- ##
 
 setup-pyessv-archive:
 	git clone "https://github.com/ES-DOC/pyessv-archive" ~/.esdoc/pyessv-archive
 
 test-cmip6:
-	python $(IMP_DIR)/CMIP6_UofT/add_CMIP6.py $(STAC_HOST) https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/birdhouse/testdata/xclim/cmip6/catalog.html
+	python $(IMP_DIR)/CMIP6_UofT/add_CMIP6.py $(STAC_HOST) $(CATALOG)
 
 del-cmip6:
 	curl --location --request DELETE '$(STAC_HOST)/collections/CMIP6_UofT'

STACpopulator/implementations/CMIP6_UofT/add_CMIP6.py

@@ -36,7 +36,7 @@
 InstitutionID = collection2literal(CV.institution_id)
 NominalResolution = collection2literal(CV.nominal_resolution)
 Realm = collection2literal(CV.realm)
-SourceID = collection2literal(CV.source_id)
+SourceID = collection2literal(CV.source_id, "source_id")
 SourceType = collection2literal(CV.source_type)
 SubExperimentID = collection2literal(CV.sub_experiment_id)
 TableID = collection2literal(CV.table_id)
@@ -74,7 +74,7 @@ class CMIP6ItemProperties(STACItemProperties, validate_assignment=True):
     physics_index: int
     realization_index: int
     forcing_index: int
-    tracking_id: str = ""
+    tracking_id: str = Field("")
     version: str = Field("")
     product: str
     license: str
@@ -162,7 +162,7 @@ def create_stac_item(self, item_name: str, item_data: MutableMapping[str, Any])
             dchelper = DataCubeHelper(item_data)
             dc_ext = DatacubeExtension.ext(item, add_if_missing=True)
             dc_ext.apply(dimensions=dchelper.dimensions, variables=dchelper.variables)
-        except:
+        except Exception:
             LOGGER.warning(f"Failed to add Datacube extension to item {item_name}")
 
         # print(json.dumps(item.to_dict()))

STACpopulator/populator_base.py

@@ -133,4 +133,5 @@ def ingest(self) -> None:
         for item_name, item_data in self._ingest_pipeline:
             LOGGER.info(f"Creating STAC representation for {item_name}")
             stac_item = self.create_stac_item(item_name, item_data)
-            post_stac_item(self.stac_host, self.collection_id, item_name, stac_item, self.update)
+            if stac_item != -1:
+                post_stac_item(self.stac_host, self.collection_id, item_name, stac_item, self.update)

STACpopulator/stac_utils.py

@@ -74,8 +74,8 @@ def load_collection_configuration() -> MutableMapping[str, Any]:
     return collection_info
 
 
-def collection2literal(collection):
-    terms = tuple(term.label for term in collection)
+def collection2literal(collection, property="label"):
+    terms = tuple(getattr(term, property) for term in collection)
     return Literal[terms]