fix session fowarding to THREDDS catalog siphon impl

CHANGES.md

@@ -2,7 +2,15 @@
 
 ## [Unreleased](https://github.com/crim-ca/stac-populator) (latest)
 
-<!-- insert list items of new changes here -->
+* Add request ``session`` keyword to all request-related functions and populator methods to allow sharing a common set
+  of settings (`auth`, SSL `verify`, `cert`) across requests toward the STAC Catalog.
+* Add `DirectoryLoader` that allows populating a STAC Catalog with Collections and Items loaded from a crawled directory
+  hierarchy that contains `collection.json` files and other `.json`/`.geojson` items.
+* Add a generic CLI `stac-populator` that can be called to run populator implementations directly
+  using command `stac-populator run <implementation> [impl-args]`.
+* Remove hardcoded `verify=False` to requests calls.
+  If needed for testing purposes, users should use a custom `requests.sessions.Session` with `verify=False` passed to
+  the populator, or alternatively, employ the CLI argument `--no-verify` that will accomplish the same behavior.
 
 ## [0.2.0](https://github.com/crim-ca/stac-populator/tree/0.2.0) (2023-11-10)
 

STACpopulator/implementations/CMIP6_UofT/add_CMIP6.py

@@ -9,6 +9,7 @@
 from pydantic import AnyHttpUrl, ConfigDict, Field, FieldValidationInfo, field_validator
 from pystac.extensions.datacube import DatacubeExtension
 
+from STACpopulator.cli import add_request_options, apply_request_options
 from STACpopulator.implementations.CMIP6_UofT.extensions import DataCubeHelper
 from STACpopulator.input import GenericLoader, ErrorLoader, THREDDSLoader
 from STACpopulator.models import GeoJSONPolygon, STACItemProperties
@@ -155,9 +156,9 @@ def create_stac_item(self, item_name: str, item_data: MutableMapping[str, Any])
 
         # Add datacube extension
         try:
-            dchelper = DataCubeHelper(item_data)
+            dc_helper = DataCubeHelper(item_data)
             dc_ext = DatacubeExtension.ext(item, add_if_missing=True)
-            dc_ext.apply(dimensions=dchelper.dimensions, variables=dchelper.variables)
+            dc_ext.apply(dimensions=dc_helper.dimensions, variables=dc_helper.variables)
         except Exception:
             LOGGER.warning(f"Failed to add Datacube extension to item {item_name}")
 
@@ -178,14 +179,16 @@ def make_parser() -> argparse.ArgumentParser:
 def runner(ns: argparse.Namespace) -> Optional[int] | NoReturn:
     LOGGER.info(f"Arguments to call: {vars(ns)}")
 
-    if ns.mode == "full":
-        data_loader = THREDDSLoader(ns.thredds_catalog_URL)
-    else:
-        # To be implemented
-        data_loader = ErrorLoader()
+    with Session() as session:
+        apply_request_options(session, ns)
+        if ns.mode == "full":
+            data_loader = THREDDSLoader(ns.thredds_catalog_URL, session=session)
+        else:
+            # To be implemented
+            data_loader = ErrorLoader()
 
-    c = CMIP6populator(ns.stac_host, data_loader, ns.update)
-    c.ingest()
+        c = CMIP6populator(ns.stac_host, data_loader, update=ns.update, session=session)
+        c.ingest()
 
 
 def main(*args: str) -> Optional[int]:

STACpopulator/input.py

@@ -9,7 +9,8 @@
 import siphon
 import xncml
 from colorlog import ColoredFormatter
-from siphon.catalog import TDSCatalog
+from requests.sessions import Session
+from siphon.catalog import TDSCatalog, session_manager
 
 from STACpopulator.stac_utils import numpy_to_python_datatypes, url_validate
 
@@ -52,8 +53,35 @@ def reset(self):
         raise NotImplementedError
 
 
+class THREDDSCatalog(TDSCatalog):
+    """
+    Patch to apply a custom request session.
+
+    Because of how :class:`TDSCatalog` automatically loads and parses right away from ``__init__`` call,
+    we need to hack around how the ``session`` attribute gets defined.
+    """
+    def __init__(self, catalog_url: str, session: Optional[Session] = None) -> None:
+        self._session = session
+        super().__init__(catalog_url)
+
+    @property
+    def session(self) -> Session:
+        if self._session is None:
+            self._session = session_manager.create_session()
+        return self._session
+
+    @session.setter
+    def session(self, session: Session) -> None:
+        pass  # ignore to bypass TDSCatalog.__init__ enforcing create_session !
+
+
 class THREDDSLoader(GenericLoader):
-    def __init__(self, thredds_catalog_url: str, depth: Optional[int] = None) -> None:
+    def __init__(
+        self,
+        thredds_catalog_url: str,
+        depth: Optional[int] = None,
+        session: Optional[Session] = None,
+    ) -> None:
         """Constructor
 
         :param thredds_catalog_url: the URL to the THREDDS catalog to ingest