diff --git a/README.md b/README.md index 8cad8a9..06563c7 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ For example, using the [CMIP6_UofT][CMIP6_UofT] implementation, the script can b python STACpopulator/implementations/CMIP6_UofT/add_CMIP6.py \ "http://localhost:8880/stac/" \ "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/birdhouse/testdata/xclim/cmip6/catalog.html" \ - "STACpopulator/implementations/CMIP6_UofT/collection_config.yml" + --config "STACpopulator/implementations/CMIP6_UofT/collection_config.yml" ``` *Note*:
diff --git a/STACpopulator/cli.py b/STACpopulator/cli.py index 11ed00f..ac54176 100644 --- a/STACpopulator/cli.py +++ b/STACpopulator/cli.py @@ -5,12 +5,11 @@ import sys from types import ModuleType import warnings -from datetime import datetime, timezone from typing import Callable from STACpopulator import __version__, implementations from STACpopulator.exceptions import STACPopulatorError -from STACpopulator.logging import setup_logging +from STACpopulator.log import setup_logging def add_parser_args(parser: argparse.ArgumentParser) -> dict[str, Callable]: @@ -21,10 +20,6 @@ def add_parser_args(parser: argparse.ArgumentParser) -> dict[str, Callable]: version=f"%(prog)s {__version__}", help="prints the version of the library and exits", ) - parser.add_argument("--debug", action="store_const", const=logging.DEBUG, help="set logger level to debug") - parser.add_argument( - "--log-file", help="file to write log output to. By default logs will be written to the current directory." - ) commands_subparser = parser.add_subparsers( title="command", dest="command", description="STAC populator command to execute.", required=True ) @@ -52,8 +47,7 @@ def implementation_modules() -> dict[str, ModuleType]: def run(ns: argparse.Namespace) -> int: if ns.command == "run": - logfile_name = ns.log_file or f"{ns.populator}_log_{datetime.now(timezone.utc).isoformat() + 'Z'}.jsonl" - setup_logging(logfile_name, ns.debug or logging.INFO) + setup_logging(ns.log_file, ns.debug or logging.INFO) return implementation_modules()[ns.populator].runner(ns) or 0 diff --git a/STACpopulator/implementations/CMIP6_UofT/add_CMIP6.py b/STACpopulator/implementations/CMIP6_UofT/add_CMIP6.py index 7db2d53..43c4808 100644 --- a/STACpopulator/implementations/CMIP6_UofT/add_CMIP6.py +++ b/STACpopulator/implementations/CMIP6_UofT/add_CMIP6.py @@ -9,7 +9,8 @@ from pystac.extensions.datacube import DatacubeExtension from requests.sessions import Session -from STACpopulator.requests import add_request_options, apply_request_options +from STACpopulator.log import add_logging_options +from STACpopulator.request_utils import add_request_options, apply_request_options from STACpopulator.extensions.cmip6 import CMIP6Helper, CMIP6Properties from STACpopulator.extensions.datacube import DataCubeHelper from STACpopulator.extensions.thredds import THREDDSExtension, THREDDSHelper @@ -31,7 +32,6 @@ def __init__( update: Optional[bool] = False, session: Optional[Session] = None, config_file: Optional[Union[os.PathLike[str], str]] = None, - log_debug: Optional[bool] = False, ) -> None: """Constructor @@ -40,7 +40,7 @@ def __init__( :param data_loader: loader to iterate over ingestion data. """ super().__init__( - stac_host, data_loader, update=update, session=session, config_file=config_file, log_debug=log_debug + stac_host, data_loader, update=update, session=session, config_file=config_file ) def create_stac_item( @@ -106,6 +106,7 @@ def add_parser_args(parser: argparse.ArgumentParser) -> None: ), ) add_request_options(parser) + add_logging_options(parser) def runner(ns: argparse.Namespace) -> int: @@ -120,7 +121,7 @@ def runner(ns: argparse.Namespace) -> int: data_loader = ErrorLoader() c = CMIP6populator( - ns.stac_host, data_loader, update=ns.update, session=session, config_file=ns.config, log_debug=ns.debug + ns.stac_host, data_loader, update=ns.update, session=session, config_file=ns.config ) c.ingest() return 0 diff --git a/STACpopulator/implementations/DirectoryLoader/crawl_directory.py b/STACpopulator/implementations/DirectoryLoader/crawl_directory.py index 9b4cf01..fa98d2c 100644 --- a/STACpopulator/implementations/DirectoryLoader/crawl_directory.py +++ b/STACpopulator/implementations/DirectoryLoader/crawl_directory.py @@ -6,7 +6,8 @@ from requests.sessions import Session -from STACpopulator.requests import add_request_options, apply_request_options +from STACpopulator.log import add_logging_options +from STACpopulator.request_utils import add_request_options, apply_request_options from STACpopulator.input import STACDirectoryLoader from STACpopulator.models import GeoJSONPolygon from STACpopulator.populator_base import STACpopulatorBase @@ -51,6 +52,7 @@ def add_parser_args(parser: argparse.ArgumentParser) -> None: help="Limit search of STAC Collections only to first top-most matches in the crawled directory structure.", ) add_request_options(parser) + add_logging_options(parser) def runner(ns: argparse.Namespace) -> int: diff --git a/STACpopulator/logging.py b/STACpopulator/log.py similarity index 87% rename from STACpopulator/logging.py rename to STACpopulator/log.py index e278360..f195b47 100644 --- a/STACpopulator/logging.py +++ b/STACpopulator/log.py @@ -1,3 +1,4 @@ +import argparse import datetime as dt import json import logging.config @@ -126,3 +127,14 @@ def filter(self, record: logging.LogRecord) -> bool | logging.LogRecord: }, "loggers": {"root": {"level": "DEBUG", "handlers": ["stderr", "file"]}}, } + +def add_logging_options(parser: argparse.ArgumentParser) -> None: + """ + Adds arguments to a parser to configure logging options. + """ + parser.add_argument("--debug", action="store_const", const=logging.DEBUG, help="set logger level to debug") + parser.add_argument( + "--log-file", + default=f"stac_populator_log_{dt.datetime.now(dt.timezone.utc).isoformat() + 'Z'}.jsonl", + help="file to write log output to. By default logs will be written to the current directory." + ) diff --git a/STACpopulator/populator_base.py b/STACpopulator/populator_base.py index 44341f3..8305cf9 100644 --- a/STACpopulator/populator_base.py +++ b/STACpopulator/populator_base.py @@ -27,10 +27,9 @@ def __init__( self, stac_host: str, data_loader: GenericLoader, - update: Optional[bool] = False, + update: bool = False, session: Optional[Session] = None, config_file: Optional[Union[os.PathLike[str], str]] = "collection_config.yml", - log_debug: Optional[bool] = False, ) -> None: """Constructor @@ -41,7 +40,6 @@ def __init__( :raises RuntimeError: Raised if one of the required definitions is not found in the collection info filename """ - super().__init__() self._collection_config_path = config_file self._collection_info: MutableMapping[str, Any] = None self._session = session diff --git a/STACpopulator/requests.py b/STACpopulator/request_utils.py similarity index 100% rename from STACpopulator/requests.py rename to STACpopulator/request_utils.py