Merge pull request #43 from pennsignals/extract_logger_config

Extract logger config

src/dsdk/model.py

@@ -4,32 +4,15 @@
 from __future__ import annotations
 
 from abc import ABC
-from logging import (
-    INFO,
-    Logger,
-    LoggerAdapter,
-    NullHandler,
-    basicConfig,
-    getLogger,
-)
+from logging import INFO
 from typing import TYPE_CHECKING, Optional, cast
 
 from configargparse import ArgParser as ArgumentParser
 
 from .service import Model, Service
-from .utils import load_pickle_file
+from .utils import get_logger, load_pickle_file
 
-# TODO Add import calling function from parent application
-EXTRA = {"callingfunc": ""}
-logger = getLogger(__name__)
-FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \
-    "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \
-        %(message)s}'
-basicConfig(format=FORMAT)
-logger.setLevel(INFO)
-# Add extra kwargs to message format
-logger.addHandler(NullHandler())
-logger = cast(Logger, LoggerAdapter(logger, EXTRA))
+logger = get_logger(__name__, INFO)
 
 
 if TYPE_CHECKING:

src/dsdk/mongo.py

@@ -5,14 +5,7 @@
 
 from abc import ABC
 from contextlib import contextmanager
-from logging import (
-    INFO,
-    Logger,
-    LoggerAdapter,
-    NullHandler,
-    basicConfig,
-    getLogger,
-)
+from logging import INFO
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -26,7 +19,7 @@
 from configargparse import ArgParser as ArgumentParser
 
 from .service import Batch, Model, Service
-from .utils import retry
+from .utils import get_logger, retry
 
 try:
     # Since not everyone will use mongo
@@ -41,17 +34,7 @@
     Database = None
     AutoReconnect = None
 
-# TODO Add import calling function from parent application
-EXTRA = {"callingfunc": ""}
-logger = getLogger(__name__)
-FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \
-    "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \
-        %(message)s}'
-basicConfig(format=FORMAT)
-logger.setLevel(INFO)
-# Add extra kwargs to message format
-logger.addHandler(NullHandler())
-logger = cast(Logger, LoggerAdapter(logger, EXTRA))
+logger = get_logger(__name__, INFO)
 
 
 if TYPE_CHECKING:
@@ -119,19 +102,19 @@ def open_batch(
             with self.open_mongo() as database:
                 key = insert_one(database.batches, doc)
                 logger.info(
-                    '"action": "insert", "database": "%s", "collection": "%s"',
-                    database.name,
-                    database.collection.name,
+                    f'"action": "insert", '
+                    f'"database": "{database.name}", '
+                    f'"collection": "{database.collection.name}"'
                 )
             yield batch
 
         key, doc = batch.as_update_doc()
         with self.open_mongo() as database:
             update_one(database.batches, key, doc)
             logger.info(
-                '"action": "update", "database": "%s", "collection": "%s"',
-                database.name,
-                database.collection.name,
+                f'"action": "update", '
+                f'"database": "{database.name}", '
+                f'"collection": "{database.collection.name}"'
             )
 
     def store_evidence(self, batch: Batch, *args, **kwargs) -> None:
@@ -157,11 +140,10 @@ def store_evidence(self, batch: Batch, *args, **kwargs) -> None:
                 # TODO: Better exception
             df.drop(columns=["batch_id"], inplace=True)
             logger.info(
-                '"action": "insert_many", "database": "%s", \
-                    "collection": "%s", "count": %s',
-                database.name,
-                database.collection.name,
-                len(df.index),
+                f'"action": "insert_many", '
+                f'"database": "{database.name}", '
+                f'"collection": "{database.collection.name}", '
+                f'"count": {len(df.index)}'
             )
 
 

src/dsdk/mssql.py

@@ -5,38 +5,22 @@
 
 from abc import ABC
 from contextlib import contextmanager
-from logging import (
-    INFO,
-    Logger,
-    LoggerAdapter,
-    NullHandler,
-    basicConfig,
-    getLogger,
-)
+from logging import INFO
 from typing import TYPE_CHECKING, Generator, Optional, cast
 
 from configargparse import ArgParser as ArgumentParser
 
 from .service import Service
+from .utils import get_logger
+
+logger = get_logger(__name__, INFO)
 
 try:
     # Since not everyone will use mssql
     from sqlalchemy import create_engine
 except ImportError:
     create_engine = None
 
-# TODO Add import calling function from parent application
-EXTRA = {"callingfunc": ""}
-logger = getLogger(__name__)
-FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \
-    "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \
-        %(message)s}'
-basicConfig(format=FORMAT)
-logger.setLevel(INFO)
-# Add extra kwargs to message format
-logger.addHandler(NullHandler())
-logger = cast(Logger, LoggerAdapter(logger, EXTRA))
-
 
 if TYPE_CHECKING:
     BaseMixin = Service

src/dsdk/service.py

@@ -6,31 +6,16 @@
 from collections import OrderedDict
 from contextlib import contextmanager
 from datetime import datetime, timezone
-from logging import (
-    INFO,
-    Logger,
-    LoggerAdapter,
-    NullHandler,
-    basicConfig,
-    getLogger,
-)
+from logging import INFO
 from sys import argv as sys_argv
 from typing import Any, Dict, Generator, Optional, Sequence, Tuple, cast
 
 from configargparse import ArgParser as ArgumentParser
 from configargparse import Namespace
 
-# TODO Add import calling function from parent application
-EXTRA = {"callingfunc": ""}
-logger = getLogger(__name__)
-FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \
-    "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \
-        %(message)s}'
-basicConfig(format=FORMAT)
-logger.setLevel(INFO)
-# Add extra kwargs to message format
-logger.addHandler(NullHandler())
-logger = cast(Logger, LoggerAdapter(logger, EXTRA))
+from .utils import get_logger
+
+logger = get_logger(__name__, INFO)
 
 
 class Interval:  # pylint: disable=too-few-public-methods
@@ -176,7 +161,7 @@ def open_batch(  # pylint: disable=no-self-use,unused-argument
         record = Interval(on=datetime.now(timezone.utc), end=None)
         yield Batch(key, record)
         record.end = datetime.now(timezone.utc)
-        logger.info('"key": "%s"', key)
+        logger.info(f'"action": "open_batch", ' f'"key": "{key}"')
 
     def store_evidence(  # pylint: disable=no-self-use,unused-argument
         self, batch: Batch, *args, **kwargs
@@ -185,7 +170,11 @@ def store_evidence(  # pylint: disable=no-self-use,unused-argument
         while args:
             key, df, *args = args  # type: ignore
             batch.evidence[key] = df
-        logger.info('"key": "%s", "count": %s', key, len(batch.evidence))
+        logger.info(
+            f'"action": "store_evidence", '
+            f'"key": "{key}", '
+            f'"count": {len(batch.evidence)}'
+        )
 
 
 class Task:  # pylint: disable=too-few-public-methods

src/dsdk/utils.py

@@ -6,17 +6,54 @@
 from functools import wraps
 from json import dump as json_dump
 from json import load as json_load
-from logging import NullHandler, getLogger
+from logging import INFO, Formatter, LoggerAdapter, StreamHandler, getLogger
 from pickle import dump as pickle_dump
 from pickle import load as pickle_load
+from sys import stdout
 from time import sleep as default_sleep
 from typing import Any, Callable, Dict, List, Optional, Sequence
 
 from pandas import DataFrame
 from pandas import concat as pd_concat
 
-logger = getLogger(__name__)
-logger.addHandler(NullHandler())
+
+def get_logger(name, level=INFO):
+    """Get logger.
+
+    Actual handlers are typically set by the application.
+    Libraries (like DSDK) typically use a NullHandler, so that the application
+        logger configuration is used.
+
+    Use this function to hide the logger implementation/config for now.
+    Show that the conventions demonstrated here work for the applications.
+    """
+    # TODO Pass calling function from parent application
+    defaults = {"callingfunc": ""}
+    formatter_string = " - ".join(
+        (
+            "%(asctime)-15s",
+            "%(name)s",
+            "%(levelname)s",
+            ", ".join(
+                (
+                    '{"callingfunc": "%(callingfunc)s"',
+                    '"module": "%(module)s"',
+                    '"function": "%(funcName)s"',
+                    "%(message)s}",
+                )
+            ),
+        )
+    )
+    handler = StreamHandler(stdout)
+    handler.setLevel(level)
+    handler.setFormatter(Formatter(formatter_string))
+    result = getLogger(name)
+    result.propagate = False
+    result.addHandler(handler)
+    return LoggerAdapter(result, defaults)
+
+
+logger = get_logger(__name__)
 
 
 def chunks(sequence: Sequence[Any], n: int):