From d50575351caba9a740d37ac7116137dd202af611 Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Tue, 12 May 2020 13:46:16 -0400 Subject: [PATCH 1/3] Extract logger config --- src/dsdk/model.py | 23 +++-------------------- src/dsdk/mongo.py | 23 +++-------------------- src/dsdk/mssql.py | 24 ++++-------------------- src/dsdk/service.py | 23 ++++------------------- src/dsdk/utils.py | 42 +++++++++++++++++++++++++++++++++++++++--- 5 files changed, 53 insertions(+), 82 deletions(-) diff --git a/src/dsdk/model.py b/src/dsdk/model.py index 49dbc4c..7d56849 100644 --- a/src/dsdk/model.py +++ b/src/dsdk/model.py @@ -4,32 +4,15 @@ from __future__ import annotations from abc import ABC -from logging import ( - INFO, - Logger, - LoggerAdapter, - NullHandler, - basicConfig, - getLogger, -) +from logging import INFO from typing import TYPE_CHECKING, Optional, cast from configargparse import ArgParser as ArgumentParser from .service import Model, Service -from .utils import load_pickle_file +from .utils import get_logger, load_pickle_file -# TODO Add import calling function from parent application -EXTRA = {"callingfunc": ""} -logger = getLogger(__name__) -FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ - "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \ - %(message)s}' -basicConfig(format=FORMAT) -logger.setLevel(INFO) -# Add extra kwargs to message format -logger.addHandler(NullHandler()) -logger = cast(Logger, LoggerAdapter(logger, EXTRA)) +logger = get_logger(__file__, INFO) if TYPE_CHECKING: diff --git a/src/dsdk/mongo.py b/src/dsdk/mongo.py index 2146c1a..fce97e2 100644 --- a/src/dsdk/mongo.py +++ b/src/dsdk/mongo.py @@ -5,14 +5,7 @@ from abc import ABC from contextlib import contextmanager -from logging import ( - INFO, - Logger, - LoggerAdapter, - NullHandler, - basicConfig, - getLogger, -) +from logging import INFO from typing import ( TYPE_CHECKING, Any, @@ -26,7 +19,7 @@ from configargparse import ArgParser as ArgumentParser from .service import Batch, Model, Service -from .utils import retry +from .utils import get_logger, retry try: # Since not everyone will use mongo @@ -41,17 +34,7 @@ Database = None AutoReconnect = None -# TODO Add import calling function from parent application -EXTRA = {"callingfunc": ""} -logger = getLogger(__name__) -FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ - "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \ - %(message)s}' -basicConfig(format=FORMAT) -logger.setLevel(INFO) -# Add extra kwargs to message format -logger.addHandler(NullHandler()) -logger = cast(Logger, LoggerAdapter(logger, EXTRA)) +logger = get_logger(__name__, INFO) if TYPE_CHECKING: diff --git a/src/dsdk/mssql.py b/src/dsdk/mssql.py index 3ecbaa8..c908c36 100644 --- a/src/dsdk/mssql.py +++ b/src/dsdk/mssql.py @@ -5,19 +5,15 @@ from abc import ABC from contextlib import contextmanager -from logging import ( - INFO, - Logger, - LoggerAdapter, - NullHandler, - basicConfig, - getLogger, -) +from logging import INFO from typing import TYPE_CHECKING, Generator, Optional, cast from configargparse import ArgParser as ArgumentParser from .service import Service +from .utils import get_logger + +logger = get_logger(__file__, INFO) try: # Since not everyone will use mssql @@ -25,18 +21,6 @@ except ImportError: create_engine = None -# TODO Add import calling function from parent application -EXTRA = {"callingfunc": ""} -logger = getLogger(__name__) -FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ - "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \ - %(message)s}' -basicConfig(format=FORMAT) -logger.setLevel(INFO) -# Add extra kwargs to message format -logger.addHandler(NullHandler()) -logger = cast(Logger, LoggerAdapter(logger, EXTRA)) - if TYPE_CHECKING: BaseMixin = Service diff --git a/src/dsdk/service.py b/src/dsdk/service.py index 907bde3..10e2d08 100644 --- a/src/dsdk/service.py +++ b/src/dsdk/service.py @@ -6,31 +6,16 @@ from collections import OrderedDict from contextlib import contextmanager from datetime import datetime, timezone -from logging import ( - INFO, - Logger, - LoggerAdapter, - NullHandler, - basicConfig, - getLogger, -) +from logging import INFO from sys import argv as sys_argv from typing import Any, Dict, Generator, Optional, Sequence, Tuple, cast from configargparse import ArgParser as ArgumentParser from configargparse import Namespace -# TODO Add import calling function from parent application -EXTRA = {"callingfunc": ""} -logger = getLogger(__name__) -FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ - "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \ - %(message)s}' -basicConfig(format=FORMAT) -logger.setLevel(INFO) -# Add extra kwargs to message format -logger.addHandler(NullHandler()) -logger = cast(Logger, LoggerAdapter(logger, EXTRA)) +from .utils import get_logger + +logger = get_logger(__file__, INFO) class Interval: # pylint: disable=too-few-public-methods diff --git a/src/dsdk/utils.py b/src/dsdk/utils.py index 50ff444..2ecd8a9 100644 --- a/src/dsdk/utils.py +++ b/src/dsdk/utils.py @@ -6,17 +6,53 @@ from functools import wraps from json import dump as json_dump from json import load as json_load -from logging import NullHandler, getLogger +from logging import INFO, Formatter, LoggerAdapter, StreamHandler, getLogger from pickle import dump as pickle_dump from pickle import load as pickle_load +from sys import stdout from time import sleep as default_sleep from typing import Any, Callable, Dict, List, Optional, Sequence from pandas import DataFrame from pandas import concat as pd_concat -logger = getLogger(__name__) -logger.addHandler(NullHandler()) + +def get_logger(name, level=INFO): + """Get logger. + + Actual handlers are typically set by the application. + Libraries (like DSDK) typically use a NullHandler, so that the application + logger configuration is used. + + Use this function to hide the logger implementation/config for now. + Show that the conventions demonstrated here work for the applications. + """ + # TODO Pass calling function from parent application + defaults = {"callingfunc": ""} + formatter_string = " - ".join( + ( + "%(asctime)-15s", + "%(name)s", + "%(levelname)s", + ", ".join( + ( + '{"callingfunc": "%(callingfunc)s"', + '"module": "%(module)s"', + '"function": "%(funcName)s"', + "%(message)s}", + ) + ), + ) + ) + handler = StreamHandler(stdout) + handler.setLevel(level) + handler.setFormatter(Formatter(formatter_string)) + result = getLogger(name) + result.addHandler(handler) + return LoggerAdapter(result, defaults) + + +logger = get_logger(__file__) def chunks(sequence: Sequence[Any], n: int): From d75716ba2f5a4fb42402cf9b427984efb777acd6 Mon Sep 17 00:00:00 2001 From: Darryl Mendillo <1238655+darrylmendillo@users.noreply.github.com> Date: Tue, 12 May 2020 17:29:44 -0400 Subject: [PATCH 2/3] Cleaned up logs with f strings, deduped logs --- src/dsdk/model.py | 2 +- src/dsdk/mongo.py | 21 ++++++++++----------- src/dsdk/mssql.py | 2 +- src/dsdk/service.py | 10 +++++++--- src/dsdk/utils.py | 3 ++- 5 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/dsdk/model.py b/src/dsdk/model.py index 7d56849..33641d9 100644 --- a/src/dsdk/model.py +++ b/src/dsdk/model.py @@ -12,7 +12,7 @@ from .service import Model, Service from .utils import get_logger, load_pickle_file -logger = get_logger(__file__, INFO) +logger = get_logger(__name__, INFO) if TYPE_CHECKING: diff --git a/src/dsdk/mongo.py b/src/dsdk/mongo.py index fce97e2..52c1a52 100644 --- a/src/dsdk/mongo.py +++ b/src/dsdk/mongo.py @@ -102,9 +102,9 @@ def open_batch( with self.open_mongo() as database: key = insert_one(database.batches, doc) logger.info( - '"action": "insert", "database": "%s", "collection": "%s"', - database.name, - database.collection.name, + f'"action": "insert", ' + f'"database": "{database.name}", ' + f'"collection": "{database.collection.name}"' ) yield batch @@ -112,9 +112,9 @@ def open_batch( with self.open_mongo() as database: update_one(database.batches, key, doc) logger.info( - '"action": "update", "database": "%s", "collection": "%s"', - database.name, - database.collection.name, + f'"action": "update", ' + f'"database": "{database.name}", ' + f'"collection": "{database.collection.name}"' ) def store_evidence(self, batch: Batch, *args, **kwargs) -> None: @@ -140,11 +140,10 @@ def store_evidence(self, batch: Batch, *args, **kwargs) -> None: # TODO: Better exception df.drop(columns=["batch_id"], inplace=True) logger.info( - '"action": "insert_many", "database": "%s", \ - "collection": "%s", "count": %s', - database.name, - database.collection.name, - len(df.index), + f'"action": "insert_many", ' + f'"database": "{database.name}", ' + f'"collection": "{database.collection.name}", ' + f'"count": {len(df.index)}' ) diff --git a/src/dsdk/mssql.py b/src/dsdk/mssql.py index c908c36..02bb709 100644 --- a/src/dsdk/mssql.py +++ b/src/dsdk/mssql.py @@ -13,7 +13,7 @@ from .service import Service from .utils import get_logger -logger = get_logger(__file__, INFO) +logger = get_logger(__name__, INFO) try: # Since not everyone will use mssql diff --git a/src/dsdk/service.py b/src/dsdk/service.py index 10e2d08..41fa8f9 100644 --- a/src/dsdk/service.py +++ b/src/dsdk/service.py @@ -15,7 +15,7 @@ from .utils import get_logger -logger = get_logger(__file__, INFO) +logger = get_logger(__name__, INFO) class Interval: # pylint: disable=too-few-public-methods @@ -161,7 +161,7 @@ def open_batch( # pylint: disable=no-self-use,unused-argument record = Interval(on=datetime.now(timezone.utc), end=None) yield Batch(key, record) record.end = datetime.now(timezone.utc) - logger.info('"key": "%s"', key) + logger.info(f'"action": "open_batch", ' f'"key": "{key}"') def store_evidence( # pylint: disable=no-self-use,unused-argument self, batch: Batch, *args, **kwargs @@ -170,7 +170,11 @@ def store_evidence( # pylint: disable=no-self-use,unused-argument while args: key, df, *args = args # type: ignore batch.evidence[key] = df - logger.info('"key": "%s", "count": %s', key, len(batch.evidence)) + logger.info( + f'"action": "store_evidence", ' + f'"key": "{key}", ' + f'"count": "{len(batch.evidence)}"' + ) class Task: # pylint: disable=too-few-public-methods diff --git a/src/dsdk/utils.py b/src/dsdk/utils.py index 2ecd8a9..e59472a 100644 --- a/src/dsdk/utils.py +++ b/src/dsdk/utils.py @@ -48,11 +48,12 @@ def get_logger(name, level=INFO): handler.setLevel(level) handler.setFormatter(Formatter(formatter_string)) result = getLogger(name) + result.propagate = False result.addHandler(handler) return LoggerAdapter(result, defaults) -logger = get_logger(__file__) +logger = get_logger(__name__) def chunks(sequence: Sequence[Any], n: int): From 1929e06f17f65556da606ac04fd5c33913ca8188 Mon Sep 17 00:00:00 2001 From: Darryl Mendillo <1238655+darrylmendillo@users.noreply.github.com> Date: Tue, 12 May 2020 17:41:48 -0400 Subject: [PATCH 3/3] Changed service count to int --- src/dsdk/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dsdk/service.py b/src/dsdk/service.py index 41fa8f9..01a8907 100644 --- a/src/dsdk/service.py +++ b/src/dsdk/service.py @@ -173,7 +173,7 @@ def store_evidence( # pylint: disable=no-self-use,unused-argument logger.info( f'"action": "store_evidence", ' f'"key": "{key}", ' - f'"count": "{len(batch.evidence)}"' + f'"count": {len(batch.evidence)}' )