From d9372fa2b299c65b0c93c2aabd2311ab9c0c499e Mon Sep 17 00:00:00 2001 From: Darryl Mendillo <1238655+darrylmendillo@users.noreply.github.com> Date: Thu, 7 May 2020 16:02:06 -0400 Subject: [PATCH 01/14] Add mongo insert and update to EvidenceMixin --- .gitignore | 1 + src/dsdk/mongo.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3abdcc3..ab39d9e 100644 --- a/.gitignore +++ b/.gitignore @@ -58,6 +58,7 @@ docs/_build *~ .*.sw[po] .build +.vscode .ve .env .cache diff --git a/src/dsdk/mongo.py b/src/dsdk/mongo.py index 05653ca..ad81883 100644 --- a/src/dsdk/mongo.py +++ b/src/dsdk/mongo.py @@ -103,12 +103,13 @@ def open_batch( doc = batch.as_insert_doc(model) # <- model dependency with self.open_mongo() as database: insert_one(database.batches, doc) - + logger.info('{"mongo": "insert"}') yield batch key, doc = batch.as_update_doc() with self.open_mongo() as database: update_one(database.batches, key, doc) + logger.info('{"mongo": "update"}') def store_evidence(self, batch: Batch, *args, **kwargs) -> None: """Store Evidence.""" From 11a178bcf1afdef51ccc1d974321718994c024c7 Mon Sep 17 00:00:00 2001 From: Darryl Mendillo <1238655+darrylmendillo@users.noreply.github.com> Date: Fri, 8 May 2020 14:32:41 -0400 Subject: [PATCH 02/14] Added mongo update, insert logs --- src/dsdk/mongo.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/src/dsdk/mongo.py b/src/dsdk/mongo.py index ad81883..81dd9f9 100644 --- a/src/dsdk/mongo.py +++ b/src/dsdk/mongo.py @@ -102,14 +102,22 @@ def open_batch( with super().open_batch(key) as batch: doc = batch.as_insert_doc(model) # <- model dependency with self.open_mongo() as database: - insert_one(database.batches, doc) - logger.info('{"mongo": "insert"}') + key = insert_one(database.batches, doc) + logger.info( + '{"mongo.insert": {"database": "%s", "collection": "%s"}}', + database.name, + database.collection.name, + ) yield batch key, doc = batch.as_update_doc() with self.open_mongo() as database: update_one(database.batches, key, doc) - logger.info('{"mongo": "update"}') + logger.info( + '{"mongo.update": {"database": "%s", "collection": "%s"}}', + database.name, + database.collection.name, + ) def store_evidence(self, batch: Batch, *args, **kwargs) -> None: """Store Evidence.""" @@ -126,8 +134,21 @@ def store_evidence(self, batch: Batch, *args, **kwargs) -> None: result = insert_many(database[key], docs) assert columns.shape[0] == len( result.inserted_ids - ) # TODO: Better exception + ), logger.error( + '{"mongo.insert_many": {"database": "%s", "collection": "%s", \ + "message": "columns.shape[0] != len(results.inserted_ids)"}}', + database.name, + database.collection.name, + ) + + # TODO: Better exception df.drop(columns=["batch_id"], inplace=True) + logger.info( + '{"mongo.insert_many": {"database": "%s", "collection": "%s", "count": %s"}}', + database.name, + database.collection.name, + len(df.index), + ) @contextmanager From 45870b0dba521c0319d80f52e5ef234aecf5cf1c Mon Sep 17 00:00:00 2001 From: Darryl Mendillo <1238655+darrylmendillo@users.noreply.github.com> Date: Fri, 8 May 2020 16:55:56 -0400 Subject: [PATCH 03/14] Added logs to service and mssql --- src/dsdk/mssql.py | 1 + src/dsdk/service.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/dsdk/mssql.py b/src/dsdk/mssql.py index 86d03b1..1ad0cee 100644 --- a/src/dsdk/mssql.py +++ b/src/dsdk/mssql.py @@ -67,3 +67,4 @@ def open_mssql(self) -> Generator: """Open mssql.""" with self._mssql.connect() as con: yield con + logger.info('{"mssql" "connect"}') \ No newline at end of file diff --git a/src/dsdk/service.py b/src/dsdk/service.py index 72ba818..eaaf20d 100644 --- a/src/dsdk/service.py +++ b/src/dsdk/service.py @@ -123,6 +123,7 @@ def __call__(self) -> Batch: with self.open_batch() as batch: for task in self.pipeline: task(batch, self) + logger.info('{"check": "pipeline", "%s"}',','.join(map(str, self.pipeline))) return batch def check(self) -> None: @@ -154,6 +155,7 @@ def open_batch( # pylint: disable=no-self-use,unused-argument record = Interval(on=datetime.now(timezone.utc), end=None) yield Batch(key, record) record.end = datetime.now(timezone.utc) + logger.info('{"batch.open": "batch", "key": "%s"}',key) def store_evidence( # pylint: disable=no-self-use,unused-argument self, batch: Batch, *args, **kwargs @@ -162,7 +164,7 @@ def store_evidence( # pylint: disable=no-self-use,unused-argument while args: key, df, *args = args # type: ignore batch.evidence[key] = df - + logger.info('{"store.evidence": "batch", "key": "%s", "count": "%s"}',key, len(batch.evidence)) class Task: # pylint: disable=too-few-public-methods """Task.""" From 2de668ff55b9d363573d0a9a38c2084d811c7875 Mon Sep 17 00:00:00 2001 From: Darryl Mendillo <1238655+darrylmendillo@users.noreply.github.com> Date: Mon, 11 May 2020 15:08:49 -0400 Subject: [PATCH 04/14] Changed format to include module and function --- src/dsdk/model.py | 9 ++++++++- src/dsdk/mongo.py | 20 +++++++++++++------- src/dsdk/mssql.py | 12 +++++++++--- src/dsdk/service.py | 15 +++++++++++---- 4 files changed, 41 insertions(+), 15 deletions(-) diff --git a/src/dsdk/model.py b/src/dsdk/model.py index c2f519a..bee1152 100644 --- a/src/dsdk/model.py +++ b/src/dsdk/model.py @@ -4,7 +4,7 @@ from __future__ import annotations from abc import ABC -from logging import NullHandler, getLogger +from logging import NullHandler, getLogger, basicConfig, LoggerAdapter, INFO from typing import TYPE_CHECKING, Optional, cast from configargparse import ArgParser as ArgumentParser @@ -12,8 +12,15 @@ from .service import Model, Service from .utils import load_pickle_file +# TODO Add import calling function from parent application +extra = {'callingfunc':''} logger = getLogger(__name__) +FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s {"callingfunc": "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", %(message)s}' +basicConfig(format=FORMAT) +logger.setLevel(INFO) +# Add extra kwargs to message format logger.addHandler(NullHandler()) +logger = LoggerAdapter(logger, extra) if TYPE_CHECKING: diff --git a/src/dsdk/mongo.py b/src/dsdk/mongo.py index 81dd9f9..e4da393 100644 --- a/src/dsdk/mongo.py +++ b/src/dsdk/mongo.py @@ -5,7 +5,7 @@ from abc import ABC from contextlib import contextmanager -from logging import NullHandler, getLogger +from logging import NullHandler, getLogger, basicConfig, LoggerAdapter, INFO from typing import ( TYPE_CHECKING, Any, @@ -34,9 +34,15 @@ Database = None AutoReconnect = None - +# TODO Add import calling function from parent application +extra = {'callingfunc':''} logger = getLogger(__name__) +FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s {"callingfunc": "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", %(message)s}' +basicConfig(format=FORMAT) +logger.setLevel(INFO) +# Add extra kwargs to message format logger.addHandler(NullHandler()) +logger = LoggerAdapter(logger, extra) if TYPE_CHECKING: @@ -104,7 +110,7 @@ def open_batch( with self.open_mongo() as database: key = insert_one(database.batches, doc) logger.info( - '{"mongo.insert": {"database": "%s", "collection": "%s"}}', + '"action": "insert": "database": "%s", "collection": "%s"', database.name, database.collection.name, ) @@ -114,7 +120,7 @@ def open_batch( with self.open_mongo() as database: update_one(database.batches, key, doc) logger.info( - '{"mongo.update": {"database": "%s", "collection": "%s"}}', + '"action": "update", "database": "%s", "collection": "%s"', database.name, database.collection.name, ) @@ -135,8 +141,8 @@ def store_evidence(self, batch: Batch, *args, **kwargs) -> None: assert columns.shape[0] == len( result.inserted_ids ), logger.error( - '{"mongo.insert_many": {"database": "%s", "collection": "%s", \ - "message": "columns.shape[0] != len(results.inserted_ids)"}}', + '"action" "insert_many", "database": "%s", "collection": "%s", \ + "message": "columns.shape[0] != len(results.inserted_ids)"', database.name, database.collection.name, ) @@ -144,7 +150,7 @@ def store_evidence(self, batch: Batch, *args, **kwargs) -> None: # TODO: Better exception df.drop(columns=["batch_id"], inplace=True) logger.info( - '{"mongo.insert_many": {"database": "%s", "collection": "%s", "count": %s"}}', + '"action": "insert_many", "database": "%s", "collection": "%s", "count": %s"', database.name, database.collection.name, len(df.index), diff --git a/src/dsdk/mssql.py b/src/dsdk/mssql.py index 1ad0cee..7c0c0a7 100644 --- a/src/dsdk/mssql.py +++ b/src/dsdk/mssql.py @@ -5,7 +5,7 @@ from abc import ABC from contextlib import contextmanager -from logging import NullHandler, getLogger +from logging import NullHandler, getLogger, basicConfig, LoggerAdapter, INFO from typing import TYPE_CHECKING, Generator, Optional, cast from configargparse import ArgParser as ArgumentParser @@ -18,9 +18,15 @@ except ImportError: create_engine = None - +# TODO Add import calling function from parent application +extra = {'callingfunc':''} logger = getLogger(__name__) +FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s {"callingfunc": "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", %(message)s}' +basicConfig(format=FORMAT) +logger.setLevel(INFO) +# Add extra kwargs to message format logger.addHandler(NullHandler()) +logger = LoggerAdapter(logger, extra) if TYPE_CHECKING: @@ -67,4 +73,4 @@ def open_mssql(self) -> Generator: """Open mssql.""" with self._mssql.connect() as con: yield con - logger.info('{"mssql" "connect"}') \ No newline at end of file + logger.info('"action": "connect"') \ No newline at end of file diff --git a/src/dsdk/service.py b/src/dsdk/service.py index eaaf20d..fba699e 100644 --- a/src/dsdk/service.py +++ b/src/dsdk/service.py @@ -6,15 +6,22 @@ from collections import OrderedDict from contextlib import contextmanager from datetime import datetime, timezone -from logging import NullHandler, getLogger +from logging import NullHandler, getLogger, basicConfig, LoggerAdapter, INFO from sys import argv as sys_argv from typing import Any, Dict, Generator, Optional, Sequence, Tuple, cast from configargparse import ArgParser as ArgumentParser from configargparse import Namespace +# TODO Add import calling function from parent application +extra = {'callingfunc':''} logger = getLogger(__name__) +FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s {"callingfunc": "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", %(message)s}' +basicConfig(format=FORMAT) +logger.setLevel(INFO) +# Add extra kwargs to message format logger.addHandler(NullHandler()) +logger = LoggerAdapter(logger, extra) class Interval: # pylint: disable=too-few-public-methods @@ -123,7 +130,7 @@ def __call__(self) -> Batch: with self.open_batch() as batch: for task in self.pipeline: task(batch, self) - logger.info('{"check": "pipeline", "%s"}',','.join(map(str, self.pipeline))) + logger.info('"%s"',', '.join(map(lambda s: str(s).split(' ')[0][1:], self.pipeline))) return batch def check(self) -> None: @@ -155,7 +162,7 @@ def open_batch( # pylint: disable=no-self-use,unused-argument record = Interval(on=datetime.now(timezone.utc), end=None) yield Batch(key, record) record.end = datetime.now(timezone.utc) - logger.info('{"batch.open": "batch", "key": "%s"}',key) + logger.info('"key": "%s"',key) def store_evidence( # pylint: disable=no-self-use,unused-argument self, batch: Batch, *args, **kwargs @@ -164,7 +171,7 @@ def store_evidence( # pylint: disable=no-self-use,unused-argument while args: key, df, *args = args # type: ignore batch.evidence[key] = df - logger.info('{"store.evidence": "batch", "key": "%s", "count": "%s"}',key, len(batch.evidence)) + logger.info('"key": "%s", "count": "%s"}',key, len(batch.evidence)) class Task: # pylint: disable=too-few-public-methods """Task.""" From ff301c40b4a73bb3fdb080dc333be337a2785585 Mon Sep 17 00:00:00 2001 From: Darryl Mendillo <1238655+darrylmendillo@users.noreply.github.com> Date: Mon, 11 May 2020 15:24:53 -0400 Subject: [PATCH 05/14] Fized log formatting syntax --- src/dsdk/model.py | 6 +++--- src/dsdk/mongo.py | 6 +++--- src/dsdk/mssql.py | 8 ++++---- src/dsdk/service.py | 18 ++++++++++++------ 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/src/dsdk/model.py b/src/dsdk/model.py index bee1152..67a0963 100644 --- a/src/dsdk/model.py +++ b/src/dsdk/model.py @@ -4,7 +4,7 @@ from __future__ import annotations from abc import ABC -from logging import NullHandler, getLogger, basicConfig, LoggerAdapter, INFO +from logging import INFO, LoggerAdapter, NullHandler, basicConfig, getLogger from typing import TYPE_CHECKING, Optional, cast from configargparse import ArgParser as ArgumentParser @@ -13,9 +13,9 @@ from .utils import load_pickle_file # TODO Add import calling function from parent application -extra = {'callingfunc':''} +extra = {"callingfunc": ""} logger = getLogger(__name__) -FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s {"callingfunc": "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", %(message)s}' +FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", %(message)s}' basicConfig(format=FORMAT) logger.setLevel(INFO) # Add extra kwargs to message format diff --git a/src/dsdk/mongo.py b/src/dsdk/mongo.py index e4da393..3c419f5 100644 --- a/src/dsdk/mongo.py +++ b/src/dsdk/mongo.py @@ -5,7 +5,7 @@ from abc import ABC from contextlib import contextmanager -from logging import NullHandler, getLogger, basicConfig, LoggerAdapter, INFO +from logging import INFO, LoggerAdapter, NullHandler, basicConfig, getLogger from typing import ( TYPE_CHECKING, Any, @@ -35,9 +35,9 @@ AutoReconnect = None # TODO Add import calling function from parent application -extra = {'callingfunc':''} +extra = {"callingfunc": ""} logger = getLogger(__name__) -FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s {"callingfunc": "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", %(message)s}' +FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", %(message)s}' basicConfig(format=FORMAT) logger.setLevel(INFO) # Add extra kwargs to message format diff --git a/src/dsdk/mssql.py b/src/dsdk/mssql.py index 7c0c0a7..953925f 100644 --- a/src/dsdk/mssql.py +++ b/src/dsdk/mssql.py @@ -5,7 +5,7 @@ from abc import ABC from contextlib import contextmanager -from logging import NullHandler, getLogger, basicConfig, LoggerAdapter, INFO +from logging import INFO, LoggerAdapter, NullHandler, basicConfig, getLogger from typing import TYPE_CHECKING, Generator, Optional, cast from configargparse import ArgParser as ArgumentParser @@ -19,9 +19,9 @@ create_engine = None # TODO Add import calling function from parent application -extra = {'callingfunc':''} +extra = {"callingfunc": ""} logger = getLogger(__name__) -FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s {"callingfunc": "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", %(message)s}' +FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", %(message)s}' basicConfig(format=FORMAT) logger.setLevel(INFO) # Add extra kwargs to message format @@ -73,4 +73,4 @@ def open_mssql(self) -> Generator: """Open mssql.""" with self._mssql.connect() as con: yield con - logger.info('"action": "connect"') \ No newline at end of file + logger.info('"action": "connect"') diff --git a/src/dsdk/service.py b/src/dsdk/service.py index fba699e..ab073b4 100644 --- a/src/dsdk/service.py +++ b/src/dsdk/service.py @@ -6,7 +6,7 @@ from collections import OrderedDict from contextlib import contextmanager from datetime import datetime, timezone -from logging import NullHandler, getLogger, basicConfig, LoggerAdapter, INFO +from logging import INFO, LoggerAdapter, NullHandler, basicConfig, getLogger from sys import argv as sys_argv from typing import Any, Dict, Generator, Optional, Sequence, Tuple, cast @@ -14,9 +14,9 @@ from configargparse import Namespace # TODO Add import calling function from parent application -extra = {'callingfunc':''} +extra = {"callingfunc": ""} logger = getLogger(__name__) -FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s {"callingfunc": "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", %(message)s}' +FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", %(message)s}' basicConfig(format=FORMAT) logger.setLevel(INFO) # Add extra kwargs to message format @@ -130,7 +130,12 @@ def __call__(self) -> Batch: with self.open_batch() as batch: for task in self.pipeline: task(batch, self) - logger.info('"%s"',', '.join(map(lambda s: str(s).split(' ')[0][1:], self.pipeline))) + logger.info( + '"%s"', + ", ".join( + map(lambda s: str(s).split(" ")[0][1:], self.pipeline) + ), + ) return batch def check(self) -> None: @@ -162,7 +167,7 @@ def open_batch( # pylint: disable=no-self-use,unused-argument record = Interval(on=datetime.now(timezone.utc), end=None) yield Batch(key, record) record.end = datetime.now(timezone.utc) - logger.info('"key": "%s"',key) + logger.info('"key": "%s"', key) def store_evidence( # pylint: disable=no-self-use,unused-argument self, batch: Batch, *args, **kwargs @@ -171,7 +176,8 @@ def store_evidence( # pylint: disable=no-self-use,unused-argument while args: key, df, *args = args # type: ignore batch.evidence[key] = df - logger.info('"key": "%s", "count": "%s"}',key, len(batch.evidence)) + logger.info('"key": "%s", "count": "%s"}', key, len(batch.evidence)) + class Task: # pylint: disable=too-few-public-methods """Task.""" From 6ea348b905c9759c2ee2970328d9daadd6d5b3c5 Mon Sep 17 00:00:00 2001 From: Darryl Mendillo <1238655+darrylmendillo@users.noreply.github.com> Date: Mon, 11 May 2020 16:29:44 -0400 Subject: [PATCH 06/14] Reformat for linters --- src/dsdk/model.py | 17 +++++++++++++---- src/dsdk/mongo.py | 32 ++++++++++++++++++++------------ src/dsdk/mssql.py | 17 +++++++++++++---- src/dsdk/service.py | 17 +++++++++++++---- 4 files changed, 59 insertions(+), 24 deletions(-) diff --git a/src/dsdk/model.py b/src/dsdk/model.py index 67a0963..2c1fcec 100644 --- a/src/dsdk/model.py +++ b/src/dsdk/model.py @@ -4,7 +4,14 @@ from __future__ import annotations from abc import ABC -from logging import INFO, LoggerAdapter, NullHandler, basicConfig, getLogger +from logging import ( + INFO, + Logger, + LoggerAdapter, + NullHandler, + basicConfig, + getLogger, +) from typing import TYPE_CHECKING, Optional, cast from configargparse import ArgParser as ArgumentParser @@ -13,14 +20,16 @@ from .utils import load_pickle_file # TODO Add import calling function from parent application -extra = {"callingfunc": ""} +EXTRA = {"callingfunc": ""} logger = getLogger(__name__) -FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", %(message)s}' +FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ + "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \ + %(message)s}' basicConfig(format=FORMAT) logger.setLevel(INFO) # Add extra kwargs to message format logger.addHandler(NullHandler()) -logger = LoggerAdapter(logger, extra) +logger = cast(Logger, LoggerAdapter(logger, EXTRA)) if TYPE_CHECKING: diff --git a/src/dsdk/mongo.py b/src/dsdk/mongo.py index 3c419f5..1e649a0 100644 --- a/src/dsdk/mongo.py +++ b/src/dsdk/mongo.py @@ -5,7 +5,14 @@ from abc import ABC from contextlib import contextmanager -from logging import INFO, LoggerAdapter, NullHandler, basicConfig, getLogger +from logging import ( + INFO, + Logger, + LoggerAdapter, + NullHandler, + basicConfig, + getLogger, +) from typing import ( TYPE_CHECKING, Any, @@ -35,14 +42,16 @@ AutoReconnect = None # TODO Add import calling function from parent application -extra = {"callingfunc": ""} +EXTRA = {"callingfunc": ""} logger = getLogger(__name__) -FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", %(message)s}' +FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ + "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \ + %(message)s}' basicConfig(format=FORMAT) logger.setLevel(INFO) # Add extra kwargs to message format logger.addHandler(NullHandler()) -logger = LoggerAdapter(logger, extra) +logger = cast(Logger, LoggerAdapter(logger, EXTRA)) if TYPE_CHECKING: @@ -138,19 +147,18 @@ def store_evidence(self, batch: Batch, *args, **kwargs) -> None: docs = columns.to_dict(orient="records") with self.open_mongo() as database: result = insert_many(database[key], docs) - assert columns.shape[0] == len( - result.inserted_ids - ), logger.error( - '"action" "insert_many", "database": "%s", "collection": "%s", \ - "message": "columns.shape[0] != len(results.inserted_ids)"', - database.name, - database.collection.name, + assert columns.shape[0] == len(result.inserted_ids), ( + '"action" "insert_many", "database": "%s", "collection": \ + "%s", "message": "columns.shape[0] != \ + len(results.inserted_ids)"' + % (database.name, database.collection.name) ) # TODO: Better exception df.drop(columns=["batch_id"], inplace=True) logger.info( - '"action": "insert_many", "database": "%s", "collection": "%s", "count": %s"', + '"action": "insert_many", "database": "%s", \ + "collection": "%s", "count": %s"', database.name, database.collection.name, len(df.index), diff --git a/src/dsdk/mssql.py b/src/dsdk/mssql.py index 953925f..3ecbaa8 100644 --- a/src/dsdk/mssql.py +++ b/src/dsdk/mssql.py @@ -5,7 +5,14 @@ from abc import ABC from contextlib import contextmanager -from logging import INFO, LoggerAdapter, NullHandler, basicConfig, getLogger +from logging import ( + INFO, + Logger, + LoggerAdapter, + NullHandler, + basicConfig, + getLogger, +) from typing import TYPE_CHECKING, Generator, Optional, cast from configargparse import ArgParser as ArgumentParser @@ -19,14 +26,16 @@ create_engine = None # TODO Add import calling function from parent application -extra = {"callingfunc": ""} +EXTRA = {"callingfunc": ""} logger = getLogger(__name__) -FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", %(message)s}' +FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ + "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \ + %(message)s}' basicConfig(format=FORMAT) logger.setLevel(INFO) # Add extra kwargs to message format logger.addHandler(NullHandler()) -logger = LoggerAdapter(logger, extra) +logger = cast(Logger, LoggerAdapter(logger, EXTRA)) if TYPE_CHECKING: diff --git a/src/dsdk/service.py b/src/dsdk/service.py index ab073b4..c2b689d 100644 --- a/src/dsdk/service.py +++ b/src/dsdk/service.py @@ -6,7 +6,14 @@ from collections import OrderedDict from contextlib import contextmanager from datetime import datetime, timezone -from logging import INFO, LoggerAdapter, NullHandler, basicConfig, getLogger +from logging import ( + INFO, + Logger, + LoggerAdapter, + NullHandler, + basicConfig, + getLogger, +) from sys import argv as sys_argv from typing import Any, Dict, Generator, Optional, Sequence, Tuple, cast @@ -14,14 +21,16 @@ from configargparse import Namespace # TODO Add import calling function from parent application -extra = {"callingfunc": ""} +EXTRA = {"callingfunc": ""} logger = getLogger(__name__) -FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", %(message)s}' +FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ + "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \ + %(message)s}' basicConfig(format=FORMAT) logger.setLevel(INFO) # Add extra kwargs to message format logger.addHandler(NullHandler()) -logger = LoggerAdapter(logger, extra) +logger = cast(Logger, LoggerAdapter(logger, EXTRA)) class Interval: # pylint: disable=too-few-public-methods From aef51d1af817a74abdd04698b944f526e4742335 Mon Sep 17 00:00:00 2001 From: Darryl Mendillo <1238655+darrylmendillo@users.noreply.github.com> Date: Tue, 12 May 2020 08:22:07 -0400 Subject: [PATCH 07/14] Fixed pipeline log --- src/dsdk/model.py | 4 ++-- src/dsdk/mongo.py | 2 +- src/dsdk/mssql.py | 2 +- src/dsdk/service.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/dsdk/model.py b/src/dsdk/model.py index 2c1fcec..f46b317 100644 --- a/src/dsdk/model.py +++ b/src/dsdk/model.py @@ -23,8 +23,8 @@ EXTRA = {"callingfunc": ""} logger = getLogger(__name__) FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ - "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \ - %(message)s}' + %(callingfunc)s, "module": %(module)s, "function": %(funcName)s, \ + %(message)s}' basicConfig(format=FORMAT) logger.setLevel(INFO) # Add extra kwargs to message format diff --git a/src/dsdk/mongo.py b/src/dsdk/mongo.py index 1e649a0..8d55b9c 100644 --- a/src/dsdk/mongo.py +++ b/src/dsdk/mongo.py @@ -45,7 +45,7 @@ EXTRA = {"callingfunc": ""} logger = getLogger(__name__) FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ - "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \ + %(callingfunc)s, "module": %(module)s, "function": %(funcName)s, \ %(message)s}' basicConfig(format=FORMAT) logger.setLevel(INFO) diff --git a/src/dsdk/mssql.py b/src/dsdk/mssql.py index 3ecbaa8..39075ef 100644 --- a/src/dsdk/mssql.py +++ b/src/dsdk/mssql.py @@ -29,7 +29,7 @@ EXTRA = {"callingfunc": ""} logger = getLogger(__name__) FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ - "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \ + %(callingfunc)s, "module": %(module)s, "function": %(funcName)s, \ %(message)s}' basicConfig(format=FORMAT) logger.setLevel(INFO) diff --git a/src/dsdk/service.py b/src/dsdk/service.py index c2b689d..ad5d0a9 100644 --- a/src/dsdk/service.py +++ b/src/dsdk/service.py @@ -24,7 +24,7 @@ EXTRA = {"callingfunc": ""} logger = getLogger(__name__) FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ - "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \ + %(callingfunc)s, "module": %(module)s, "function": %(funcName)s, \ %(message)s}' basicConfig(format=FORMAT) logger.setLevel(INFO) @@ -140,7 +140,7 @@ def __call__(self) -> Batch: for task in self.pipeline: task(batch, self) logger.info( - '"%s"', + '"pipeline:" %s', ", ".join( map(lambda s: str(s).split(" ")[0][1:], self.pipeline) ), From 0701a57e898b9c44e7663336457441d57b44f7ef Mon Sep 17 00:00:00 2001 From: Darryl Mendillo <1238655+darrylmendillo@users.noreply.github.com> Date: Tue, 12 May 2020 08:27:03 -0400 Subject: [PATCH 08/14] Added temp calling function --- src/dsdk/model.py | 2 +- src/dsdk/mongo.py | 2 +- src/dsdk/mssql.py | 2 +- src/dsdk/service.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/dsdk/model.py b/src/dsdk/model.py index f46b317..150d2f8 100644 --- a/src/dsdk/model.py +++ b/src/dsdk/model.py @@ -20,7 +20,7 @@ from .utils import load_pickle_file # TODO Add import calling function from parent application -EXTRA = {"callingfunc": ""} +EXTRA = {"callingfunc": "callingfunc"} logger = getLogger(__name__) FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ %(callingfunc)s, "module": %(module)s, "function": %(funcName)s, \ diff --git a/src/dsdk/mongo.py b/src/dsdk/mongo.py index 8d55b9c..d76982b 100644 --- a/src/dsdk/mongo.py +++ b/src/dsdk/mongo.py @@ -42,7 +42,7 @@ AutoReconnect = None # TODO Add import calling function from parent application -EXTRA = {"callingfunc": ""} +EXTRA = {"callingfunc": "callingfunc"} logger = getLogger(__name__) FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ %(callingfunc)s, "module": %(module)s, "function": %(funcName)s, \ diff --git a/src/dsdk/mssql.py b/src/dsdk/mssql.py index 39075ef..0c0c6e9 100644 --- a/src/dsdk/mssql.py +++ b/src/dsdk/mssql.py @@ -26,7 +26,7 @@ create_engine = None # TODO Add import calling function from parent application -EXTRA = {"callingfunc": ""} +EXTRA = {"callingfunc": "callingfunc"} logger = getLogger(__name__) FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ %(callingfunc)s, "module": %(module)s, "function": %(funcName)s, \ diff --git a/src/dsdk/service.py b/src/dsdk/service.py index ad5d0a9..9036d5f 100644 --- a/src/dsdk/service.py +++ b/src/dsdk/service.py @@ -21,7 +21,7 @@ from configargparse import Namespace # TODO Add import calling function from parent application -EXTRA = {"callingfunc": ""} +EXTRA = {"callingfunc": "callingfunc"} logger = getLogger(__name__) FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ %(callingfunc)s, "module": %(module)s, "function": %(funcName)s, \ From 7ac32ce9ada1cbf84767d8ba413cd1c144cdf28a Mon Sep 17 00:00:00 2001 From: Darryl Mendillo <1238655+darrylmendillo@users.noreply.github.com> Date: Tue, 12 May 2020 08:34:39 -0400 Subject: [PATCH 09/14] Added "" to json logs --- src/dsdk/model.py | 4 ++-- src/dsdk/mongo.py | 4 ++-- src/dsdk/mssql.py | 4 ++-- src/dsdk/service.py | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/dsdk/model.py b/src/dsdk/model.py index 150d2f8..49dbc4c 100644 --- a/src/dsdk/model.py +++ b/src/dsdk/model.py @@ -20,10 +20,10 @@ from .utils import load_pickle_file # TODO Add import calling function from parent application -EXTRA = {"callingfunc": "callingfunc"} +EXTRA = {"callingfunc": ""} logger = getLogger(__name__) FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ - %(callingfunc)s, "module": %(module)s, "function": %(funcName)s, \ + "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \ %(message)s}' basicConfig(format=FORMAT) logger.setLevel(INFO) diff --git a/src/dsdk/mongo.py b/src/dsdk/mongo.py index d76982b..1e649a0 100644 --- a/src/dsdk/mongo.py +++ b/src/dsdk/mongo.py @@ -42,10 +42,10 @@ AutoReconnect = None # TODO Add import calling function from parent application -EXTRA = {"callingfunc": "callingfunc"} +EXTRA = {"callingfunc": ""} logger = getLogger(__name__) FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ - %(callingfunc)s, "module": %(module)s, "function": %(funcName)s, \ + "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \ %(message)s}' basicConfig(format=FORMAT) logger.setLevel(INFO) diff --git a/src/dsdk/mssql.py b/src/dsdk/mssql.py index 0c0c6e9..3ecbaa8 100644 --- a/src/dsdk/mssql.py +++ b/src/dsdk/mssql.py @@ -26,10 +26,10 @@ create_engine = None # TODO Add import calling function from parent application -EXTRA = {"callingfunc": "callingfunc"} +EXTRA = {"callingfunc": ""} logger = getLogger(__name__) FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ - %(callingfunc)s, "module": %(module)s, "function": %(funcName)s, \ + "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \ %(message)s}' basicConfig(format=FORMAT) logger.setLevel(INFO) diff --git a/src/dsdk/service.py b/src/dsdk/service.py index 9036d5f..76a3f75 100644 --- a/src/dsdk/service.py +++ b/src/dsdk/service.py @@ -21,10 +21,10 @@ from configargparse import Namespace # TODO Add import calling function from parent application -EXTRA = {"callingfunc": "callingfunc"} +EXTRA = {"callingfunc": ""} logger = getLogger(__name__) FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ - %(callingfunc)s, "module": %(module)s, "function": %(funcName)s, \ + "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \ %(message)s}' basicConfig(format=FORMAT) logger.setLevel(INFO) @@ -140,7 +140,7 @@ def __call__(self) -> Batch: for task in self.pipeline: task(batch, self) logger.info( - '"pipeline:" %s', + '"pipeline:" "%s"', ", ".join( map(lambda s: str(s).split(" ")[0][1:], self.pipeline) ), From 972c29a6f43c8a4343e77289a08d21cb4ef9d205 Mon Sep 17 00:00:00 2001 From: Darryl Mendillo <1238655+darrylmendillo@users.noreply.github.com> Date: Tue, 12 May 2020 08:43:40 -0400 Subject: [PATCH 10/14] Fixed json log syntax --- src/dsdk/mongo.py | 2 +- src/dsdk/service.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dsdk/mongo.py b/src/dsdk/mongo.py index 1e649a0..1b7df79 100644 --- a/src/dsdk/mongo.py +++ b/src/dsdk/mongo.py @@ -158,7 +158,7 @@ def store_evidence(self, batch: Batch, *args, **kwargs) -> None: df.drop(columns=["batch_id"], inplace=True) logger.info( '"action": "insert_many", "database": "%s", \ - "collection": "%s", "count": %s"', + "collection": "%s", "count": %s', database.name, database.collection.name, len(df.index), diff --git a/src/dsdk/service.py b/src/dsdk/service.py index 76a3f75..907bde3 100644 --- a/src/dsdk/service.py +++ b/src/dsdk/service.py @@ -140,7 +140,7 @@ def __call__(self) -> Batch: for task in self.pipeline: task(batch, self) logger.info( - '"pipeline:" "%s"', + '"pipeline": "%s"', ", ".join( map(lambda s: str(s).split(" ")[0][1:], self.pipeline) ), @@ -185,7 +185,7 @@ def store_evidence( # pylint: disable=no-self-use,unused-argument while args: key, df, *args = args # type: ignore batch.evidence[key] = df - logger.info('"key": "%s", "count": "%s"}', key, len(batch.evidence)) + logger.info('"key": "%s", "count": %s', key, len(batch.evidence)) class Task: # pylint: disable=too-few-public-methods From 21e0ff2d5e16504a101b23106287397e188ada04 Mon Sep 17 00:00:00 2001 From: Darryl Mendillo <1238655+darrylmendillo@users.noreply.github.com> Date: Tue, 12 May 2020 08:47:29 -0400 Subject: [PATCH 11/14] Fix log syntax --- src/dsdk/mongo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dsdk/mongo.py b/src/dsdk/mongo.py index 1b7df79..51c3984 100644 --- a/src/dsdk/mongo.py +++ b/src/dsdk/mongo.py @@ -119,7 +119,7 @@ def open_batch( with self.open_mongo() as database: key = insert_one(database.batches, doc) logger.info( - '"action": "insert": "database": "%s", "collection": "%s"', + '"action": "insert", "database": "%s", "collection": "%s"', database.name, database.collection.name, ) From d50575351caba9a740d37ac7116137dd202af611 Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Tue, 12 May 2020 13:46:16 -0400 Subject: [PATCH 12/14] Extract logger config --- src/dsdk/model.py | 23 +++-------------------- src/dsdk/mongo.py | 23 +++-------------------- src/dsdk/mssql.py | 24 ++++-------------------- src/dsdk/service.py | 23 ++++------------------- src/dsdk/utils.py | 42 +++++++++++++++++++++++++++++++++++++++--- 5 files changed, 53 insertions(+), 82 deletions(-) diff --git a/src/dsdk/model.py b/src/dsdk/model.py index 49dbc4c..7d56849 100644 --- a/src/dsdk/model.py +++ b/src/dsdk/model.py @@ -4,32 +4,15 @@ from __future__ import annotations from abc import ABC -from logging import ( - INFO, - Logger, - LoggerAdapter, - NullHandler, - basicConfig, - getLogger, -) +from logging import INFO from typing import TYPE_CHECKING, Optional, cast from configargparse import ArgParser as ArgumentParser from .service import Model, Service -from .utils import load_pickle_file +from .utils import get_logger, load_pickle_file -# TODO Add import calling function from parent application -EXTRA = {"callingfunc": ""} -logger = getLogger(__name__) -FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ - "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \ - %(message)s}' -basicConfig(format=FORMAT) -logger.setLevel(INFO) -# Add extra kwargs to message format -logger.addHandler(NullHandler()) -logger = cast(Logger, LoggerAdapter(logger, EXTRA)) +logger = get_logger(__file__, INFO) if TYPE_CHECKING: diff --git a/src/dsdk/mongo.py b/src/dsdk/mongo.py index 2146c1a..fce97e2 100644 --- a/src/dsdk/mongo.py +++ b/src/dsdk/mongo.py @@ -5,14 +5,7 @@ from abc import ABC from contextlib import contextmanager -from logging import ( - INFO, - Logger, - LoggerAdapter, - NullHandler, - basicConfig, - getLogger, -) +from logging import INFO from typing import ( TYPE_CHECKING, Any, @@ -26,7 +19,7 @@ from configargparse import ArgParser as ArgumentParser from .service import Batch, Model, Service -from .utils import retry +from .utils import get_logger, retry try: # Since not everyone will use mongo @@ -41,17 +34,7 @@ Database = None AutoReconnect = None -# TODO Add import calling function from parent application -EXTRA = {"callingfunc": ""} -logger = getLogger(__name__) -FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ - "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \ - %(message)s}' -basicConfig(format=FORMAT) -logger.setLevel(INFO) -# Add extra kwargs to message format -logger.addHandler(NullHandler()) -logger = cast(Logger, LoggerAdapter(logger, EXTRA)) +logger = get_logger(__name__, INFO) if TYPE_CHECKING: diff --git a/src/dsdk/mssql.py b/src/dsdk/mssql.py index 3ecbaa8..c908c36 100644 --- a/src/dsdk/mssql.py +++ b/src/dsdk/mssql.py @@ -5,19 +5,15 @@ from abc import ABC from contextlib import contextmanager -from logging import ( - INFO, - Logger, - LoggerAdapter, - NullHandler, - basicConfig, - getLogger, -) +from logging import INFO from typing import TYPE_CHECKING, Generator, Optional, cast from configargparse import ArgParser as ArgumentParser from .service import Service +from .utils import get_logger + +logger = get_logger(__file__, INFO) try: # Since not everyone will use mssql @@ -25,18 +21,6 @@ except ImportError: create_engine = None -# TODO Add import calling function from parent application -EXTRA = {"callingfunc": ""} -logger = getLogger(__name__) -FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ - "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \ - %(message)s}' -basicConfig(format=FORMAT) -logger.setLevel(INFO) -# Add extra kwargs to message format -logger.addHandler(NullHandler()) -logger = cast(Logger, LoggerAdapter(logger, EXTRA)) - if TYPE_CHECKING: BaseMixin = Service diff --git a/src/dsdk/service.py b/src/dsdk/service.py index 907bde3..10e2d08 100644 --- a/src/dsdk/service.py +++ b/src/dsdk/service.py @@ -6,31 +6,16 @@ from collections import OrderedDict from contextlib import contextmanager from datetime import datetime, timezone -from logging import ( - INFO, - Logger, - LoggerAdapter, - NullHandler, - basicConfig, - getLogger, -) +from logging import INFO from sys import argv as sys_argv from typing import Any, Dict, Generator, Optional, Sequence, Tuple, cast from configargparse import ArgParser as ArgumentParser from configargparse import Namespace -# TODO Add import calling function from parent application -EXTRA = {"callingfunc": ""} -logger = getLogger(__name__) -FORMAT = '%(asctime)-15s - %(name)s - %(levelname)s - {"callingfunc": \ - "%(callingfunc)s", "module": "%(module)s", "function": "%(funcName)s", \ - %(message)s}' -basicConfig(format=FORMAT) -logger.setLevel(INFO) -# Add extra kwargs to message format -logger.addHandler(NullHandler()) -logger = cast(Logger, LoggerAdapter(logger, EXTRA)) +from .utils import get_logger + +logger = get_logger(__file__, INFO) class Interval: # pylint: disable=too-few-public-methods diff --git a/src/dsdk/utils.py b/src/dsdk/utils.py index 50ff444..2ecd8a9 100644 --- a/src/dsdk/utils.py +++ b/src/dsdk/utils.py @@ -6,17 +6,53 @@ from functools import wraps from json import dump as json_dump from json import load as json_load -from logging import NullHandler, getLogger +from logging import INFO, Formatter, LoggerAdapter, StreamHandler, getLogger from pickle import dump as pickle_dump from pickle import load as pickle_load +from sys import stdout from time import sleep as default_sleep from typing import Any, Callable, Dict, List, Optional, Sequence from pandas import DataFrame from pandas import concat as pd_concat -logger = getLogger(__name__) -logger.addHandler(NullHandler()) + +def get_logger(name, level=INFO): + """Get logger. + + Actual handlers are typically set by the application. + Libraries (like DSDK) typically use a NullHandler, so that the application + logger configuration is used. + + Use this function to hide the logger implementation/config for now. + Show that the conventions demonstrated here work for the applications. + """ + # TODO Pass calling function from parent application + defaults = {"callingfunc": ""} + formatter_string = " - ".join( + ( + "%(asctime)-15s", + "%(name)s", + "%(levelname)s", + ", ".join( + ( + '{"callingfunc": "%(callingfunc)s"', + '"module": "%(module)s"', + '"function": "%(funcName)s"', + "%(message)s}", + ) + ), + ) + ) + handler = StreamHandler(stdout) + handler.setLevel(level) + handler.setFormatter(Formatter(formatter_string)) + result = getLogger(name) + result.addHandler(handler) + return LoggerAdapter(result, defaults) + + +logger = get_logger(__file__) def chunks(sequence: Sequence[Any], n: int): From d75716ba2f5a4fb42402cf9b427984efb777acd6 Mon Sep 17 00:00:00 2001 From: Darryl Mendillo <1238655+darrylmendillo@users.noreply.github.com> Date: Tue, 12 May 2020 17:29:44 -0400 Subject: [PATCH 13/14] Cleaned up logs with f strings, deduped logs --- src/dsdk/model.py | 2 +- src/dsdk/mongo.py | 21 ++++++++++----------- src/dsdk/mssql.py | 2 +- src/dsdk/service.py | 10 +++++++--- src/dsdk/utils.py | 3 ++- 5 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/dsdk/model.py b/src/dsdk/model.py index 7d56849..33641d9 100644 --- a/src/dsdk/model.py +++ b/src/dsdk/model.py @@ -12,7 +12,7 @@ from .service import Model, Service from .utils import get_logger, load_pickle_file -logger = get_logger(__file__, INFO) +logger = get_logger(__name__, INFO) if TYPE_CHECKING: diff --git a/src/dsdk/mongo.py b/src/dsdk/mongo.py index fce97e2..52c1a52 100644 --- a/src/dsdk/mongo.py +++ b/src/dsdk/mongo.py @@ -102,9 +102,9 @@ def open_batch( with self.open_mongo() as database: key = insert_one(database.batches, doc) logger.info( - '"action": "insert", "database": "%s", "collection": "%s"', - database.name, - database.collection.name, + f'"action": "insert", ' + f'"database": "{database.name}", ' + f'"collection": "{database.collection.name}"' ) yield batch @@ -112,9 +112,9 @@ def open_batch( with self.open_mongo() as database: update_one(database.batches, key, doc) logger.info( - '"action": "update", "database": "%s", "collection": "%s"', - database.name, - database.collection.name, + f'"action": "update", ' + f'"database": "{database.name}", ' + f'"collection": "{database.collection.name}"' ) def store_evidence(self, batch: Batch, *args, **kwargs) -> None: @@ -140,11 +140,10 @@ def store_evidence(self, batch: Batch, *args, **kwargs) -> None: # TODO: Better exception df.drop(columns=["batch_id"], inplace=True) logger.info( - '"action": "insert_many", "database": "%s", \ - "collection": "%s", "count": %s', - database.name, - database.collection.name, - len(df.index), + f'"action": "insert_many", ' + f'"database": "{database.name}", ' + f'"collection": "{database.collection.name}", ' + f'"count": {len(df.index)}' ) diff --git a/src/dsdk/mssql.py b/src/dsdk/mssql.py index c908c36..02bb709 100644 --- a/src/dsdk/mssql.py +++ b/src/dsdk/mssql.py @@ -13,7 +13,7 @@ from .service import Service from .utils import get_logger -logger = get_logger(__file__, INFO) +logger = get_logger(__name__, INFO) try: # Since not everyone will use mssql diff --git a/src/dsdk/service.py b/src/dsdk/service.py index 10e2d08..41fa8f9 100644 --- a/src/dsdk/service.py +++ b/src/dsdk/service.py @@ -15,7 +15,7 @@ from .utils import get_logger -logger = get_logger(__file__, INFO) +logger = get_logger(__name__, INFO) class Interval: # pylint: disable=too-few-public-methods @@ -161,7 +161,7 @@ def open_batch( # pylint: disable=no-self-use,unused-argument record = Interval(on=datetime.now(timezone.utc), end=None) yield Batch(key, record) record.end = datetime.now(timezone.utc) - logger.info('"key": "%s"', key) + logger.info(f'"action": "open_batch", ' f'"key": "{key}"') def store_evidence( # pylint: disable=no-self-use,unused-argument self, batch: Batch, *args, **kwargs @@ -170,7 +170,11 @@ def store_evidence( # pylint: disable=no-self-use,unused-argument while args: key, df, *args = args # type: ignore batch.evidence[key] = df - logger.info('"key": "%s", "count": %s', key, len(batch.evidence)) + logger.info( + f'"action": "store_evidence", ' + f'"key": "{key}", ' + f'"count": "{len(batch.evidence)}"' + ) class Task: # pylint: disable=too-few-public-methods diff --git a/src/dsdk/utils.py b/src/dsdk/utils.py index 2ecd8a9..e59472a 100644 --- a/src/dsdk/utils.py +++ b/src/dsdk/utils.py @@ -48,11 +48,12 @@ def get_logger(name, level=INFO): handler.setLevel(level) handler.setFormatter(Formatter(formatter_string)) result = getLogger(name) + result.propagate = False result.addHandler(handler) return LoggerAdapter(result, defaults) -logger = get_logger(__file__) +logger = get_logger(__name__) def chunks(sequence: Sequence[Any], n: int): From 1929e06f17f65556da606ac04fd5c33913ca8188 Mon Sep 17 00:00:00 2001 From: Darryl Mendillo <1238655+darrylmendillo@users.noreply.github.com> Date: Tue, 12 May 2020 17:41:48 -0400 Subject: [PATCH 14/14] Changed service count to int --- src/dsdk/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dsdk/service.py b/src/dsdk/service.py index 41fa8f9..01a8907 100644 --- a/src/dsdk/service.py +++ b/src/dsdk/service.py @@ -173,7 +173,7 @@ def store_evidence( # pylint: disable=no-self-use,unused-argument logger.info( f'"action": "store_evidence", ' f'"key": "{key}", ' - f'"count": "{len(batch.evidence)}"' + f'"count": {len(batch.evidence)}' )