From af70b665bbb8cf267e72ad3315699b09e2d63bd5 Mon Sep 17 00:00:00 2001 From: v-lerie Date: Mon, 7 Jul 2025 01:40:47 -0400 Subject: [PATCH 1/8] Added structured logging using Python logging to mimic GCP logs --- changelog_entry.yaml | 4 ++ policyengine_api/libs/simulation_api.py | 4 +- policyengine_api/services/economy_service.py | 2 +- policyengine_api/utils/logger.py | 46 ++++++++++++++++++++ policyengine_api/utils/python_logging.py | 3 ++ 5 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 policyengine_api/utils/logger.py create mode 100644 policyengine_api/utils/python_logging.py diff --git a/changelog_entry.yaml b/changelog_entry.yaml index e69de29bb..7973e1589 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -0,0 +1,4 @@ +- bump: patch + changes: + added: + - Implemented structured logging in policyengine_api/utils/logger.py to produce GCP-compatible JSON logs. \ No newline at end of file diff --git a/policyengine_api/libs/simulation_api.py b/policyengine_api/libs/simulation_api.py index 1fbd12b48..b5d8cba9a 100644 --- a/policyengine_api/libs/simulation_api.py +++ b/policyengine_api/libs/simulation_api.py @@ -5,7 +5,7 @@ import time from typing import Any, Literal, Annotated from dotenv import load_dotenv -from policyengine_api.gcp_logging import logger +from policyengine_api.utils.python_logging import logger from google.cloud.workflows import executions_v1 @@ -14,6 +14,8 @@ ExecutionState = executions_v1.types.Execution.State + + class SimulationAPI: project: str location: str diff --git a/policyengine_api/services/economy_service.py b/policyengine_api/services/economy_service.py index e1cebffd1..434957ca2 100644 --- a/policyengine_api/services/economy_service.py +++ b/policyengine_api/services/economy_service.py @@ -3,7 +3,7 @@ ReformImpactsService, ) from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS -from policyengine_api.gcp_logging import logger +from policyengine_api.utils.python_logging import logger from policyengine_api.libs.simulation_api import SimulationAPI from policyengine_api.data.model_setup import get_dataset_version from policyengine.simulation import SimulationOptions diff --git a/policyengine_api/utils/logger.py b/policyengine_api/utils/logger.py new file mode 100644 index 000000000..abda033b6 --- /dev/null +++ b/policyengine_api/utils/logger.py @@ -0,0 +1,46 @@ +import logging +import json +import sys +from datetime import datetime, timezone + + +class Logger: + def __init__(self, name = "policyengine-api", level = logging.INFO): + self.logger = logging.getLogger(name) + self.logger.setLevel(level) + + if not self.logger.handlers: + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(logging.Formatter('%(message)s')) + self.logger.addHandler(handler) + + def log_struct(self, payload: dict, severity: str = "INFO", message: str = None): + log_entry = { + "severity": severity.upper(), + "timestamp": datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + } + + if message: + log_entry["message"] = message + + if isinstance(payload, dict): + log_entry.update(payload) + + json_log = json.dumps(log_entry, ensure_ascii=False) + self._emit(severity, json_log) + + def _emit(self, severity: str, msg: str): + level = severity.upper() + if level == "DEBUG": + self.logger.debug(msg) + elif level == "INFO": + self.logger.info(msg) + elif level == "WARNING": + self.logger.warning(msg) + elif level == "ERROR": + self.logger.error(msg) + elif level == "CRITICAL": + self.logger.critical(msg) + else: + self.logger.info(msg) \ No newline at end of file diff --git a/policyengine_api/utils/python_logging.py b/policyengine_api/utils/python_logging.py new file mode 100644 index 000000000..49db6c8aa --- /dev/null +++ b/policyengine_api/utils/python_logging.py @@ -0,0 +1,3 @@ +from policyengine_api.utils.logger import Logger + +logger = Logger("policyengine-api") From c0a748816d3c24267a416f54e8aeba241cc4d0e4 Mon Sep 17 00:00:00 2001 From: v-lerie Date: Mon, 7 Jul 2025 01:44:35 -0400 Subject: [PATCH 2/8] Added structured logging using Python logging to mimic GCP logs --- policyengine_api/libs/simulation_api.py | 2 -- policyengine_api/utils/logger.py | 17 ++++++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/policyengine_api/libs/simulation_api.py b/policyengine_api/libs/simulation_api.py index b5d8cba9a..492097e76 100644 --- a/policyengine_api/libs/simulation_api.py +++ b/policyengine_api/libs/simulation_api.py @@ -14,8 +14,6 @@ ExecutionState = executions_v1.types.Execution.State - - class SimulationAPI: project: str location: str diff --git a/policyengine_api/utils/logger.py b/policyengine_api/utils/logger.py index abda033b6..2cf063809 100644 --- a/policyengine_api/utils/logger.py +++ b/policyengine_api/utils/logger.py @@ -5,21 +5,24 @@ class Logger: - def __init__(self, name = "policyengine-api", level = logging.INFO): + def __init__(self, name="policyengine-api", level=logging.INFO): self.logger = logging.getLogger(name) self.logger.setLevel(level) if not self.logger.handlers: handler = logging.StreamHandler(sys.stdout) - handler.setFormatter(logging.Formatter('%(message)s')) + handler.setFormatter(logging.Formatter("%(message)s")) self.logger.addHandler(handler) - def log_struct(self, payload: dict, severity: str = "INFO", message: str = None): + def log_struct( + self, payload: dict, severity: str = "INFO", message: str = None + ): log_entry = { "severity": severity.upper(), - "timestamp": datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z") - - } + "timestamp": datetime.now(timezone.utc) + .isoformat(timespec="milliseconds") + .replace("+00:00", "Z"), + } if message: log_entry["message"] = message @@ -43,4 +46,4 @@ def _emit(self, severity: str, msg: str): elif level == "CRITICAL": self.logger.critical(msg) else: - self.logger.info(msg) \ No newline at end of file + self.logger.info(msg) From 4408030be667fd6c4d83616cb2c4f084bccdd8a9 Mon Sep 17 00:00:00 2001 From: v-lerie Date: Sun, 13 Jul 2025 18:13:53 -0400 Subject: [PATCH 3/8] Added test_logger.py --- tests/unit/test_logger.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/unit/test_logger.py diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py new file mode 100644 index 000000000..629c72f8a --- /dev/null +++ b/tests/unit/test_logger.py @@ -0,0 +1,13 @@ +from policyengine_api.utils.logger import Logger + +def test_log_struct(capsys): + logger = Logger("test-logger") + logger.log_struct( + {"event": "test_event", "user_id": "u123"}, + severity="INFO", + message="Logging from test" + ) + captured = capsys.readouterr() + assert '"severity": "INFO"' in captured.out + assert '"event": "test_event"' in captured.out + assert '"message": "Logging from test"' in captured.out From 69e71fc9a831622c6588f1a822f841358e4b26cd Mon Sep 17 00:00:00 2001 From: v-lerie Date: Sun, 13 Jul 2025 18:21:05 -0400 Subject: [PATCH 4/8] Added test_logger.py --- tests/unit/test_logger.py | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 tests/unit/test_logger.py diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py deleted file mode 100644 index 629c72f8a..000000000 --- a/tests/unit/test_logger.py +++ /dev/null @@ -1,13 +0,0 @@ -from policyengine_api.utils.logger import Logger - -def test_log_struct(capsys): - logger = Logger("test-logger") - logger.log_struct( - {"event": "test_event", "user_id": "u123"}, - severity="INFO", - message="Logging from test" - ) - captured = capsys.readouterr() - assert '"severity": "INFO"' in captured.out - assert '"event": "test_event"' in captured.out - assert '"message": "Logging from test"' in captured.out From d01e035e943f06bc4b11288d37c13c00c45bdfee Mon Sep 17 00:00:00 2001 From: v-lerie Date: Sun, 13 Jul 2025 18:27:02 -0400 Subject: [PATCH 5/8] Added test_logger.py --- tests/unit/test_logger.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/unit/test_logger.py diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py new file mode 100644 index 000000000..b94784d4d --- /dev/null +++ b/tests/unit/test_logger.py @@ -0,0 +1,13 @@ +from policyengine_api.utils.logger import Logger + +def test_log_struct(capsys): + logger = Logger("test-logger") + logger.log_struct( + {"event": "test_event", "user_id": "abc123"}, + severity="INFO", + message="Logging from test" + ) + captured = capsys.readouterr() + assert '"severity": "INFO"' in captured.out + assert '"event": "test_event"' in captured.out + assert '"message": "Logging from test"' in captured.out From f7ad1e55d3288f7f6ec5e40ba66db333197d263f Mon Sep 17 00:00:00 2001 From: v-lerie Date: Sun, 13 Jul 2025 18:28:13 -0400 Subject: [PATCH 6/8] Added test_logger.py --- tests/unit/test_logger.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index b94784d4d..f116760cc 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -1,11 +1,12 @@ from policyengine_api.utils.logger import Logger + def test_log_struct(capsys): logger = Logger("test-logger") logger.log_struct( {"event": "test_event", "user_id": "abc123"}, severity="INFO", - message="Logging from test" + message="Logging from test", ) captured = capsys.readouterr() assert '"severity": "INFO"' in captured.out From 26fa54964ddb7e7bf5448d464a0237cfd8764c86 Mon Sep 17 00:00:00 2001 From: v-lerie Date: Wed, 16 Jul 2025 23:30:51 -0400 Subject: [PATCH 7/8] Changed log_text to log_struct in simulation_api.py --- changelog_entry.yaml | 2 +- policyengine_api/libs/simulation_api.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog_entry.yaml b/changelog_entry.yaml index 7973e1589..ecf5de99a 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -1,4 +1,4 @@ - bump: patch changes: added: - - Implemented structured logging in policyengine_api/utils/logger.py to produce GCP-compatible JSON logs. \ No newline at end of file + - Implemented structured logging in policyengine_api/utils/logger.py to produce GCP-compatible JSON logs.engine \ No newline at end of file diff --git a/policyengine_api/libs/simulation_api.py b/policyengine_api/libs/simulation_api.py index 492097e76..637f2b4d2 100644 --- a/policyengine_api/libs/simulation_api.py +++ b/policyengine_api/libs/simulation_api.py @@ -21,7 +21,7 @@ class SimulationAPI: def __init__(self): if os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") is None: - logger.log_text( + logger.log_struct( "GOOGLE_APPLICATION_CREDENTIALS not set; unable to run simulation API.", severity="ERROR", ) From f1a73c9d80021e6d424a83760960a749ad808ad5 Mon Sep 17 00:00:00 2001 From: v-lerie Date: Wed, 16 Jul 2025 23:32:54 -0400 Subject: [PATCH 8/8] Changed log_text to log_struct in simulation_api.py --- changelog_entry.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog_entry.yaml b/changelog_entry.yaml index ecf5de99a..7973e1589 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -1,4 +1,4 @@ - bump: patch changes: added: - - Implemented structured logging in policyengine_api/utils/logger.py to produce GCP-compatible JSON logs.engine \ No newline at end of file + - Implemented structured logging in policyengine_api/utils/logger.py to produce GCP-compatible JSON logs. \ No newline at end of file