From ef8857f3cf6e6072a629440392e0314da041debd Mon Sep 17 00:00:00 2001 From: Sarah G Date: Tue, 12 Sep 2023 12:06:00 +0200 Subject: [PATCH] Use a log file instead of STDOUT --- backend/Dockerfile | 12 ------ backend/src/main.py | 55 ++++++++++++++++++++---- backend/tests/test_api.py | 26 +++++++++++ docker-compose-dev.yml | 1 - infra/kube/helm/templates/configmap.yaml | 1 + infra/kube/helm/values.yaml | 4 +- 6 files changed, 76 insertions(+), 23 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 0cbe0b5d..08dc27f6 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -2,18 +2,6 @@ FROM python:3.9-slim-buster as base WORKDIR /app -# Create a non-privileged user that the app will run under. -# See https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#user -ARG UID=10001 -RUN adduser \ - --disabled-password \ - --gecos "" \ - --home "/nonexistent" \ - --shell "/sbin/nologin" \ - --no-create-home \ - --uid "${UID}" \ - appuser - # certificates config ARG CACERT_LOCATION COPY ./cert/. /etc/ssl/certs/ diff --git a/backend/src/main.py b/backend/src/main.py index 360aaa62..8648906b 100644 --- a/backend/src/main.py +++ b/backend/src/main.py @@ -1,6 +1,5 @@ import os -import sys import logging from logging.handlers import TimedRotatingFileHandler from datetime import datetime @@ -46,17 +45,27 @@ def init_variable(var_name: str, path: str) -> str: return VAR -def setup_logs() -> logging.Logger: - """Setup environment for logs +def setup_logs(log_dir: str) -> logging.Logger: + """Setups environment for logs Args: + log_dir (str): folder for log storage logging.Logger: logger object """ print(">>> Reload logs config") + # clear previous logs + for f in os.listdir(log_dir): + os.remove(os.path.join(log_dir, f)) # configure new logs formatter = GelfFormatter() logger = logging.getLogger("Basegun") - handler = logging.StreamHandler(sys.stdout) + # new log file at midnight + log_file = os.path.join(log_dir, "log.json") + handler = TimedRotatingFileHandler( + log_file, + when="midnight", + interval=1, + backupCount=7) logger.setLevel(logging.INFO) handler.setFormatter(formatter) logger.addHandler(handler) @@ -64,7 +73,13 @@ def setup_logs() -> logging.Logger: def get_device(user_agent) -> str: - """Explicitly give the device of a user-agent object + """Explicitly gives the device of a user-agent object + + Args: + user_agent: info given by the user browser + + Returns: + str: mobile, pc, tablet or other """ if user_agent.is_mobile: return "mobile" @@ -76,7 +91,18 @@ def get_device(user_agent) -> str: return "other" -def get_base_logs(user_agent, user_id) -> dict: +def get_base_logs(user_agent, user_id: str) -> dict: + """Generates the common information for custom logs in basegun. + Each function can add some info specific to the current process, + then we insert these custom logs as extra + + Args: + user_agent: user agent object + user_id (str): UUID identifying a unique user + + Returns: + dict: the base custom information + """ extras_logging = { "bg_date": datetime.now().isoformat(), "bg_user_id": user_id, @@ -91,7 +117,7 @@ def get_base_logs(user_agent, user_id) -> dict: def upload_image_ovh(content: bytes, img_name: str): - """ Uploads an image to owh swift container basegun-public + """Uploads an image to basegun ovh swift container path uploaded-images/WORKSPACE/img_name where WORKSPACE is dev, preprod or prod @@ -158,7 +184,8 @@ def upload_image_ovh(content: bytes, img_name: str): ) # Logs -logger = setup_logs() +PATH_LOGS = init_variable("PATH_LOGS", "../logs") +logger = setup_logs(PATH_LOGS) # Load model MODEL_PATH = os.path.join( @@ -216,6 +243,18 @@ def version(): return APP_VERSION +@app.get("/logs") +def logs(): + if "WORKSPACE" in os.environ and os.environ["WORKSPACE"] != "prod": + with open(os.path.join(PATH_LOGS, "log.json"), "r") as f: + lines = f.readlines() + res = [json.loads(l) for l in lines] + res.reverse() + return res + else: + return PlainTextResponse("Forbidden") + + @app.post("/upload") async def imageupload( request: Request, diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index d833518b..eedd145a 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -59,6 +59,21 @@ def test_upload_and_logs(self): self.assertEqual(image_one.size, image_two.size) diff = ImageChops.difference(image_one, image_two) self.assertFalse(diff.getbbox()) + # checks that the result is written in logs + r = requests.get(self.url + "/logs") + self.assertEqual(r.status_code, 200) + # checks the latest log "Upload to OVH" + self.assertEqual(r.json()[0]["_bg_image_url"], r.json()[1]["_bg_image_url"]) + self.assertEqual(r.json()[0]["short_message"], "Upload to OVH successful") + # checks the previous log "Identification request" + log = r.json()[1] + self.check_log_base(log) + self.assertEqual(log["short_message"], "Identification request") + self.assertTrue("-" in log["_bg_user_id"]) + self.assertEqual(log["_bg_geolocation"], geoloc) + self.assertEqual(log["_bg_label"], "revolver") + self.assertAlmostEqual(log["_bg_confidence"], 98.43, places=1) + self.assertTrue(log["_bg_upload_time"]>=0) def test_feedback_and_logs(self): """Checks that the feedback works properly""" @@ -68,7 +83,18 @@ def test_feedback_and_logs(self): image_url = "https://storage.gra.cloud.ovh.net/v1/test" r = requests.post(self.url + "/identification-feedback", json={"image_url": image_url, "feedback": True, "confidence": confidence, "label": label, "confidence_level": confidence_level}) + + self.assertEqual(r.status_code, 200) + r = requests.get(self.url + "/logs") self.assertEqual(r.status_code, 200) + log = r.json()[0] + self.check_log_base(log) + self.assertEqual(log["short_message"], "Identification feedback") + self.assertEqual(log["_bg_image_url"], image_url) + self.assertTrue(log["_bg_feedback_bool"]) + self.assertEqual(log["_bg_confidence"], confidence) + self.assertEqual(log["_bg_label"], label) + self.assertEqual(log["_bg_confidence_level"], confidence_level) def test_geoloc_api(self): """Checks that the geolocation api works properly""" diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index d27ca20b..b1ee9dc5 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -11,7 +11,6 @@ services: context: ./backend target: ${BUILD_TARGET:-dev} container_name: basegun-backend - user: appuser environment: - PATH_LOGS=/tmp/logs - OS_USERNAME diff --git a/infra/kube/helm/templates/configmap.yaml b/infra/kube/helm/templates/configmap.yaml index 72dc8749..8daa0ba6 100644 --- a/infra/kube/helm/templates/configmap.yaml +++ b/infra/kube/helm/templates/configmap.yaml @@ -5,6 +5,7 @@ metadata: data: WORKSPACE: {{ .Values.backend.config.workspace }} PATH_IMGS: {{ .Values.backend.config.path_imgs }} + PATH_LOGS: /tmp/log/basegun/ --- apiVersion: v1 kind: ConfigMap diff --git a/infra/kube/helm/values.yaml b/infra/kube/helm/values.yaml index 2b832d0a..9e2eeb46 100644 --- a/infra/kube/helm/values.yaml +++ b/infra/kube/helm/values.yaml @@ -52,8 +52,8 @@ backend: pullPolicy: IfNotPresent service: type: ClusterIP - port: 8000 - containerPort: 8000 + port: 5000 + containerPort: 5000 autoscaling: enabled: false minReplicas: 1