Merge pull request #181 from datalab-mi/fix/revert-to-log-file

Use a log file instead of STDOUT for logging

README.md

@@ -61,7 +61,7 @@ If your network imposes you to use a custom certicifate (error "SSL" at build),
 ### The website sends an error
 Try to find error log
 * In terminal, run `docker logs basegun-backend`
-* If you cannot access terminal or don't see anything, go to `localhost:8000/logs` or `preprod.basegun.fr/logs` to see latest logs.
+* If you cannot access terminal or don't see anything, go to `localhost:5000/logs` or `preprod.basegun.fr/logs` to see latest logs.
 
 => Error "missing model": Download model from the url specified in the [backend Dockerfile](https://github.com/datalab-mi/Basegun/blob/develop/backend/Dockerfile).
 

backend/Dockerfile

@@ -1,19 +1,6 @@
 FROM python:3.9-slim-buster as base
-
 WORKDIR /app
 
-# Create a non-privileged user that the app will run under.
-# See https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#user
-ARG UID=10001
-RUN adduser \
-    --disabled-password \
-    --gecos "" \
-    --home "/nonexistent" \
-    --shell "/sbin/nologin" \
-    --no-create-home \
-    --uid "${UID}" \
-    appuser
-
 # certificates config
 ARG CACERT_LOCATION
 COPY ./cert/. /etc/ssl/certs/
@@ -45,13 +32,13 @@ RUN mkdir -p src/weights \
 # launch website
 FROM base as dev
 RUN pip --default-timeout=300 install --no-cache-dir -r requirements/dev.txt
-CMD ["uvicorn", "src.main:app", "--reload", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["uvicorn", "src.main:app", "--reload", "--host", "0.0.0.0", "--port", "5000"]
 
 FROM base as test
 RUN pip install -r requirements/dev.txt && pip install requests && rm -r /root/.cache
 COPY tests/ tests/
-CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "5000"]
 
 FROM base as prod
 RUN pip install --no-cache-dir -r requirements/prod.txt
-CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "5000"]

backend/README.md

@@ -20,12 +20,12 @@ If you are in a network blocked with proxy, remember to add arg `--build_arg htt
 
 ### Without Docker
 ```bash
-uvicorn src.main:app --reload --host 0.0.0.0 --port 8000
+uvicorn src.main:app --reload --host 0.0.0.0 --port 5000
 ```
 
 ### With Docker
 ```bash
-docker run --rm -d -p 8000:8000 --name basegun_back -e VERSION=1.2 basegun-back:dev
+docker run --rm -d -p 5000:5000 --name basegun_back -e VERSION=1.2 basegun-back:dev
 ```
 Remember afterwards to stop container `docker stop basegun_back`
 

backend/src/main.py

@@ -1,6 +1,5 @@
 
 import os
-import sys
 import logging
 from logging.handlers import TimedRotatingFileHandler
 from datetime import datetime
@@ -46,25 +45,41 @@ def init_variable(var_name: str, path: str) -> str:
     return VAR
 
 
-def setup_logs() -> logging.Logger:
-    """Setup environment for logs
+def setup_logs(log_dir: str) -> logging.Logger:
+    """Setups environment for logs
 
     Args:
+        log_dir (str): folder for log storage
         logging.Logger: logger object
     """
     print(">>> Reload logs config")
+    # clear previous logs
+    for f in os.listdir(log_dir):
+        os.remove(os.path.join(log_dir, f))
     # configure new logs
     formatter = GelfFormatter()
     logger = logging.getLogger("Basegun")
-    handler = logging.StreamHandler(sys.stdout)
+    # new log file at midnight
+    log_file = os.path.join(log_dir, "log.json")
+    handler = TimedRotatingFileHandler(
+        log_file,
+        when="midnight",
+        interval=1,
+        backupCount=7)
     logger.setLevel(logging.INFO)
     handler.setFormatter(formatter)
     logger.addHandler(handler)
     return logger
 
 
 def get_device(user_agent) -> str:
-    """Explicitly give the device of a user-agent object
+    """Explicitly gives the device of a user-agent object
+
+    Args:
+        user_agent: info given by the user browser
+
+    Returns:
+        str: mobile, pc, tablet or other
     """
     if user_agent.is_mobile:
         return "mobile"
@@ -76,7 +91,18 @@ def get_device(user_agent) -> str:
         return "other"
 
 
-def get_base_logs(user_agent, user_id) -> dict:
+def get_base_logs(user_agent, user_id: str) -> dict:
+    """Generates the common information for custom logs in basegun.
+        Each function can add some info specific to the current process,
+        then we insert these custom logs as extra
+
+    Args:
+        user_agent: user agent object
+        user_id (str): UUID identifying a unique user
+
+    Returns:
+        dict: the base custom information
+    """
     extras_logging = {
         "bg_date": datetime.now().isoformat(),
         "bg_user_id": user_id,
@@ -91,7 +117,7 @@ def get_base_logs(user_agent, user_id) -> dict:
 
 
 def upload_image_ovh(content: bytes, img_name: str):
-    """ Uploads an image to owh swift container basegun-public
+    """Uploads an image to basegun ovh swift container
         path uploaded-images/WORKSPACE/img_name
         where WORKSPACE is dev, preprod or prod
 
@@ -158,7 +184,8 @@ def upload_image_ovh(content: bytes, img_name: str):
 )
 
 # Logs
-logger = setup_logs()
+PATH_LOGS = init_variable("PATH_LOGS", "../logs")
+logger = setup_logs(PATH_LOGS)
 
 # Load model
 MODEL_PATH = os.path.join(
@@ -216,6 +243,18 @@ def version():
     return APP_VERSION
 
 
+@app.get("/logs")
+def logs():
+    if "WORKSPACE" in os.environ and os.environ["WORKSPACE"] != "prod":
+        with open(os.path.join(PATH_LOGS, "log.json"), "r") as f:
+            lines = f.readlines()
+            res = [json.loads(l) for l in lines]
+            res.reverse()
+            return res
+    else:
+        return PlainTextResponse("Forbidden")
+
+
 @app.post("/upload")
 async def imageupload(
     request: Request,

backend/tests/test_api.py

@@ -9,7 +9,7 @@
 class TestModel(unittest.TestCase):
     def __init__(self, *args, **kwargs):
         super(TestModel, self).__init__(*args, **kwargs)
-        self.url = "http://localhost:8000"
+        self.url = "http://localhost:5000"
 
     def test_home(self):
         """Checks that the route / is alive"""
@@ -59,6 +59,21 @@ def test_upload_and_logs(self):
                 self.assertEqual(image_one.size, image_two.size)
                 diff = ImageChops.difference(image_one, image_two)
                 self.assertFalse(diff.getbbox())
+        # checks that the result is written in logs
+        r = requests.get(self.url + "/logs")
+        self.assertEqual(r.status_code, 200)
+        # checks the latest log "Upload to OVH"
+        self.assertEqual(r.json()[0]["_bg_image_url"], r.json()[1]["_bg_image_url"])
+        self.assertEqual(r.json()[0]["short_message"], "Upload to OVH successful")
+        # checks the previous log "Identification request"
+        log = r.json()[1]
+        self.check_log_base(log)
+        self.assertEqual(log["short_message"], "Identification request")
+        self.assertTrue("-" in log["_bg_user_id"])
+        self.assertEqual(log["_bg_geolocation"], geoloc)
+        self.assertEqual(log["_bg_label"], "revolver")
+        self.assertAlmostEqual(log["_bg_confidence"], 98.43, places=1)
+        self.assertTrue(log["_bg_upload_time"]>=0)
 
     def test_feedback_and_logs(self):
         """Checks that the feedback works properly"""
@@ -68,7 +83,18 @@ def test_feedback_and_logs(self):
         image_url = "https://storage.gra.cloud.ovh.net/v1/test"
         r = requests.post(self.url + "/identification-feedback",
                 json={"image_url": image_url, "feedback": True, "confidence": confidence, "label": label, "confidence_level": confidence_level})
+
+        self.assertEqual(r.status_code, 200)
+        r = requests.get(self.url + "/logs")
         self.assertEqual(r.status_code, 200)
+        log = r.json()[0]
+        self.check_log_base(log)
+        self.assertEqual(log["short_message"], "Identification feedback")
+        self.assertEqual(log["_bg_image_url"], image_url)
+        self.assertTrue(log["_bg_feedback_bool"])
+        self.assertEqual(log["_bg_confidence"], confidence)
+        self.assertEqual(log["_bg_label"], label)
+        self.assertEqual(log["_bg_confidence_level"], confidence_level)
 
     def test_geoloc_api(self):
         """Checks that the geolocation api works properly"""

docker-compose-dev.yml

@@ -11,7 +11,6 @@ services:
       context: ./backend
       target: ${BUILD_TARGET:-dev}
     container_name: basegun-backend
-    user: appuser
     environment:
       - PATH_LOGS=/tmp/logs
       - OS_USERNAME
@@ -26,7 +25,7 @@ services:
       - REQUESTS_CA_BUNDLE=$CACERT_LOCATION
     image: basegun-backend:${TAG:-2.0}-dev
     ports:
-      - 8000:8000
+      - 5000:5000
     volumes:
       - $PWD/backend/src:/app/src
       - $PWD/backend/tests:/app/tests

docker-compose-prod.yml

@@ -17,7 +17,7 @@ services:
       - WORKSPACE=${WORKSPACE:-prod}
     image: basegun-backend:${TAG}-prod
     ports:
-      - 8000:8000
+      - 5000:5000
 
   frontend:
     build:

frontend/nginx.conf

@@ -16,7 +16,7 @@ http {
         }
 
         location /api/ {
-            proxy_pass http://basegun-backend:8000/;
+            proxy_pass http://basegun-backend:5000/;
             client_max_body_size 5M;
             proxy_read_timeout 1800;
             proxy_connect_timeout 1800;

frontend/vite.config.js

@@ -61,7 +61,7 @@ export default defineConfig({
     host: true,
     proxy: {
       '^/api': {
-        target: `http://${apiHost}:8000`,
+        target: `http://${apiHost}:5000`,
         changeOrigin: true,
         rewrite: (path) => path.replace(/^\/api/, '')
       }

infra/kube/helm/templates/configmap.yaml

@@ -5,6 +5,7 @@ metadata:
 data:
   WORKSPACE: {{ .Values.backend.config.workspace }}
   PATH_IMGS: {{ .Values.backend.config.path_imgs }}
+  PATH_LOGS: /tmp/log/basegun/
 ---
 apiVersion: v1
 kind: ConfigMap

infra/kube/helm/values.yaml

@@ -52,8 +52,8 @@ backend:
     pullPolicy: IfNotPresent
   service:
     type: ClusterIP
-    port: 8000
-    containerPort: 8000
+    port: 5000
+    containerPort: 5000
   autoscaling:
     enabled: false
     minReplicas: 1