Merge pull request #158 from datalab-mi/develop

Tutos factices

.github/workflows/preprod.yml

@@ -46,7 +46,7 @@ jobs:
     needs: build-push
     with:
       image_version: "develop"
-      branch: "develop"
+      branch: ${{ github.ref_name }}
       volume_size: 10
       flavor: "s1-2"
       workspace: "preprod"

Makefile

@@ -1,7 +1,7 @@
 SHELL	:= /bin/bash
 DOCKER	:= $(shell type -p docker)
 DC		:= $(shell type -p docker-compose)
-TAG		:= 1.5
+TAG		:= 2.0
 APP_NAME	:= basegun
 REG		:= ghcr.io
 ORG		:= datalab-mi

backend/src/main.py

@@ -84,6 +84,64 @@ def get_device(user_agent) -> str:
         return "other"
 
 
+def get_base_logs(user_agent, user_id) -> dict:
+    extras_logging = {
+        "bg_date": datetime.now().isoformat(),
+        "bg_user_id": user_id,
+        "bg_version": APP_VERSION,
+        "bg_model": MODEL_VERSION,
+        "bg_device": get_device(user_agent),
+        "bg_device_family": user_agent.device.family,
+        "bg_device_os": user_agent.os.family,
+        "bg_device_browser": user_agent.browser.family,
+    }
+    return extras_logging
+
+
+def upload_image_ovh(content: bytes, img_name: str):
+    """ Uploads an image to owh swift container basegun-public
+        path uploaded-images/WORKSPACE/img_name
+        where WORKSPACE is dev, preprod or prod
+
+    Args:
+        content (bytes): file content
+        img_name (str): name we want to give on ovh
+    """
+    num_tries = 0
+    LIMIT_TRIES = 5
+    image_path = os.path.join(CLOUD_PATH, img_name)
+    start = time.time()
+
+    if not conn:
+        logger.exception("Variables not set for using OVH swift.", extra={
+            "bg_error_type": "NameError"
+        })
+        return
+
+    while num_tries <= LIMIT_TRIES:
+        num_tries += 1
+        extras_logging = {
+            "bg_date": datetime.now().isoformat(),
+            "bg_upload_time": time.time()-start,
+            "bg_image_url": image_path
+        }
+        try:
+            conn.put_object("basegun-public",
+                            f'uploaded-images/{os.environ["WORKSPACE"]}/{img_name}',
+                            contents=content)
+            # if success, get out of the loop
+            logger.info("Upload to OVH successful", extra=extras_logging)
+            break
+        except Exception as e:
+            if (num_tries <= LIMIT_TRIES and e.__class__.__name__ == "ClientException"):
+                # we try uploading another time
+                time.sleep(30)
+                continue
+            else:
+                extras_logging["bg_error_type"] = e.__class__.__name__
+                logger.exception(e, extra=extras_logging)
+
+
 ####################
 #      SETUP       #
 ####################
@@ -151,50 +209,6 @@ def get_device(user_agent) -> str:
     logger.warn('Variables necessary for OVH connection not set !')
 
 
-def upload_image_ovh(content: bytes, img_name: str):
-    """ Uploads an image to owh swift container basegun-public
-        path uploaded-images/WORKSPACE/img_name
-        where WORKSPACE is dev, preprod or prod
-
-    Args:
-        content (bytes): file content
-        img_name (str): name we want to give on ovh
-    """
-    num_tries = 0
-    LIMIT_TRIES = 5
-    image_path = os.path.join(CLOUD_PATH, img_name)
-    start = time.time()
-
-    if not conn:
-        logger.exception("Variables not set for using OVH swift.", extra={
-            "bg_error_type": "NameError"
-        })
-        return
-
-    while num_tries <= LIMIT_TRIES:
-        num_tries += 1
-        extras_logging = {
-            "bg_date": datetime.now().isoformat(),
-            "bg_upload_time": time.time()-start,
-            "bg_image_url": image_path
-        }
-        try:
-            conn.put_object("basegun-public",
-                            f'uploaded-images/{os.environ["WORKSPACE"]}/{img_name}',
-                            contents=content)
-            # if success, get out of the loop
-            logger.info("Upload to OVH successful", extra=extras_logging)
-            break
-        except Exception as e:
-            if (num_tries <= LIMIT_TRIES and e.__class__.__name__ == "ClientException"):
-                # we try uploading another time
-                time.sleep(30)
-                continue
-            else:
-                extras_logging["bg_error_type"] = e.__class__.__name__
-                logger.exception(e, extra=extras_logging)
-
-
 ####################
 #     ROUTES       #
 ####################
@@ -232,17 +246,9 @@ async def imageupload(
 
     # prepare content logs
     user_agent = parse(request.headers.get("user-agent"))
-    extras_logging = {
-        "bg_date": datetime.now().isoformat(),
-        "bg_upload_time": round(time.time()-date, 2),
-        "bg_geolocation": geolocation,
-        "bg_device": get_device(user_agent),
-        "bg_device_family": user_agent.device.family,
-        "bg_device_os": user_agent.os.family,
-        "bg_device_browser": user_agent.browser.family,
-        "bg_version": APP_VERSION,
-        "bg_model": MODEL_VERSION,
-    }
+    extras_logging = get_base_logs(user_agent, user_id)
+    extras_logging["bg_geolocation"] = geolocation
+    extras_logging["bg_upload_time"] = round(time.time() - date, 2)
 
     try:
         img_name = str(uuid4()) + os.path.splitext(image.filename)[1]
@@ -257,7 +263,7 @@ async def imageupload(
         if not user_id:
             user_id = uuid4()
             response.set_cookie(key="user_id", value=user_id)
-        extras_logging["bg_user_id"] = user_id
+            extras_logging["bg_user_id"] = user_id
 
         # send image to model for prediction
         start = time.time()
@@ -287,24 +293,47 @@ async def imageupload(
         raise HTTPException(status_code=500, detail=str(e))
 
 
-@app.post("/feedback")
+@app.post("/identification-feedback")
 async def log_feedback(request: Request, user_id: Union[str, None] = Cookie(None)):
     res = await request.json()
+
     user_agent = parse(request.headers.get("user-agent"))
+    extras_logging = get_base_logs(user_agent, user_id)
+
+    extras_logging["bg_feedback_bool"] = res["feedback"]
+    for key in ["image_url", "label", "confidence", "confidence_level"]:
+        extras_logging["bg_"+key] = res[key]
 
-    extras_logging = {
-        "bg_date": datetime.now().isoformat(),
-        "bg_image_url": res["image_url"],
-        "bg_feedback_bool": res["feedback"],
-        "bg_label": res["label"],
-        "bg_confidence": res["confidence"],
-        "bg_confidence_level": res["confidence_level"],
-        "bg_user_id": user_id,
-        "bg_device": get_device(user_agent),
-        "bg_device_family": user_agent.device.family,
-        "bg_device_os": user_agent.os.family,
-        "bg_device_browser": user_agent.browser.family,
-        "bg_version": APP_VERSION,
-    }
     logger.info("Identification feedback", extra=extras_logging)
+    return
+
+
+@app.post("/tutorial-feedback")
+async def log_tutorial_feedback(request: Request, user_id: Union[str, None] = Cookie(None)):
+    res = await request.json()
+
+    user_agent = parse(request.headers.get("user-agent"))
+    extras_logging = get_base_logs(user_agent, user_id)
+
+    for key in ["image_url", "label", "confidence", "confidence_level",
+        "tutorial_feedback", "tutorial_option", "route_name"]:
+        extras_logging["bg_"+key] = res[key]
+
+    logger.info("Tutorial feedback", extra=extras_logging)
+    return
+
+
+@app.post("/identification-dummy")
+async def log_identification_dummy(request: Request, user_id: Union[str, None] = Cookie(None)):
+    res = await request.json()
+
+    user_agent = parse(request.headers.get("user-agent"))
+    extras_logging = get_base_logs(user_agent, user_id)
+
+    # to know if the firearm is dummy or real
+    extras_logging["bg_dummy_bool"] = res["is_dummy"]
+    for key in ["image_url", "label", "confidence", "confidence_level", "tutorial_option"]:
+        extras_logging["bg_"+key] = res[key]
+
+    logger.info("Identification dummy", extra=extras_logging)
     return

backend/tests/test_api.py

@@ -5,6 +5,7 @@
 import requests
 from PIL import Image, ImageChops
 
+
 class TestModel(unittest.TestCase):
     def __init__(self, *args, **kwargs):
         super(TestModel, self).__init__(*args, **kwargs)
@@ -21,6 +22,15 @@ def test_version(self):
         self.assertNotEqual(r.text, "-1")
         self.assertEqual(len(r.text.split('.')), 2) # checks version has format X.Y
 
+    def check_log_base(self, log):
+        self.assertTrue(
+            {'timestamp', 'version', 'host', 'level', 'short_message', '_bg_date', '_bg_user_id',
+                '_bg_device', '_bg_device_os', '_bg_device_family', '_bg_device_browser', '_bg_version',
+                '_bg_model'}.issubset(set(log.keys()))
+        )
+        self.assertEqual(log["level"], 6)
+        self.assertTrue(log["_bg_model"].startswith("EffB"))
+
     def test_upload_and_logs(self):
         """Checks that the file upload works properly"""
         path = os.path.join(
@@ -37,13 +47,12 @@ def test_upload_and_logs(self):
         res = r.json()
 
         # checks that the json result is as expected
-        self.assertEqual(set(res.keys()), set({"label", "confidence", "confidence_level", "path"}))
         self.assertEqual(res["label"], "revolver")
         self.assertAlmostEqual(res["confidence"], 99.53, places=1)
         self.assertTrue(res["confidence_level"], "high")
         self.assertTrue("ovh" in res["path"])
         # checks that written file is exactly the same as input file
-        time.sleep(30)
+        time.sleep(10)
         response = requests.get(res["path"])
         with Image.open(path) as image_one:
             with Image.open(BytesIO(response.content)) as image_two:
@@ -58,39 +67,31 @@ def test_upload_and_logs(self):
         self.assertEqual(r.json()[0]["short_message"], "Upload to OVH successful")
         # checks the previous log "Identification request"
         log = r.json()[1]
-        self.assertEqual(
-            set(log.keys()),
-            set({'timestamp', '_bg_device', 'host', '_bg_model_time', 'version', '_bg_device_os', '_bg_device_family',
-            'short_message', '_bg_confidence', '_bg_confidence_level', '_bg_upload_time', '_bg_date', '_bg_user_id', '_bg_label', '_bg_image_url',
-            'level', '_bg_geolocation', '_bg_device_browser', '_bg_version', '_bg_model'})
-        )
-        self.assertEqual(log["level"], 6)
+        self.check_log_base(log)
         self.assertEqual(log["short_message"], "Identification request")
         self.assertTrue("-" in log["_bg_user_id"])
         self.assertEqual(log["_bg_geolocation"], geoloc)
+        self.assertEqual(log["_bg_label"], "revolver")
+        self.assertAlmostEqual(log["_bg_confidence"], 99.53, places=1)
         self.assertTrue(log["_bg_upload_time"]>=0)
 
     def test_feedback_and_logs(self):
         """Checks that the feedback works properly"""
         confidence = 90
         label = "revolver"
         confidence_level = "high"
-        r = requests.post(self.url + "/feedback",
-                json={"image_url": "test", "feedback": False, "confidence": confidence, "label": label, "confidence_level": confidence_level})
+        image_url = "https://storage.gra.cloud.ovh.net/v1/test"
+        r = requests.post(self.url + "/identification-feedback",
+                json={"image_url": image_url, "feedback": True, "confidence": confidence, "label": label, "confidence_level": confidence_level})
+
         self.assertEqual(r.status_code, 200)
         r = requests.get(self.url + "/logs")
         self.assertEqual(r.status_code, 200)
         log = r.json()[0]
-        self.assertEqual(
-            set(log.keys()),
-            set({'timestamp', '_bg_device', 'host', '_bg_feedback_bool', 'version', '_bg_device_os', '_bg_device_family',
-            'short_message', '_bg_confidence', '_bg_confidence_level', '_bg_date', '_bg_user_id', '_bg_label', '_bg_image_url',
-            'level', '_bg_device_browser', '_bg_version'})
-        )
-        self.assertEqual(log["level"], 6)
+        self.check_log_base(log)
         self.assertEqual(log["short_message"], "Identification feedback")
-        self.assertEqual(log["_bg_image_url"], "test")
-        self.assertFalse(log["_bg_feedback_bool"])
+        self.assertEqual(log["_bg_image_url"], image_url)
+        self.assertTrue(log["_bg_feedback_bool"])
         self.assertEqual(log["_bg_confidence"], confidence)
         self.assertEqual(log["_bg_label"], label)
         self.assertEqual(log["_bg_confidence_level"], confidence_level)

docker-compose-dev.yml

@@ -20,7 +20,9 @@ services:
       - OS_PROJECT_NAME
       - http_proxy
       - https_proxy
+      - no_proxy
       - WORKSPACE=dev
+      - REQUESTS_CA_BUNDLE=$CACERT_LOCATION
     image: basegun-backend:${TAG}-dev
     ports:
       - 5000:5000

frontend/index.html

@@ -3,7 +3,7 @@
   <head>
     <meta charset="utf-8">
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
-    <meta name="viewport" content="width=device-width,initial-scale=1.0">
+    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, minimum-scale=1, user-scalable=no">
     <link rel="icon" href="/favicon.ico">
     <title>Basegun - beta</title>
     <meta name="description" content="Basegun">
@@ -17,7 +17,6 @@
       <strong>We're sorry but this app doesn't work properly without JavaScript enabled. Please enable it to continue.</strong>
     </noscript>
     <div id="app"></div>
-    <!-- built files will be auto injected -->
   </body>
 </html>
-<script type="module" src="/src/main.js"></script>
+<script type="module" src="./src/main.js"></script>