Use s3 instead of swift

backend/requirements/common.txt

@@ -6,4 +6,5 @@ pyyaml>=5.4.1
 user-agents==2.2.0
 ua-parser==0.10.0
 python-openstackclient==5.8.0
-python-swiftclient==4.0.0
+python-swiftclient==4.0.0
+boto3==1.28.39

backend/src/main.py

@@ -2,28 +2,27 @@
 import os
 import sys
 import logging
-from logging.handlers import TimedRotatingFileHandler
-from datetime import datetime
+import boto3
 import time
 import json
+import swiftclient
+
+from logging.handlers import TimedRotatingFileHandler
+from datetime import datetime
 from uuid import uuid4
 from typing import Union
 from fastapi import BackgroundTasks, Cookie, FastAPI, File, Form, HTTPException, Request, Response, UploadFile
 from fastapi.responses import PlainTextResponse
 from fastapi.middleware.cors import CORSMiddleware
 from gelfformatter import GelfFormatter
 from user_agents import parse
-import swiftclient
 from src.model import load_model_inference, predict_image
 
 
+
 CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
 WORKSPACE = os.environ.get("WORKSPACE")
 
-CLOUD_PATH = f'https://storage.gra.cloud.ovh.net/v1/' + \
-    'AUTH_df731a99a3264215b973b3dee70a57af/basegun-public/' + \
-    f'uploaded-images/{os.environ["WORKSPACE"]}/'
-
 
 def init_variable(var_name: str, path: str) -> str:
     """Inits global variable for folder path
@@ -90,48 +89,25 @@ def get_base_logs(user_agent, user_id) -> dict:
     return extras_logging
 
 
-def upload_image_ovh(content: bytes, img_name: str):
-    """ Uploads an image to owh swift container basegun-public
+def upload_image(content: bytes, image_key: str):
+    """ Uploads an image to s3 bucket
         path uploaded-images/WORKSPACE/img_name
         where WORKSPACE is dev, preprod or prod
 
     Args:
         content (bytes): file content
-        img_name (str): name we want to give on ovh
+        image_key (str): path we want to have
     """
-    num_tries = 0
-    LIMIT_TRIES = 5
-    image_path = os.path.join(CLOUD_PATH, img_name)
     start = time.time()
-
-    if not conn:
-        logger.exception("Variables not set for using OVH swift.", extra={
-            "bg_error_type": "NameError"
-        })
-        return
-
-    while num_tries <= LIMIT_TRIES:
-        num_tries += 1
-        extras_logging = {
-            "bg_date": datetime.now().isoformat(),
-            "bg_upload_time": time.time()-start,
-            "bg_image_url": image_path
-        }
-        try:
-            conn.put_object("basegun-public",
-                            f'uploaded-images/{os.environ["WORKSPACE"]}/{img_name}',
-                            contents=content)
-            # if success, get out of the loop
-            logger.info("Upload to OVH successful", extra=extras_logging)
-            break
-        except Exception as e:
-            if (num_tries <= LIMIT_TRIES and e.__class__.__name__ == "ClientException"):
-                # we try uploading another time
-                time.sleep(30)
-                continue
-            else:
-                extras_logging["bg_error_type"] = e.__class__.__name__
-                logger.exception(e, extra=extras_logging)
+    s3 = boto3.resource('s3', endpoint_url=os.environ["S3_URL_ENDPOINT"])
+    object = s3.Object(os.environ["S3_BUCKET_NAME"], image_key)
+    object.put(Body=content)
+    extras_logging = {
+        "bg_date": datetime.now().isoformat(),
+        "bg_upload_time": time.time()-start,
+        "bg_image_url": os.environ["S3_URL_ENDPOINT"] + "/" + os.environ["S3_BUCKET_NAME"] +image_key
+    }
+    logger.info("Upload successful", extra=extras_logging)
 
 
 ####################
@@ -182,27 +158,6 @@ def upload_image_ovh(content: bytes, img_name: str):
     MODEL_VERSION = "-1"
 
 
-conn = None
-if all(var in os.environ for var in ["OS_USERNAME", "OS_PASSWORD", "OS_PROJECT_NAME"]) :
-    try:
-        # Connection to OVH cloud
-        conn = swiftclient.Connection(
-            authurl="https://auth.cloud.ovh.net/v3",
-            user=os.environ["OS_USERNAME"],
-            key=os.environ["OS_PASSWORD"],
-            os_options={
-                "project_name": os.environ["OS_PROJECT_NAME"],
-                "region_name": "GRA"
-            },
-            auth_version='3'
-        )
-        conn.get_account()
-    except Exception as e:
-        logger.exception(e)
-else:
-    logger.warn('Variables necessary for OVH connection not set !')
-
-
 ####################
 #     ROUTES       #
 ####################
@@ -237,8 +192,8 @@ async def imageupload(
         img_bytes = image.file.read()
 
         # upload image to OVH Cloud
-        background_tasks.add_task(upload_image_ovh, img_bytes, img_name)
-        image_path = os.path.join(CLOUD_PATH, img_name)
+        background_tasks.add_task(upload_image, img_bytes, f"/uploaded-images/{os.environ['WORKSPACE']}/{img_name}")
+        image_path = f"{os.environ['S3_URL_ENDPOINT']}/{os.environ['S3_BUCKET_NAME']}/uploaded-images/{os.environ['WORKSPACE']}/{img_name}"
         extras_logging["bg_image_url"] = image_path
 
         # set user id

backend/tests/test_api.py

@@ -1,26 +1,50 @@
 import unittest
 import os
 import time
+import boto3
+import json
+
 from io import BytesIO
 import requests
 from PIL import Image, ImageChops
+from src.main import app
 
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
 
-class TestModel(unittest.TestCase):
-    def __init__(self, *args, **kwargs):
-        super(TestModel, self).__init__(*args, **kwargs)
-        self.url = "http://localhost:8000"
+client = TestClient(app)
+
+
+BUCKET_POLICY = {
+    'Version': '2012-10-17',
+    'Statement': [{
+        'Sid': 'AddPerm',
+        'Effect': 'Allow',
+        'Principal': '*',
+        'Action': ['s3:GetObject'],
+        'Resource': f"arn:aws:s3:::{os.environ['S3_BUCKET_NAME']}/*"
+    }]
+}
 
+def create_bucket():
+    s3 = boto3.resource('s3', endpoint_url=os.environ["S3_URL_ENDPOINT"])
+    bucket = s3.Bucket(os.environ["S3_BUCKET_NAME"])
+    if bucket.creation_date is None:
+        bucket.create()
+        bucket.Policy().put(Policy=json.dumps(BUCKET_POLICY))
+
+
+class TestModel(unittest.TestCase):
     def test_home(self):
         """Checks that the route / is alive"""
-        r = requests.get(self.url)
-        self.assertEqual(r.text, "Basegun backend")
+        response = client.get("/")
+        self.assertEqual(response.text, "Basegun backend")
 
     def test_version(self):
         """Checks that the route /version sends a version"""
-        r = requests.get(self.url + '/version')
-        self.assertNotEqual(r.text, "-1")
-        self.assertEqual(len(r.text.split('.')), 2) # checks version has format X.Y
+        response = client.get("/version")
+        self.assertNotEqual(response.text, "-1")
+        self.assertEqual(len(response.text.split('.')), 2) # checks version has format X.Y
 
     def check_log_base(self, log):
         self.assertTrue(
@@ -31,16 +55,16 @@ def check_log_base(self, log):
         self.assertEqual(log["level"], 6)
         self.assertTrue(log["_bg_model"].startswith("EffB"))
 
-    def test_upload_and_logs(self):
+    def test_upload(self):
         """Checks that the file upload works properly"""
+        create_bucket()
         path = os.path.join(
             os.path.dirname(os.path.abspath(__file__)),
             "revolver.jpg")
         geoloc = "12.666,7.666"
 
-        self.assertTrue("OS_USERNAME" in os.environ)
         with open(path, 'rb') as f:
-            r = requests.post(self.url + "/upload",
+            r = client.post("/upload",
                 files={"image": f},
                 data={"date": time.time(), "geolocation": geoloc})
         self.assertEqual(r.status_code, 200)
@@ -50,9 +74,7 @@ def test_upload_and_logs(self):
         self.assertEqual(res["label"], "revolver")
         self.assertAlmostEqual(res["confidence"], 98.43, places=1)
         self.assertTrue(res["confidence_level"], "high")
-        self.assertTrue("ovh" in res["path"])
         # checks that written file is exactly the same as input file
-        time.sleep(10)
         response = requests.get(res["path"])
         with Image.open(path) as image_one:
             with Image.open(BytesIO(response.content)) as image_two:
@@ -66,7 +88,7 @@ def test_feedback_and_logs(self):
         label = "revolver"
         confidence_level = "high"
         image_url = "https://storage.gra.cloud.ovh.net/v1/test"
-        r = requests.post(self.url + "/identification-feedback",
+        r = client.post("/identification-feedback",
                 json={"image_url": image_url, "feedback": True, "confidence": confidence, "label": label, "confidence_level": confidence_level})
         self.assertEqual(r.status_code, 200)
 

docker-compose-dev.yml

@@ -13,10 +13,10 @@ services:
     container_name: basegun-backend
     user: appuser
     environment:
-      - PATH_LOGS=/tmp/logs
-      - OS_USERNAME
-      - OS_PASSWORD
-      - OS_PROJECT_NAME
+      - S3_URL_ENDPOINT=http://minio:9000
+      - S3_BUCKET_NAME=basegun-s3
+      - AWS_ACCESS_KEY_ID=minioadmin
+      - AWS_SECRET_ACCESS_KEY=minioadmin
       - http_proxy
       - https_proxy
       - UVICORN_LOG_LEVEL=${UVICORN_LOG_LEVEL}
@@ -51,3 +51,10 @@ services:
     volumes:
       - $PWD/frontend/src:/app/src
       - /app/node_modules
+
+  minio:
+    image: minio/minio
+    command: server /data --console-address ":9001"
+    ports:
+      - 9000:9000
+      - 9001:9001