From d3095f9ea7eaf972df9b83aae8934426fcc7ed46 Mon Sep 17 00:00:00 2001 From: Thomas Bernard Date: Thu, 7 Sep 2023 14:45:55 +0200 Subject: [PATCH] Use s3 instead of swift --- backend/requirements/common.txt | 3 +- backend/src/main.py | 85 ++++++++------------------------- backend/tests/test_api.py | 52 ++++++++++++++------ docker-compose-dev.yml | 15 ++++-- 4 files changed, 70 insertions(+), 85 deletions(-) diff --git a/backend/requirements/common.txt b/backend/requirements/common.txt index d8d3424b..e2800d48 100644 --- a/backend/requirements/common.txt +++ b/backend/requirements/common.txt @@ -6,4 +6,5 @@ pyyaml>=5.4.1 user-agents==2.2.0 ua-parser==0.10.0 python-openstackclient==5.8.0 -python-swiftclient==4.0.0 \ No newline at end of file +python-swiftclient==4.0.0 +boto3==1.28.39 \ No newline at end of file diff --git a/backend/src/main.py b/backend/src/main.py index 360aaa62..926865e6 100644 --- a/backend/src/main.py +++ b/backend/src/main.py @@ -2,10 +2,13 @@ import os import sys import logging -from logging.handlers import TimedRotatingFileHandler -from datetime import datetime +import boto3 import time import json +import swiftclient + +from logging.handlers import TimedRotatingFileHandler +from datetime import datetime from uuid import uuid4 from typing import Union from fastapi import BackgroundTasks, Cookie, FastAPI, File, Form, HTTPException, Request, Response, UploadFile @@ -13,17 +16,13 @@ from fastapi.middleware.cors import CORSMiddleware from gelfformatter import GelfFormatter from user_agents import parse -import swiftclient from src.model import load_model_inference, predict_image + CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) WORKSPACE = os.environ.get("WORKSPACE") -CLOUD_PATH = f'https://storage.gra.cloud.ovh.net/v1/' + \ - 'AUTH_df731a99a3264215b973b3dee70a57af/basegun-public/' + \ - f'uploaded-images/{os.environ["WORKSPACE"]}/' - def init_variable(var_name: str, path: str) -> str: """Inits global variable for folder path @@ -90,48 +89,25 @@ def get_base_logs(user_agent, user_id) -> dict: return extras_logging -def upload_image_ovh(content: bytes, img_name: str): - """ Uploads an image to owh swift container basegun-public +def upload_image(content: bytes, image_key: str): + """ Uploads an image to s3 bucket path uploaded-images/WORKSPACE/img_name where WORKSPACE is dev, preprod or prod Args: content (bytes): file content - img_name (str): name we want to give on ovh + image_key (str): path we want to have """ - num_tries = 0 - LIMIT_TRIES = 5 - image_path = os.path.join(CLOUD_PATH, img_name) start = time.time() - - if not conn: - logger.exception("Variables not set for using OVH swift.", extra={ - "bg_error_type": "NameError" - }) - return - - while num_tries <= LIMIT_TRIES: - num_tries += 1 - extras_logging = { - "bg_date": datetime.now().isoformat(), - "bg_upload_time": time.time()-start, - "bg_image_url": image_path - } - try: - conn.put_object("basegun-public", - f'uploaded-images/{os.environ["WORKSPACE"]}/{img_name}', - contents=content) - # if success, get out of the loop - logger.info("Upload to OVH successful", extra=extras_logging) - break - except Exception as e: - if (num_tries <= LIMIT_TRIES and e.__class__.__name__ == "ClientException"): - # we try uploading another time - time.sleep(30) - continue - else: - extras_logging["bg_error_type"] = e.__class__.__name__ - logger.exception(e, extra=extras_logging) + s3 = boto3.resource('s3', endpoint_url=os.environ["S3_URL_ENDPOINT"]) + object = s3.Object(os.environ["S3_BUCKET_NAME"], image_key) + object.put(Body=content) + extras_logging = { + "bg_date": datetime.now().isoformat(), + "bg_upload_time": time.time()-start, + "bg_image_url": os.environ["S3_URL_ENDPOINT"] + "/" + os.environ["S3_BUCKET_NAME"] +image_key + } + logger.info("Upload successful", extra=extras_logging) #################### @@ -182,27 +158,6 @@ def upload_image_ovh(content: bytes, img_name: str): MODEL_VERSION = "-1" -conn = None -if all(var in os.environ for var in ["OS_USERNAME", "OS_PASSWORD", "OS_PROJECT_NAME"]) : - try: - # Connection to OVH cloud - conn = swiftclient.Connection( - authurl="https://auth.cloud.ovh.net/v3", - user=os.environ["OS_USERNAME"], - key=os.environ["OS_PASSWORD"], - os_options={ - "project_name": os.environ["OS_PROJECT_NAME"], - "region_name": "GRA" - }, - auth_version='3' - ) - conn.get_account() - except Exception as e: - logger.exception(e) -else: - logger.warn('Variables necessary for OVH connection not set !') - - #################### # ROUTES # #################### @@ -237,8 +192,8 @@ async def imageupload( img_bytes = image.file.read() # upload image to OVH Cloud - background_tasks.add_task(upload_image_ovh, img_bytes, img_name) - image_path = os.path.join(CLOUD_PATH, img_name) + background_tasks.add_task(upload_image, img_bytes, f"/uploaded-images/{os.environ['WORKSPACE']}/{img_name}") + image_path = f"{os.environ['S3_URL_ENDPOINT']}/{os.environ['S3_BUCKET_NAME']}/uploaded-images/{os.environ['WORKSPACE']}/{img_name}" extras_logging["bg_image_url"] = image_path # set user id diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index d833518b..f8069e19 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -1,26 +1,50 @@ import unittest import os import time +import boto3 +import json + from io import BytesIO import requests from PIL import Image, ImageChops +from src.main import app +from fastapi import FastAPI +from fastapi.testclient import TestClient -class TestModel(unittest.TestCase): - def __init__(self, *args, **kwargs): - super(TestModel, self).__init__(*args, **kwargs) - self.url = "http://localhost:8000" +client = TestClient(app) + + +BUCKET_POLICY = { + 'Version': '2012-10-17', + 'Statement': [{ + 'Sid': 'AddPerm', + 'Effect': 'Allow', + 'Principal': '*', + 'Action': ['s3:GetObject'], + 'Resource': f"arn:aws:s3:::{os.environ['S3_BUCKET_NAME']}/*" + }] +} +def create_bucket(): + s3 = boto3.resource('s3', endpoint_url=os.environ["S3_URL_ENDPOINT"]) + bucket = s3.Bucket(os.environ["S3_BUCKET_NAME"]) + if bucket.creation_date is None: + bucket.create() + bucket.Policy().put(Policy=json.dumps(BUCKET_POLICY)) + + +class TestModel(unittest.TestCase): def test_home(self): """Checks that the route / is alive""" - r = requests.get(self.url) - self.assertEqual(r.text, "Basegun backend") + response = client.get("/") + self.assertEqual(response.text, "Basegun backend") def test_version(self): """Checks that the route /version sends a version""" - r = requests.get(self.url + '/version') - self.assertNotEqual(r.text, "-1") - self.assertEqual(len(r.text.split('.')), 2) # checks version has format X.Y + response = client.get("/version") + self.assertNotEqual(response.text, "-1") + self.assertEqual(len(response.text.split('.')), 2) # checks version has format X.Y def check_log_base(self, log): self.assertTrue( @@ -31,16 +55,16 @@ def check_log_base(self, log): self.assertEqual(log["level"], 6) self.assertTrue(log["_bg_model"].startswith("EffB")) - def test_upload_and_logs(self): + def test_upload(self): """Checks that the file upload works properly""" + create_bucket() path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "revolver.jpg") geoloc = "12.666,7.666" - self.assertTrue("OS_USERNAME" in os.environ) with open(path, 'rb') as f: - r = requests.post(self.url + "/upload", + r = client.post("/upload", files={"image": f}, data={"date": time.time(), "geolocation": geoloc}) self.assertEqual(r.status_code, 200) @@ -50,9 +74,7 @@ def test_upload_and_logs(self): self.assertEqual(res["label"], "revolver") self.assertAlmostEqual(res["confidence"], 98.43, places=1) self.assertTrue(res["confidence_level"], "high") - self.assertTrue("ovh" in res["path"]) # checks that written file is exactly the same as input file - time.sleep(10) response = requests.get(res["path"]) with Image.open(path) as image_one: with Image.open(BytesIO(response.content)) as image_two: @@ -66,7 +88,7 @@ def test_feedback_and_logs(self): label = "revolver" confidence_level = "high" image_url = "https://storage.gra.cloud.ovh.net/v1/test" - r = requests.post(self.url + "/identification-feedback", + r = client.post("/identification-feedback", json={"image_url": image_url, "feedback": True, "confidence": confidence, "label": label, "confidence_level": confidence_level}) self.assertEqual(r.status_code, 200) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index d27ca20b..ea5e023b 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -13,10 +13,10 @@ services: container_name: basegun-backend user: appuser environment: - - PATH_LOGS=/tmp/logs - - OS_USERNAME - - OS_PASSWORD - - OS_PROJECT_NAME + - S3_URL_ENDPOINT=http://minio:9000 + - S3_BUCKET_NAME=basegun-s3 + - AWS_ACCESS_KEY_ID=minioadmin + - AWS_SECRET_ACCESS_KEY=minioadmin - http_proxy - https_proxy - UVICORN_LOG_LEVEL=${UVICORN_LOG_LEVEL} @@ -51,3 +51,10 @@ services: volumes: - $PWD/frontend/src:/app/src - /app/node_modules + + minio: + image: minio/minio + command: server /data --console-address ":9001" + ports: + - 9000:9000 + - 9001:9001