Merge branch 'develop' into feat/add-tutorial-to-secure-a-firearm

.github/workflows/build-filebeat-rootless.yml

@@ -0,0 +1,25 @@
+name: Build and push Filebeat rootless image
+
+on: workflow_dispatch
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: Login to GHCR
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        with:
+          file: ./infra/filebeat.Dockerfile
+          push: true
+          tags: ghcr.io/datalab-mi/basegun/filebeat-rootless:6.5.4

.github/workflows/dso-automatic-sync.yml

@@ -0,0 +1,18 @@
+name: Sync DSO gitlab repository
+
+on:
+  workflow_dispatch:
+  push:
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    steps:
+      - run: |
+          curl "https://gitlab.apps.c6.numerique-interieur.com/api/v4/projects/62/trigger/pipeline" \
+            -X POST \
+            --fail \
+            -F token=${{ secrets.DSO_TOKEN }} \
+            -F ref=main \
+            -F variables[GIT_BRANCH_DEPLOY]=${{ github.ref_name }} \
+            -F variables[PROJECT_NAME]=infra

.github/workflows/pr.yml

@@ -22,7 +22,6 @@ jobs:
     needs: tag-pr
     secrets:
       API_OVH_TOKEN: ${{ secrets.API_OVH_TOKEN }}
-      OS_PASSWORD: ${{ secrets.OS_PASSWORD }}
-      OS_PROJECT_NAME: ${{ secrets.OS_PROJECT_NAME }}
-      OS_USERNAME: ${{ secrets.OS_USERNAME }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       X_OVH_TOKEN: ${{ secrets.PREPROD_OVH_TOKEN }}

.github/workflows/preprod.yml

@@ -6,10 +6,10 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Build the docker images
-      run: make CONTINUE=y build-prod
+      run: make build-prod
     - name: Test running the docker containers
       run: |
-        make CONTINUE=y up-prod
+        make up-prod
         ./infra/scripts/test-alive.sh localhost
         make down-prod
     - name: Login to GHCR
@@ -50,12 +50,11 @@ jobs:
       domain: preprod.basegun.fr
     secrets:
       API_OVH_TOKEN: ${{ secrets.API_OVH_TOKEN }}
-      OS_PASSWORD: ${{ secrets.OS_PASSWORD }}
-      OS_PROJECT_NAME: ${{ secrets.OS_PROJECT_NAME }}
-      OS_USERNAME: ${{ secrets.OS_USERNAME }}
-      X_OVH_TOKEN: ${{ secrets.PREPROD_OVH_TOKEN }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       JOB_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       KUBECONFIG: ${{ secrets.PREPROD_K8_CONFIG }}
+      X_OVH_TOKEN: ${{ secrets.PREPROD_OVH_TOKEN }}
 
   test:
     runs-on: ubuntu-latest

.github/workflows/prod.yml

@@ -25,10 +25,10 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Build the Docker image
-      run: make CONTINUE=y build-prod
+      run: make build-prod
     - name: Test running the docker containers
       run: |
-        make CONTINUE=y up-prod
+        make up-prod
         ./infra/scripts/test-alive.sh localhost
         make down-prod
     - name: Login to GHCR
@@ -69,12 +69,11 @@ jobs:
       domain: basegun.fr
     secrets:
       API_OVH_TOKEN: ${{ secrets.API_OVH_TOKEN }}
-      OS_PASSWORD: ${{ secrets.OS_PASSWORD }}
-      OS_PROJECT_NAME: ${{ secrets.OS_PROJECT_NAME }}
-      OS_USERNAME: ${{ secrets.OS_USERNAME }}
-      X_OVH_TOKEN: ${{ secrets.PROD_OVH_TOKEN }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       JOB_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      KUBECONFIG: ${{ secrets.PROD_K8_CONFIG }}
+      KUBECONFIG: ${{ secrets.PREPROD_K8_CONFIG }}
+      X_OVH_TOKEN: ${{ secrets.PREPROD_OVH_TOKEN }}
 
   test:
     runs-on: ubuntu-latest

.github/workflows/test-on-kube.yml

@@ -5,11 +5,9 @@ on:
     secrets:
       API_OVH_TOKEN:
         required: true
-      OS_PASSWORD:
+      AWS_ACCESS_KEY_ID:
         required: true
-      OS_PROJECT_NAME:
-        required: true
-      OS_USERNAME:
+      AWS_SECRET_ACCESS_KEY:
         required: true
       X_OVH_TOKEN:
         required: true
@@ -32,11 +30,6 @@ jobs:
           wait: 60s
           verbosity: 2
 
-      - name: Set up Helm
-        uses: azure/setup-helm@v3
-        with:
-          version: v3.11.2
-
       - name: Set up ingress controller 
         run: |
           helm repo add traefik https://traefik.github.io/charts && helm repo update
@@ -52,21 +45,20 @@ jobs:
         run: |
           TAG=$(make get-current-tag) BUILD_TARGET=test docker-compose -f docker-compose-prod.yml build
           kind load docker-image \
-            basegun-backend:$(make get-current-tag)-prod \
-            basegun-frontend:$(make get-current-tag)-prod \
+            basegun-backend:$(make get-current-tag) \
+            basegun-frontend:$(make get-current-tag) \
             --name basegun-testing
           helm upgrade --install basegun ./infra/kube/helm/  \
             --set ingress.hosts[0].host="$LOCAL_DOMAIN" \
             --set ingress.hosts[0].paths[0].path="/" \
             --set ingress.hosts[0].paths[0].pathType="Prefix"  \
             --set backend.image.repository="basegun-backend" \
-            --set backend.image.tag="$(make get-current-tag)-prod" \
+            --set backend.image.tag="$(make get-current-tag)" \
             --set frontend.image.repository="basegun-frontend" \
-            --set frontend.image.tag="$(make get-current-tag)-prod" \
+            --set frontend.image.tag="$(make get-current-tag)" \
             --set backend.secret.create="true" \
-            --set-string backend.secret.values.OS_USERNAME="${{ secrets.OS_USERNAME }}" \
-            --set-string backend.secret.values.OS_PASSWORD="${{ secrets.OS_PASSWORD }}" \
-            --set-string backend.secret.values.OS_PROJECT_NAME="${{ secrets.OS_PROJECT_NAME }}" \
+            --set-string backend.secret.values.AWS_ACCESS_KEY_ID="${{ secrets.AWS_ACCESS_KEY_ID }}" \
+            --set-string backend.secret.values.AWS_SECRET_ACCESS_KEY="${{ secrets.AWS_SECRET_ACCESS_KEY }}" \
             --set-string backend.secret.values.X_OVH_TOKEN="${{ secrets.X_OVH_TOKEN }}" \
             --set-string backend.secret.values.API_OVH_TOKEN="${{ secrets.API_OVH_TOKEN }}"
           for i in $(kubectl get deploy -o name); do kubectl rollout status $i -w --timeout=130s; done

.gitignore

@@ -11,7 +11,7 @@ logs/
 temp/
 *openrc.sh
 .terraform*
-.secrets
+*.secrets*
 version.txt
 *.crt
 *.pem

.gitlab-ci-dso.yml

@@ -0,0 +1,43 @@
+include:
+  - project: $CATALOG_PATH
+    file: vault-ci.yml
+    ref: main
+  - project: $CATALOG_PATH
+    file: kaniko-ci.yml
+    ref: main
+
+default:
+  image: alpine:latest
+
+variables:
+  TAG: "${CI_COMMIT_REF_SLUG}"
+  REGISTRY_URL: "${REGISTRY_HOST}/${PROJECT_PATH}"
+
+stages:
+  - read-secret
+  - docker-build
+
+read_secret:
+  stage: read-secret
+  extends:
+    - .vault:read_secret
+
+docker-build-frontend:
+  variables:
+    WORKING_DIR: "frontend"
+    IMAGE_NAMES: basegun-frontend:${CI_COMMIT_REF_SLUG} basegun-frontend:${CI_COMMIT_SHORT_SHA}
+    DOCKERFILE: "frontend/Dockerfile"
+    BUILD_ARGS: --build-arg=VERSION=1.1.1
+  stage: docker-build
+  extends:
+    - .kaniko:build-push
+
+docker-build-backend:
+  variables:
+    WORKING_DIR: "backend"
+    IMAGE_NAMES: basegun-backend:${CI_COMMIT_REF_SLUG} basegun-backend:${CI_COMMIT_SHORT_SHA}
+    DOCKERFILE: "backend/Dockerfile"
+    BUILD_ARGS: --build-arg=VERSION=1.1.1
+  stage: docker-build
+  extends:
+    - .kaniko:build-push

Makefile

@@ -1,7 +1,7 @@
 SHELL	:= /bin/bash
 DOCKER	:= $(shell type -p docker)
 DC		:= $(shell type -p docker-compose)
-TAG		:= 3.2
+TAG		:= 3.3
 APP_NAME	:= basegun
 REG		:= ghcr.io
 ORG		:= datalab-mi
@@ -29,10 +29,10 @@ endif
 check-dc-config-%: check-prerequisites ## Check docker-compose syntax
 	${DC} -f docker-compose-$*.yml config -q
 
-build-%: check-dc-config-% show-current-tag
+build-%: check-dc-config-%
 	TAG=${TAG} ${DC} -f docker-compose-$*.yml build
 
-up-%: check-dc-config-% show-current-tag
+up-%: check-dc-config-%
 ifeq ("$(WORKSPACE)","preprod")
 	TAG=${TAG} PORT_PROD=8080 ${DC} -f docker-compose-$*.yml up -d
 else

backend/Dockerfile

@@ -14,10 +14,10 @@ RUN apt update && apt install -y \
     && rm -rf /var/lib/apt/lists/*
 
 # install python libraries (except torch)
-COPY requirements/ requirements/
+COPY requirements.txt .
 ENV PIP_CERT=$CACERT_LOCATION
 RUN pip --default-timeout=300 install --upgrade pip \
-    && pip --default-timeout=300 install --no-cache-dir -r requirements/common.txt \
+    && pip --default-timeout=300 install --no-cache-dir -r requirements.txt \
     && rm -r /root/.cache
 
 ARG VERSION
@@ -31,14 +31,14 @@ RUN mkdir -p src/weights \
 
 # launch website
 FROM base as dev
-RUN pip --default-timeout=300 install --no-cache-dir -r requirements/dev.txt
 CMD ["uvicorn", "src.main:app", "--reload", "--host", "0.0.0.0", "--port", "5000"]
 
 FROM base as test
-RUN pip install -r requirements/dev.txt && pip install requests && rm -r /root/.cache
+RUN pip install requests && rm -r /root/.cache
 COPY tests/ tests/
 CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "5000"]
 
 FROM base as prod
-RUN pip install --no-cache-dir -r requirements/prod.txt
+RUN pip install --extra-index-url https://download.pytorch.org/whl/cpu \
+    torch==1.13.0+cpu torchvision==0.14.0+cpu && rm -r /root/.cache
 CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "5000"]

backend/requirements.txt

@@ -0,0 +1,11 @@
+--extra-index-url https://download.pytorch.org/whl/cpu
+fastapi==0.68.0
+uvicorn==0.14.0
+python-multipart>=0.0.5
+gelf-formatter==0.2.1
+pyyaml>=5.4.1
+user-agents==2.2.0
+ua-parser==0.10.0
+boto3==1.28.39
+torch==1.13.0
+torchvision==0.14.0