From c73d8552a097e35873e66e316f11b06c6d329db4 Mon Sep 17 00:00:00 2001 From: "malika.wannasi" Date: Wed, 10 Apr 2024 11:21:09 +0200 Subject: [PATCH 01/44] traduire circleci dans all and configure other yml --- .github/workflows/all.yml | 255 +++++++++++++++++++++++++ .github/workflows/docker-hub.yml | 104 ++++++++++ .github/workflows/magnify-frontend.yml | 152 +++++++++++++++ .github/workflows/magnify.yml | 199 +++++++++++++++++++ .github/workflows/secrets.enc.env | 22 +++ 5 files changed, 732 insertions(+) create mode 100644 .github/workflows/all.yml create mode 100644 .github/workflows/docker-hub.yml create mode 100644 .github/workflows/magnify-frontend.yml create mode 100644 .github/workflows/magnify.yml create mode 100644 .github/workflows/secrets.enc.env diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml new file mode 100644 index 000000000..94d87deee --- /dev/null +++ b/.github/workflows/all.yml @@ -0,0 +1,255 @@ +# Ancres du fichier de configuration +generate-version-file: &generate-version-file + run: + name: Créer un fichier version.json + command: | + # Créer un fichier version.json à la manière de Mozilla + # https://github.com/mozilla-services/Dockerflow/blob/master/docs/version_object.md + printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ + "$CIRCLE_SHA1" \ + "$CIRCLE_TAG" \ + "$CIRCLE_PROJECT_USERNAME" \ + "$CIRCLE_PROJECT_REPONAME" \ + "$CIRCLE_BUILD_URL" > sandbox/version.json + +version: 2 + +aliases: + - &checkout_fun + checkout: + path: ~/fun + + - &restore_node_modules + restore_cache: + name: Restaurer le cache node_modules + keys: + - v18-front-dependencies-{{ checksum "~/fun/src/frontend/yarn.lock" }} + - v18-front-dependencies- + +jobs: + # Tâches Git + # Vérifier que l'historique git est propre et conforme à nos attentes + lint-git: + docker: + - image: cimg/python:3.10 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + working_directory: ~/fun + steps: + - checkout + # S'assurer que les modifications n'ajoutent pas une instruction "print" à la base de code. + # Nous devons exclure le dossier ".circleci" de la recherche car la commande même qui vérifie + # l'absence de "print" inclut un "print(" lui-même. + - run: + name: Enforcer l'absence d'instructions print dans le code + command: | + ! git diff origin/main..HEAD -- ":(exclude)*.circleci/*" | grep "print(" + - run: + name: Vérifier l'absence de commits de correction (fixup) + command: | + ! git log | grep 'fixup!' + - run: + name: Installer gitlint + command: | + pip install --user requests gitlint + - run: + name: Linter les messages de commit ajoutés à main + command: | + ~/.local/bin/gitlint --commits origin/main..HEAD + + # Vérifier que le CHANGELOG a été mis à jour dans la branche actuelle + check-changelog: + docker: + - image: circleci/buildpack-deps:stretch-scm + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + working_directory: ~/fun + steps: + - checkout + - run: + name: Vérifier que le CHANGELOG a été modifié dans la branche actuelle + command: | + git whatchanged --name-only --pretty="" origin..HEAD | grep CHANGELOG + + # Vérifier que la longueur maximale des lignes du CHANGELOG ne dépasse pas 80 caractères + lint-changelog: + docker: + - image: debian:stretch + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + working_directory: ~/fun + steps: + - checkout + - run: + name: Vérifier la longueur maximale des lignes du CHANGELOG + command: | + # Obtenir la largeur de la ligne la plus longue (en ignorant les liens de publication) + test $(cat CHANGELOG.md | grep -Ev "^\[.*\]: https://github.com/openfun" | wc -L) -le 80 + + # Vérifier que le fichier de configuration de Renovate est valide + check-renovate-configuration: + docker: + - image: renovate/renovate + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + working_directory: ~/fun + steps: + - checkout + - run: + name: Exécuter la commande renovate-config-validator + command: renovate-config-validator + + # Vérifier que toutes les versions (backend, frontend) sont à jour + check-versions: + docker: + - image: cimg/base:2022.04 + working_directory: ~/fun + steps: + - checkout + - run: + name: Vérifier que toutes les versions sont identiques + command: | + BACKEND_VERSION=$(cat setup.cfg | grep "version" | cut -d" " -f3) + echo "Version magnify : ${BACKEND_VERSION}" + # Dans l'espace de travail frontend + cat src/frontend/package.json | grep "\"version\": \"${BACKEND_VERSION}\",$" + # Dans la bibliothèque frontend "@openfun/jitsi-magnify" + cat src/frontend/packages/core/package.json | grep "\"version\": \"${BACKEND_VERSION}\",$" + # Dans l'application frontend "sandbox" + cat src/frontend/sandbox/package.json | grep "\"version\": \"${BACKEND_VERSION}\",$" + + # ---- Tâches Docker ---- + # Construire l'image Docker prête pour la production + build-docker: + docker: + - image: circleci/buildpack-deps:stretch + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + working_directory: ~/fun + steps: + # Récupérer les sources du dépôt + - checkout + # Générer un fichier version.json décrivant la version de l'application + - <<: *generate-version-file + # Activer docker-in-docker + - setup_remote_docker: + version: 19.03.13 + + # Se connecter à Docker Hub avec des identifiants cryptés stockés en tant que secret + # variables d'environnement (définies dans les paramètres du projet CircleCI) si la variable + # d'environnement attendue est définie; passer en mode anonyme sinon. + - run: + name: Se connecter à DockerHub + command: > + test -n "$DOCKER_USER" && + echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin || + echo "Mode anonyme Docker Hub" + + # Chaque image est marquée avec le hachage du commit git actuel pour éviter + # les collisions dans les constructions parallèles. + - run: + name: Construire l'image de production + command: docker build -t magnify:${CIRCLE_SHA1} --target production . + - run: + name: Vérifier la disponibilité de l'image construite + command: docker images "magnify:${CIRCLE_SHA1}*" + + # ---- Tâches Backend ---- + # Construire l'environnement de développement backend + build-back: + docker: + - image: cimg/python:3.10 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + working_directory: ~/fun + steps: + - checkout + # Restaurer le cache des dépendances Python + - restore_cache: + name: Restaurer le cache des dépendances Python + keys: + - v18-back-dependencies-{{ checksum "~/fun/setup.cfg" }} + - v18-back-dependencies- + - run: + name: Installer les dépendances du backend + command: pip install -r requirements-dev.txt + # Sauvegarder le cache des dépendances Python + - save_cache: + name: Sauvegarder le cache des dépendances Python + key: v18-back-dependencies-{{ checksum "~/fun/setup.cfg" }} + paths: + - ~/.cache/pip + - ./venv/lib/python*/site-packages/ + - run: + name: Vérifier la compatibilité entre les versions des bibliothèques Python et Black + command: pip check + + # ---- Tâches Frontend ---- + # Installer les dépendances du frontend et les outils de développement + build-front: + docker: + - image: cimg/node:17.3 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + working_directory: ~/fun + steps: + - checkout + # Restaurer le cache des dépendances Node.js + - restore_cache: + name: Restaurer le cache des dépendances Node.js + <<: *restore_node_modules + - run: + name: Installer les dépendances Node.js + command: yarn install --frozen-lockfile --non-interactive + # Sauvegarder le cache des dépendances Node.js + - save_cache: + name: Sauvegarder le cache des dépendances Node.js + key: v18-front-dependencies-{{ checksum "~/fun/src/frontend/yarn.lock" }} + paths: + - ~/.cache/yarn + - ~/.cache/Cypress + - ./src/frontend/node_modules + - run: + name: Vérifier la compatibilité entre les dépendances du frontend et ESLint + command: yarn run eslint --no-error-on-unmatched-pattern + +workflows: + version: 2 + lint-git: + jobs: + - lint-git + + check-changelog: + jobs: + - check-changelog + + lint-changelog: + jobs: + - lint-changelog + + check-renovate-configuration: + jobs: + - check-renovate-configuration + + check-versions: + jobs: + - check-versions + + build-docker: + jobs: + - build-docker + + build-back: + jobs: + - build-back + + build-front: + jobs: + - build-front diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml new file mode 100644 index 000000000..425bb8bfa --- /dev/null +++ b/.github/workflows/docker-hub.yml @@ -0,0 +1,104 @@ +name: Docker Hub Workflow + +on: + workflow_dispatch: + push: + branches: + - 'main' + tags: + - 'v*' + pull_request: + branches: + - 'main' + +env: + DOCKER_USER: 1001:127 + +jobs: + build-and-push-backend: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v4 + - + name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: lasuite/magnify-backend + - + name: Load sops secrets + uses: rouja/actions-sops@main + with: + secret-file: .github/workflows/secrets.enc.env + age-key: ${{ secrets.SOPS_PRIVATE }} + - + name: Login to DockerHub + if: github.event_name != 'pull_request' + run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin + - + name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + target: backend-production + build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + build-and-push-frontend: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v4 + - + name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: lasuite/magnify-frontend + - + name: Load sops secrets + uses: rouja/actions-sops@main + with: + secret-file: .github/workflows/secrets.enc.env + age-key: ${{ secrets.SOPS_PRIVATE }} + - + name: Login to DockerHub + if: github.event_name != 'pull_request' + run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin + - + name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + target: frontend-production + build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + notify-argocd: + needs: + - build-and-push-frontend + - build-and-push-backend + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v4 + - + name: Load sops secrets + uses: rouja/actions-sops@main + with: + secret-file: .github/workflows/secrets.enc.env + age-key: ${{ secrets.SOPS_PRIVATE }} + - + name: Call argocd github webhook + run: | + data='{"ref": "'$GITHUB_REF'","repository": {"html_url":"'$GITHUB_SERVER_URL'/'$GITHUB_REPOSITORY'"}}' + sig=$(echo -n ${data} | openssl dgst -sha1 -hmac ''${ARGOCD_WEBHOOK_SECRET}'' | awk '{print "X-Hub-Signature: sha1="$2}') + curl -X POST -H 'X-GitHub-Event:push' -H "Content-Type: application/json" -H "${sig}" --data "${data}" $ARGOCD_WEBHOOK_URL diff --git a/.github/workflows/magnify-frontend.yml b/.github/workflows/magnify-frontend.yml new file mode 100644 index 000000000..ac408e353 --- /dev/null +++ b/.github/workflows/magnify-frontend.yml @@ -0,0 +1,152 @@ +name: magnify Workflow + +on: + push: + branches: + - main + tags: + - 'v*' + pull_request: + branches: + - '*' + +jobs: + install-front: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '18.x' + + - name: Restore the frontend cache + uses: actions/cache@v4 + id: front-node_modules + with: + path: 'src/frontend/**/node_modules' + key: front-node_modules-${{ hashFiles('src/frontend/**/yarn.lock') }} + + - name: Install dependencies + if: steps.front-node_modules.outputs.cache-hit != 'true' + run: cd src/frontend/ && yarn install --frozen-lockfile + + - name: Cache install frontend + if: steps.front-node_modules.outputs.cache-hit != 'true' + uses: actions/cache@v4 + with: + path: 'src/frontend/**/node_modules' + key: front-node_modules-${{ hashFiles('src/frontend/**/yarn.lock') }} + + build-front: + runs-on: ubuntu-latest + needs: install-front + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Restore the frontend cache + uses: actions/cache@v4 + id: front-node_modules + with: + path: 'src/frontend/**/node_modules' + key: front-node_modules-${{ hashFiles('src/frontend/**/yarn.lock') }} + + - name: Build CI App + run: cd src/frontend/ && yarn ci:build + + - name: Cache build frontend + uses: actions/cache@v4 + with: + path: src/frontend/apps/magnify/out/ + key: build-front-${{ github.run_id }} + + test-front: + runs-on: ubuntu-latest + needs: install-front + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Restore the frontend cache + uses: actions/cache@v4 + id: front-node_modules + with: + path: 'src/frontend/**/node_modules' + key: front-node_modules-${{ hashFiles('src/frontend/**/yarn.lock') }} + + - name: Test App + run: cd src/frontend/ && yarn app:test + + lint-front: + runs-on: ubuntu-latest + needs: install-front + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Restore the frontend cache + uses: actions/cache@v4 + id: front-node_modules + with: + path: 'src/frontend/**/node_modules' + key: front-node_modules-${{ hashFiles('src/frontend/**/yarn.lock') }} + + - name: Check linting + run: cd src/frontend/ && yarn lint + + test-e2e: + runs-on: ubuntu-latest + needs: build-front + timeout-minutes: 10 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set services env variables + run: | + make create-env-files + cat env.d/development/common.e2e.dist >> env.d/development/common + + - name: Restore the frontend cache + uses: actions/cache@v4 + id: front-node_modules + with: + path: 'src/frontend/**/node_modules' + key: front-node_modules-${{ hashFiles('src/frontend/**/yarn.lock') }} + + - name: Restore the build cache + uses: actions/cache@v4 + id: cache-build + with: + path: src/frontend/apps/magnify/out/ + key: build-front-${{ github.run_id }} + + - name: Build and Start Docker Servers + env: + DOCKER_BUILDKIT: 1 + COMPOSE_DOCKER_CLI_BUILD: 1 + run: | + docker-compose build --pull --build-arg BUILDKIT_INLINE_CACHE=1 + make run + + - name: Apply DRF migrations + run: | + make migrate + + - name: Install Playwright Browsers + run: cd src/frontend/apps/e2e && yarn install + + - name: Run e2e tests + run: cd src/frontend/ && yarn e2e:test + + - uses: actions/upload-artifact@v3 + if: always() + with: + name: playwright-report + path: src/frontend/apps/e2e/report/ + retention-days: 7 + diff --git a/.github/workflows/magnify.yml b/.github/workflows/magnify.yml new file mode 100644 index 000000000..7c6d53268 --- /dev/null +++ b/.github/workflows/magnify.yml @@ -0,0 +1,199 @@ +name: magnify Workflow + +on: + push: + branches: + - main + tags: + - 'v*' + pull_request: + branches: + - '*' + +jobs: + lint-git: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' # Makes sense only for pull requests + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: show + run: git log + - name: Enforce absence of print statements in code + run: | + ! git diff origin/${{ github.event.pull_request.base.ref }}..HEAD -- . ':(exclude)**/magnify.yml' | grep "print(" + - name: Check absence of fixup commits + run: | + ! git log | grep 'fixup!' + - name: Install gitlint + run: pip install --user requests gitlint + - name: Lint commit messages added to main + run: ~/.local/bin/gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD + + check-changelog: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Check that the CHANGELOG has been modified in the current branch + run: git whatchanged --name-only --pretty="" origin..HEAD | grep CHANGELOG + + lint-changelog: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Check CHANGELOG max line length + run: | + max_line_length=$(cat CHANGELOG.md | grep -Ev "^\[.*\]: https://github.com" | wc -L) + if [ $max_line_length -ge 80 ]; then + echo "ERROR: CHANGELOG has lines longer than 80 characters." + exit 1 + fi + + build-mails: + runs-on: ubuntu-latest + defaults: + run: + working-directory: src/mail + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: '18' + - name: Install yarn + run: npm install -g yarn + - name: Install node dependencies + run: yarn install --frozen-lockfile + - name: Build mails + run: yarn build + + lint-back: + runs-on: ubuntu-latest + defaults: + run: + working-directory: src/backend + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Install Python + uses: actions/setup-python@v3 + with: + python-version: '3.10' + - name: Install development dependencies + run: pip install --user .[dev] + - name: Check code formatting with ruff + run: ~/.local/bin/ruff format magnify --diff + - name: Lint code with ruff + run: ~/.local/bin/ruff check magnify + - name: Lint code with pylint + run: ~/.local/bin/pylint magnify + + test-back: + runs-on: ubuntu-latest + defaults: + run: + working-directory: src/backend + + services: + postgres: + image: postgres:16 + env: + POSTGRES_DB: magnify + POSTGRES_USER: dinum + POSTGRES_PASSWORD: pass + ports: + - 5432:5432 + # needed because the postgres container does not provide a healthcheck + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + env: + DJANGO_CONFIGURATION: Test + DJANGO_SETTINGS_MODULE: magnify.settings + DJANGO_SECRET_KEY: ThisIsAnExampleKeyForTestPurposeOnly + OIDC_OP_JWKS_ENDPOINT: /endpoint-for-test-purpose-only + DB_HOST: localhost + DB_NAME: magnify + DB_USER: dinum + DB_PASSWORD: pass + DB_PORT: 5432 + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Create writable /data + run: | + sudo mkdir -p /data/media && \ + sudo mkdir -p /data/static + - name: Install Python + uses: actions/setup-python@v3 + with: + python-version: '3.10' + - name: Install development dependencies + run: pip install --user .[dev] + - name: Install gettext (required to compile messages) + run: | + sudo apt-get update + sudo apt-get install -y gettext + - name: Generate a MO file from strings extracted from the project + run: python manage.py compilemessages + - name: Run tests + run: ~/.local/bin/pytest -n 2 + + i18n-crowdin: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install gettext (required to make messages) + run: | + sudo apt-get update + sudo apt-get install -y gettext + + - name: Install Python + uses: actions/setup-python@v3 + with: + python-version: '3.10' + + - name: Install development dependencies + working-directory: src/backend + run: pip install --user .[dev] + + - name: Generate the translation base file + run: ~/.local/bin/django-admin makemessages --keep-pot --all + + - name: Load sops secrets + uses: rouja/actions-sops@main + with: + secret-file: .github/workflows/secrets.enc.env + age-key: ${{ secrets.SOPS_PRIVATE }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '18.x' + cache: 'yarn' + cache-dependency-path: src/frontend/yarn.lock + + - name: Install dependencies + run: cd src/frontend/ && yarn install --frozen-lockfile + + - name: Extract the frontend translation + run: make frontend-i18n-extract + + - name: Upload files to Crowdin + run: | + docker run \ + --rm \ + -e CROWDIN_API_TOKEN=$CROWDIN_API_TOKEN \ + -e CROWDIN_PROJECT_ID=$CROWDIN_PROJECT_ID \ + -e CROWDIN_BASE_PATH=$CROWDIN_BASE_PATH \ + -v "${{ github.workspace }}:/app" \ + crowdin/cli:3.16.0 \ + crowdin upload sources -c /app/crowdin/config.yml + diff --git a/.github/workflows/secrets.enc.env b/.github/workflows/secrets.enc.env new file mode 100644 index 000000000..54456e71e --- /dev/null +++ b/.github/workflows/secrets.enc.env @@ -0,0 +1,22 @@ +SOPS_PRIVATE=ENC[AES256_GCM,data:53ysyQ9gq2PnAQKNjOL+e+Bu5SQIuOguz8Bo5CpqbpYsF0AmV1WsOutckdClbu6ApqV3m9/Cj1FJ30+L/+j05pvcpqMeehPQwGQ=,iv:VMuML9IXiEqKY9jp+ny76jnQHmewq2rqdBy1wYpZkSI=,tag:aAZgwiWDg1AG4wk3f2Fq4w==,type:str] +CROWDIN_API_TOKEN=ENC[AES256_GCM,data:bwh38oLDH4BpI2H+7oUjtVizyrYvVJ6Av4ECTnyPPthMz6DCaYQn55RXp8rQDgJj4bPRls+JcRVC94zYIjgpkDsbbcqHr620KQKHQHMgoOQ=,iv:hydpwWtCiOkhBpAYyNwDzSjhjfdUJcKX7YX3/PXteN0=,tag:eQLniL5XxkNs5yThUuQHyw==,type:str] +CROWDIN_BASE_PATH=ENC[AES256_GCM,data:LJZE454A6qg=,iv:yIjGACBJSX3S9g7PAHRFn074xL94fHvMLcTKzFYwkwo=,tag:1Z8+UbeDOvTxR80b95KumQ==,type:str] +CROWDIN_PROJECT_ID=ENC[AES256_GCM,data:THoNz661,iv:Ixd0D9tnpEWd2yqZui1HJQEO/h7YsAC1R9Vjj8OHBjA=,tag:wfDHhzaXLD3NwY5zDj24RA==,type:str] +DOCKER_HUB_PASSWORD=ENC[AES256_GCM,data:jj92OOVMtsagOXQ=,iv:r/u8M70PspZMFCbi8a3FvuCDtWt+9YGArPNHZRpHA+k=,tag:WM3vzVkuQZVdHa3wh4satg==,type:str] +DOCKER_HUB_USER=ENC[AES256_GCM,data:btdtLdLApQ==,iv:y1o2zwyzusBS6JiQSEtZwS2zctISo+UgAFhyZ53vbKQ=,tag:ZLkMJydgjMBmbbKq979z7g==,type:str] +ARGOCD_WEBHOOK_URL=ENC[AES256_GCM,data:0TnoZv7vQI+8MZ/7EITx0Mvez66G6BcCzw+Mic+NH2qh0BdZBH8ynkYBleKw9V6TbucgHasa7duL,iv:GeE5tSpjAndThrXrzz8Dk6ah9Bxv6JQCJmKAfsToDi0=,tag:O2pIhA0ge1xygIv0izSMxg==,type:str] +ARGOCD_WEBHOOK_SECRET=ENC[AES256_GCM,data:SrdWdV24lGztyUnFXeOYGAhqTErRFakIm7hBw8n4NKW6ll6AgeZKY6w7pbvgFknQ+NlRd/EK7bYk7CZtPDGU6zM=,iv:IkWxnTWrvzWwNh4RSt3N7iPHA7K7jkzSHa4CHptxxvU=,tag:XFVYBRsuDF/La1/8ADQ2jw==,type:str] +sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBESDdJSzBaaVlEbHRjSlIy\naVoyY2l6RVVqVXhOekV4NHdHQjV6Q0IzSEJNCk9JY3BFQ2tFWXBZVFMyWTJUYjdz\nMVdheTd4cjhFREl5MmNncmlobVNyUUUKLS0tIEg1MHBsV2FoRkFlN2JoNlFuTFFS\nNG5yUXZpQVY4Z1FGZmVLUjBqQWhSQTgKfT7hD5LVWg2NOrdyeIiVt6BX/4dt6fpN\nyydn2U0yxMg9fUZ7KkixAaWpChL3rvi3OWM07h6EdsznTwehLiMFTw==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_0__map_recipient=age15fyxdwmg5mvldtqqus87xspuws2u0cpvwheehrtvkexj4tnsqqysw6re2x +sops_age__list_1__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJa3EzbDJBeHcrUE44SXpM\ndlVheHdxc2I4ellwcHlUQkhWL2NiMFpBYUd3CmJxZUZhL0tZVkViQTZFRVRFbndC\nd2ljZUJxczZqSmdqcXlzYkZlZ2t4MTgKLS0tIFFmbHE1NXpOYlRnb2wzSTRVbTQ4\nMDhTNzN6WHovMXFhek5pbXZlMW1PdEkKJlydhV9Es+y2ngMwZMGnuF+JnEV1TGZH\nkWoBHxTSA7WEgwnhGaCe7kuzXrvv2ikrV1Ww7sN4wmqfCGC2sdkPBQ==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_1__map_recipient=age16hnlml8yv4ynwy0seer57g8qww075crd0g7nsundz3pj4wk7m3vqftszg7 +sops_age__list_2__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzZEpFZU5maklnN1N4S0kw\nRGFNYzBGR2tFT2d5VzlRYU9NUWVvZld0REQ4CldvTlFtK0RFU0tuNjVhNEM4VzlC\nWjJhUEZVY0l0T05yNVBabXNEdndlbVkKLS0tIGxxdEROcWxpSHczMkN0dkdicnVZ\nT1BXR1hSa2l1SXdYS3RoWWh6NGdWSHcKZJd6HYESjLomY7/S9+eCCN4cFXERipNl\nWtOVZXlufN5BMxX8n8TlKS34oD1t6/CMaZZdmp2SHHslipA+CGRZ5g==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_2__map_recipient=age1plkp8td6zzfcavjusmsfrlk54t9vn8jjxm8zaz7cmnr7kzl2nfnsd54hwg +sops_age__list_3__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzakNpcGkzWlp6NWt1NFU0\ncmhFek1DTU5YS1MyYzRoOGJ2RXdjRU5WcEZBCjN5eUp6WVh0YmdNMzdHTUNJTVZM\ncHZTY3pxbHd0TmhSWmQyVndZS1JjZ00KLS0tIFNxYjZXRHBKbjNxVitQaGlKQVh0\ncHAwbzFyL3hUVmN2dVNQaklIcXZKQjgKr4IO6BoTFO7Km9V/h8tF3UNRCGUXymIw\nnQGL0ZDyIQw7MMBQQ2mksYPSBTFmaejbSd29UkhVnYFuCjJ+LVmX1w==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_3__map_recipient=age12g6f5fse25tgrwweleh4jls3qs52hey2edh759smulwmk5lnzadslu2cp3 +sops_age__list_4__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpS0hNdDk2Lys4Rk9nVlV1\nWHVwOHcxT3RmZkVSMWh6L0M1bGRrNEt3c1M0CmdseVlqaFZYZjd6KzI0ejdDSG55\nNkFlMGpiOFhMZWtKYkVodGpmUWRsMjgKLS0tIG5ZbVFadk5XVlREZFFEcWNiSDhw\nVnh5b3BURGU4bCtQQzR3b3hxcXdGSlEKBw7E/umovQnucE4oYeuoHFlEtYBMVXPL\n6YjZzBpBxJ+4kZpMvqsXzowQ7ZDEods9pEcuJmHqxrRpLeOrYrykTA==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_4__map_recipient=age1qy04neuzwpasmvljqrcvhwnf0kz5cpyteze38c8avp0czewskasszv9pyw +sops_lastmodified=2024-04-03T15:36:15Z +sops_mac=ENC[AES256_GCM,data:1v44C4K4YjV1m7tZKRgj8SiDamdD+L4p3TVwwOl6+05KCOh2uH2ohH+5MH7MTFL489oqaadpjBQfELSJ8h/4fN5MT6+Trbtk5QFLv4moLZx1tSCE1Tuam2cicFem2mlOrxb0pK/tU1qzCLvZke3yvFmiJEa+92u7y96hXM4VR6Y=,iv:23T3Tl5DvRH8zvef7ftbr5GWk+YFfLCzZ/eEzqjMKXY=,tag:TIch+2911w5qleXo55zM0w==,type:str] +sops_unencrypted_suffix=_unencrypted +sops_version=3.8.1 From fdae58e540bb0ec7693072bd41ce1009df443a11 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Wed, 10 Apr 2024 16:28:32 +0200 Subject: [PATCH 02/44] job --- .github/workflows/all.yml | 255 -------------------------------------- 1 file changed, 255 deletions(-) delete mode 100644 .github/workflows/all.yml diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml deleted file mode 100644 index 94d87deee..000000000 --- a/.github/workflows/all.yml +++ /dev/null @@ -1,255 +0,0 @@ -# Ancres du fichier de configuration -generate-version-file: &generate-version-file - run: - name: Créer un fichier version.json - command: | - # Créer un fichier version.json à la manière de Mozilla - # https://github.com/mozilla-services/Dockerflow/blob/master/docs/version_object.md - printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ - "$CIRCLE_SHA1" \ - "$CIRCLE_TAG" \ - "$CIRCLE_PROJECT_USERNAME" \ - "$CIRCLE_PROJECT_REPONAME" \ - "$CIRCLE_BUILD_URL" > sandbox/version.json - -version: 2 - -aliases: - - &checkout_fun - checkout: - path: ~/fun - - - &restore_node_modules - restore_cache: - name: Restaurer le cache node_modules - keys: - - v18-front-dependencies-{{ checksum "~/fun/src/frontend/yarn.lock" }} - - v18-front-dependencies- - -jobs: - # Tâches Git - # Vérifier que l'historique git est propre et conforme à nos attentes - lint-git: - docker: - - image: cimg/python:3.10 - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - working_directory: ~/fun - steps: - - checkout - # S'assurer que les modifications n'ajoutent pas une instruction "print" à la base de code. - # Nous devons exclure le dossier ".circleci" de la recherche car la commande même qui vérifie - # l'absence de "print" inclut un "print(" lui-même. - - run: - name: Enforcer l'absence d'instructions print dans le code - command: | - ! git diff origin/main..HEAD -- ":(exclude)*.circleci/*" | grep "print(" - - run: - name: Vérifier l'absence de commits de correction (fixup) - command: | - ! git log | grep 'fixup!' - - run: - name: Installer gitlint - command: | - pip install --user requests gitlint - - run: - name: Linter les messages de commit ajoutés à main - command: | - ~/.local/bin/gitlint --commits origin/main..HEAD - - # Vérifier que le CHANGELOG a été mis à jour dans la branche actuelle - check-changelog: - docker: - - image: circleci/buildpack-deps:stretch-scm - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - working_directory: ~/fun - steps: - - checkout - - run: - name: Vérifier que le CHANGELOG a été modifié dans la branche actuelle - command: | - git whatchanged --name-only --pretty="" origin..HEAD | grep CHANGELOG - - # Vérifier que la longueur maximale des lignes du CHANGELOG ne dépasse pas 80 caractères - lint-changelog: - docker: - - image: debian:stretch - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - working_directory: ~/fun - steps: - - checkout - - run: - name: Vérifier la longueur maximale des lignes du CHANGELOG - command: | - # Obtenir la largeur de la ligne la plus longue (en ignorant les liens de publication) - test $(cat CHANGELOG.md | grep -Ev "^\[.*\]: https://github.com/openfun" | wc -L) -le 80 - - # Vérifier que le fichier de configuration de Renovate est valide - check-renovate-configuration: - docker: - - image: renovate/renovate - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - working_directory: ~/fun - steps: - - checkout - - run: - name: Exécuter la commande renovate-config-validator - command: renovate-config-validator - - # Vérifier que toutes les versions (backend, frontend) sont à jour - check-versions: - docker: - - image: cimg/base:2022.04 - working_directory: ~/fun - steps: - - checkout - - run: - name: Vérifier que toutes les versions sont identiques - command: | - BACKEND_VERSION=$(cat setup.cfg | grep "version" | cut -d" " -f3) - echo "Version magnify : ${BACKEND_VERSION}" - # Dans l'espace de travail frontend - cat src/frontend/package.json | grep "\"version\": \"${BACKEND_VERSION}\",$" - # Dans la bibliothèque frontend "@openfun/jitsi-magnify" - cat src/frontend/packages/core/package.json | grep "\"version\": \"${BACKEND_VERSION}\",$" - # Dans l'application frontend "sandbox" - cat src/frontend/sandbox/package.json | grep "\"version\": \"${BACKEND_VERSION}\",$" - - # ---- Tâches Docker ---- - # Construire l'image Docker prête pour la production - build-docker: - docker: - - image: circleci/buildpack-deps:stretch - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - working_directory: ~/fun - steps: - # Récupérer les sources du dépôt - - checkout - # Générer un fichier version.json décrivant la version de l'application - - <<: *generate-version-file - # Activer docker-in-docker - - setup_remote_docker: - version: 19.03.13 - - # Se connecter à Docker Hub avec des identifiants cryptés stockés en tant que secret - # variables d'environnement (définies dans les paramètres du projet CircleCI) si la variable - # d'environnement attendue est définie; passer en mode anonyme sinon. - - run: - name: Se connecter à DockerHub - command: > - test -n "$DOCKER_USER" && - echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin || - echo "Mode anonyme Docker Hub" - - # Chaque image est marquée avec le hachage du commit git actuel pour éviter - # les collisions dans les constructions parallèles. - - run: - name: Construire l'image de production - command: docker build -t magnify:${CIRCLE_SHA1} --target production . - - run: - name: Vérifier la disponibilité de l'image construite - command: docker images "magnify:${CIRCLE_SHA1}*" - - # ---- Tâches Backend ---- - # Construire l'environnement de développement backend - build-back: - docker: - - image: cimg/python:3.10 - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - working_directory: ~/fun - steps: - - checkout - # Restaurer le cache des dépendances Python - - restore_cache: - name: Restaurer le cache des dépendances Python - keys: - - v18-back-dependencies-{{ checksum "~/fun/setup.cfg" }} - - v18-back-dependencies- - - run: - name: Installer les dépendances du backend - command: pip install -r requirements-dev.txt - # Sauvegarder le cache des dépendances Python - - save_cache: - name: Sauvegarder le cache des dépendances Python - key: v18-back-dependencies-{{ checksum "~/fun/setup.cfg" }} - paths: - - ~/.cache/pip - - ./venv/lib/python*/site-packages/ - - run: - name: Vérifier la compatibilité entre les versions des bibliothèques Python et Black - command: pip check - - # ---- Tâches Frontend ---- - # Installer les dépendances du frontend et les outils de développement - build-front: - docker: - - image: cimg/node:17.3 - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - working_directory: ~/fun - steps: - - checkout - # Restaurer le cache des dépendances Node.js - - restore_cache: - name: Restaurer le cache des dépendances Node.js - <<: *restore_node_modules - - run: - name: Installer les dépendances Node.js - command: yarn install --frozen-lockfile --non-interactive - # Sauvegarder le cache des dépendances Node.js - - save_cache: - name: Sauvegarder le cache des dépendances Node.js - key: v18-front-dependencies-{{ checksum "~/fun/src/frontend/yarn.lock" }} - paths: - - ~/.cache/yarn - - ~/.cache/Cypress - - ./src/frontend/node_modules - - run: - name: Vérifier la compatibilité entre les dépendances du frontend et ESLint - command: yarn run eslint --no-error-on-unmatched-pattern - -workflows: - version: 2 - lint-git: - jobs: - - lint-git - - check-changelog: - jobs: - - check-changelog - - lint-changelog: - jobs: - - lint-changelog - - check-renovate-configuration: - jobs: - - check-renovate-configuration - - check-versions: - jobs: - - check-versions - - build-docker: - jobs: - - build-docker - - build-back: - jobs: - - build-back - - build-front: - jobs: - - build-front From 85550f5e8c4d81b21abbe00c061803091073ae29 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Wed, 10 Apr 2024 17:43:53 +0200 Subject: [PATCH 03/44] job fix --- .circleci/config.yml | 1596 +++++++++++++++++++++--------------------- 1 file changed, 798 insertions(+), 798 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4b1d1b20b..9a3c32133 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,798 +1,798 @@ -# Configuration file anchors -generate-version-file: &generate-version-file - run: - name: Create a version.json - command: | - # Create a version.json à-la-mozilla - # https://github.com/mozilla-services/Dockerflow/blob/master/docs/version_object.md - printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ - "$CIRCLE_SHA1" \ - "$CIRCLE_TAG" \ - "$CIRCLE_PROJECT_USERNAME" \ - "$CIRCLE_PROJECT_REPONAME" \ - "$CIRCLE_BUILD_URL" > sandbox/version.json - -version: 2 - -aliases: - - &checkout_fun - checkout: - path: ~/fun - - - &restore_node_modules - restore_cache: - name: Restore node_modules cache - keys: - - v18-front-dependencies-{{ checksum "~/fun/src/frontend/yarn.lock" }} - - v18-front-dependencies- - -jobs: - # Git jobs - # Check that the git history is clean and complies with our expectations - lint-git: - docker: - - image: cimg/python:3.10 - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - working_directory: ~/fun - steps: - - checkout - # Make sure the changes don't add a "print" statement to the code base. - # We should exclude the ".circleci" folder from the search as the very command that checks - # the absence of "print" is including a "print(" itself. - - run: - name: enforce absence of print statements in code - command: | - ! git diff origin/main..HEAD -- ":(exclude)*.circleci/*" | grep "print(" - - run: - name: Check absence of fixup commits - command: | - ! git log | grep 'fixup!' - - run: - name: Install gitlint - command: | - pip install --user requests gitlint - - run: - name: lint commit messages added to main - command: | - ~/.local/bin/gitlint --commits origin/main..HEAD - - # Check that the CHANGELOG has been updated in the current branch - check-changelog: - docker: - - image: circleci/buildpack-deps:stretch-scm - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - working_directory: ~/fun - steps: - - checkout - - run: - name: Check that the CHANGELOG has been modified in the current branch - command: | - git whatchanged --name-only --pretty="" origin..HEAD | grep CHANGELOG - - # Check that the CHANGELOG max line length does not exceed 80 characters - lint-changelog: - docker: - - image: debian:stretch - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - working_directory: ~/fun - steps: - - checkout - - run: - name: Check CHANGELOG max line length - command: | - # Get the longuest line width (ignoring release links) - test $(cat CHANGELOG.md | grep -Ev "^\[.*\]: https://github.com/openfun" | wc -L) -le 80 - - # Check that renovate configuration file is valid - check-renovate-configuration: - docker: - - image: renovate/renovate - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - working_directory: ~/fun - steps: - - checkout - - run: - name: Run renovate-config-validator command - command: renovate-config-validator - - # Check that all versions (backend, frontend) are up-to-date - check-versions: - docker: - - image: cimg/base:2022.04 - working_directory: ~/fun - steps: - - checkout - - run: - name: Check that all versions are the same - command: | - BACKEND_VERSION=$(cat setup.cfg | grep "version" | cut -d" " -f3) - echo "magnify version: ${BACKEND_VERSION}" - # In the frontend workspace - cat src/frontend/package.json | grep "\"version\": \"${BACKEND_VERSION}\",$" - # In the "@openfun/jitsi-magnify" frontend library - cat src/frontend/packages/core/package.json | grep "\"version\": \"${BACKEND_VERSION}\",$" - # In the "sandbox" frontend application - cat src/frontend/sandbox/package.json | grep "\"version\": \"${BACKEND_VERSION}\",$" - - # ---- Docker jobs ---- - # Build the Docker image ready for production - build-docker: - docker: - - image: circleci/buildpack-deps:stretch - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - working_directory: ~/fun - steps: - # Checkout repository sources - - checkout - # Generate a version.json file describing app release - - <<: *generate-version-file - # Activate docker-in-docker - - setup_remote_docker: - version: 19.03.13 - - # Login to Docker Hub with encrypted credentials stored as secret - # environment variables (set in CircleCI project settings) if the expected - # environment variable is set; switch to anonymous mode otherwise. - - run: - name: Login to DockerHub - command: > - test -n "$DOCKER_USER" && - echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin || - echo "Docker Hub anonymous mode" - - # Each image is tagged with the current git commit sha1 to avoid - # collisions in parallel builds. - - run: - name: Build production image - command: docker build -t magnify:${CIRCLE_SHA1} --target production . - - run: - name: Check built image availability - command: docker images "magnify:${CIRCLE_SHA1}*" - - # ---- Backend jobs ---- - # Build backend development environment - build-back: - docker: - - image: cimg/python:3.10 - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - working_directory: ~/fun - steps: - - checkout - - restore_cache: - keys: - - v2-back-dependencies-{{ .Revision }} - - run: - name: Install development dependencies - command: pip install --user .[dev,sandbox] - - save_cache: - paths: - - ~/.local - key: v2-back-dependencies-{{ .Revision }} - - # Build backend translations - build-back-i18n: - docker: - - image: cimg/python:3.10 - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - working_directory: ~/fun/src/magnify - steps: - - *checkout_fun - - restore_cache: - keys: - - v2-back-dependencies-{{ .Revision }} - - run: - name: Install gettext (required to make messages) - command: | - sudo apt-get update - sudo apt-get install -y gettext - # Generate and persist the translations base file - - run: - name: Generate a POT file from strings extracted from the project - command: ~/.local/bin/django-admin makemessages --keep-pot --all - - persist_to_workspace: - root: ~/fun - paths: - - src/magnify/locale/django.pot - - lint-back: - docker: - - image: cimg/python:3.10 - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - working_directory: ~/fun - steps: - - checkout - - restore_cache: - keys: - - v2-back-dependencies-{{ .Revision }} - - run: - name: Lint code with flake8 - command: ~/.local/bin/flake8 src/magnify/apps sandbox tests - - run: - name: Lint code with isort - command: ~/.local/bin/isort --check-only src/magnify/apps sandbox tests - - run: - name: Lint code with black - command: ~/.local/bin/black . --check - - run: - name: Lint code with pylint - command: ~/.local/bin/pylint src/magnify/apps sandbox tests - - run: - name: Lint code with bandit - command: ~/.local/bin/bandit -qr src/magnify/apps sandbox - - # Restore back POT & front json files containing strings to translate and upload them to our - # translation management tool - upload-i18n-strings: - docker: - - image: crowdin/cli:3.3.0 - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - working_directory: ~/fun - steps: - - checkout - - attach_workspace: - at: ~/fun - - run: - name: upload files to crowdin - command: crowdin upload sources -c crowdin/config.yml - - test-back-mysql-8: - docker: - - image: cimg/python:3.10 - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - environment: - DJANGO_SETTINGS_MODULE: settings - DJANGO_CONFIGURATION: Test - DJANGO_SECRET_KEY: ThisIsAnExampleKeyForTestPurposeOnly - JWT_JITSI_SECRET_KEY: ThisIsAnExampleKeyForDevPurposeOnly - MAGNIFY_API_URL: http://localhost:8070/api - KEYCLOAK_URL: http://localhost:8080 - PYTHONPATH: /home/circleci/fun/sandbox - magnify_ES_HOST: localhost - DB_ENGINE: django.db.backends.mysql - # The DB_HOST should match the host name and cannot be set from here - # where it will be escaped. See the test command instead: - # DB_HOST=${HOSTNAME} - DB_HOST: - DB_NAME: magnify - DB_USER: fun - DB_PASSWORD: pass - DB_PORT: 3306 - # services - - image: circleci/mysql:8.0-ram - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - environment: - MYSQL_ROOT_PASSWORD: - MYSQL_DATABASE: test_magnify - MYSQL_USER: fun - MYSQL_PASSWORD: pass - command: --default-authentication-plugin=mysql_native_password - - image: docker.io/bitnami/redis:6.0-debian-10 - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - name: redis-primary - environment: - ALLOW_EMPTY_PASSWORD: yes - REDIS_REPLICATION_MODE: master - - image: docker.io/bitnami/redis-sentinel:6.0-debian-10 - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - name: redis-sentinel - environment: - REDIS_MASTER_HOST: redis-primary - working_directory: ~/fun - steps: - - checkout - - restore_cache: - keys: - - v2-back-dependencies-{{ .Revision }} - # Attach the frontend production build - - attach_workspace: - at: ~/fun - # While running tests, we need to make the /data directory writable for - # the circleci user - - run: - name: Create writable /data - command: | - sudo mkdir /data && \ - sudo chown circleci:circleci /data - # Run back-end (Django) test suite - # - # Nota bene: - # - # 1. to run the django test suite, we need to ensure that the MySQL - # service is up and ready. To achieve this, we wrap the - # pytest command execution with dockerize, a tiny tool installed in the - # CircleCI image. In our case, dockerize will wait up to one minute - # that the database container opened its expected tcp port (3306). - # 2. We should avoid using localhost for the DB_HOST with MySQL as the - # client will try to use a local socket (_e.g._ - # `/var/run/mysqld/mysqld.sock`) instead of the database host and port - # ¯\_(ツ)_/¯. - - run: - name: Run tests - command: | - DB_HOST=${HOSTNAME} dockerize \ - -wait tcp://${HOSTNAME}:3306 \ - -timeout 60s \ - ~/.local/bin/pytest - - test-back-postgresql: - docker: - - image: cimg/python:3.10 - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - environment: - DJANGO_SETTINGS_MODULE: settings - DJANGO_CONFIGURATION: Test - DJANGO_SECRET_KEY: ThisIsAnExampleKeyForTestPurposeOnly - JWT_JITSI_SECRET_KEY: ThisIsAnExampleKeyForDevPurposeOnly - MAGNIFY_API_URL: http://localhost:8070/api - KEYCLOAK_URL: http://localhost:8080 - PYTHONPATH: /home/circleci/fun/sandbox - DB_HOST: localhost - DB_NAME: magnify - DB_USER: fun - DB_PASSWORD: pass - DB_PORT: 5432 - # services - - image: cimg/postgres:14.2 - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - environment: - POSTGRES_DB: test_magnify - POSTGRES_USER: fun - POSTGRES_PASSWORD: pass - - image: docker.io/bitnami/redis:6.0-debian-10 - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - name: redis-primary - environment: - ALLOW_EMPTY_PASSWORD: yes - REDIS_REPLICATION_MODE: master - - image: docker.io/bitnami/redis-sentinel:6.0-debian-10 - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - name: redis-sentinel - environment: - REDIS_MASTER_HOST: redis-primary - working_directory: ~/fun - steps: - - checkout - - restore_cache: - keys: - - v2-back-dependencies-{{ .Revision }} - # Attach the frontend production build - - attach_workspace: - at: ~/fun - # While running tests, we need to make the /data directory writable for - # the circleci user - - run: - name: Create writable /data - command: | - sudo mkdir /data && \ - sudo chown circleci:circleci /data - # Run back-end (Django) test suite - # - # Nota bene: to run the django test suite, we need to ensure that the - # MySQL service is up and ready. To achieve this, we wrap the pytest - # command execution with dockerize, a tiny tool installed - # in the CircleCI image. In our case, dockerize will wait up to one minute - # that the database container opened its expected tcp port (3306). - - run: - name: Run tests - command: | - dockerize \ - -wait tcp://localhost:5432 \ - -timeout 60s \ - ~/.local/bin/pytest - - # ---- Packaging jobs ---- - package-back: - docker: - - image: cimg/python:3.10 - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - working_directory: ~/fun - steps: - - checkout - # Ensure we restore frontend production builds in magnify's static - # directory - - attach_workspace: - at: ~/fun - - run: - name: Build python package - command: python setup.py sdist bdist_wheel - # Persist build packages to the workspace - - persist_to_workspace: - root: ~/fun - paths: - - dist - # Store packages as artifacts to download/test them - - store_artifacts: - path: ~/fun/dist - - # Publishing to PyPI requires that: - # * you already registered to pypi.org - # * you have define both the TWINE_USERNAME & TWINE_PASSWORD secret - # environment variables in CircleCI UI (with your PyPI credentials) - pypi: - docker: - - image: cimg/python:3.10 - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - working_directory: ~/fun - steps: - - checkout - # Restore built python packages - - attach_workspace: - at: ~/fun - - run: - name: List built packages - command: ls dist/* - - run: - name: Install base requirements (twine) - command: pip install --user .[ci] - - run: - name: Upload built packages to PyPI - command: ~/.local/bin/twine upload dist/* - - # ---- Front-end jobs ---- - build-front: - docker: - - image: cimg/node:16.15 - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - working_directory: ~/fun/src/frontend - steps: - - *checkout_fun - - *restore_node_modules - # If the yarn.lock file is not up-to-date with the package.json file, - # using the --frozen-lockfile should fail. - - run: - name: Install front-end dependencies - command: yarn install --frozen-lockfile - - run: - name: Build front-end - command: yarn build - - run: - name: Use formatjs-cli to generate frontend.json files - command: yarn extract-translations - - persist_to_workspace: - root: ~/fun - paths: - - src/frontend/sandbox/i18n/frontend.json - - src/frontend/packages/core/i18n/frontend.json - - src/frontend/packages/core/dist - - save_cache: - paths: - - ./node_modules - - ./sandbox/node_modules - - ./packages/core/node_modules - key: v18-front-dependencies-{{ checksum "yarn.lock" }} - - lint-front: - docker: - - image: cimg/node:16.15 - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - working_directory: ~/fun/src/frontend - steps: - - *checkout_fun - - *restore_node_modules - - run: - name: Lint code with eslint - command: yarn lint - - run: - name: Lint code with prettier - command: yarn format:check - - test-front: - docker: - - image: cimg/node:16.15 - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - parallelism: 5 - resource_class: large - working_directory: ~/fun/src/frontend - steps: - - *checkout_fun - - *restore_node_modules - - run: - path: packages/core - name: Test @openfun/jitsi-magnify - command: | - TEST=$(circleci tests glob "src/**/*.test.tsx" | circleci tests split --split-by=timings) - yarn test $TEST - - run: - name: Test frontend demo - command: yarn test-demo - - # Publishing to npm requires that you have define the NPM_TOKEN secret - # environment variables in CircleCI UI (with your PyPI credentials) - npm: - docker: - - image: cimg/node:16.15 - auth: - username: $DOCKER_USER - password: $DOCKER_PASS - working_directory: ~/fun - steps: - - *checkout_fun - - attach_workspace: - at: ~/fun - - run: - name: Authenticate with registry - command: echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/fun/.npmrc - - run: - name: Publish @openfun/jitsi-magnify package - command: npm publish src/frontend/packages/core - - # ---- DockerHub publication job ---- - hub: - docker: - - image: cimg/base:2022.05 - working_directory: ~/fun - steps: - - *checkout_fun - # Generate a version.json file describing app release - - <<: *generate-version-file - # Activate docker-in-docker - - setup_remote_docker: - version: 19.03.13 - - run: - name: Build production image - command: docker build -t magnify:${CIRCLE_SHA1} --target production . - - run: - name: Check built images availability - command: docker images "magnify:${CIRCLE_SHA1}*" - # Login to DockerHub to Publish new images - # - # Nota bene: you'll need to define the following secrets environment vars - # in CircleCI interface: - # - # - DOCKER_HUB_USER - # - DOCKER_HUB_PASSWORD - - run: - name: Login to DockerHub - command: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin - # Tag docker images with the same pattern used in Git (Semantic Versioning) - # - # Git tag: v1.0.1 - # Docker tag: 1.0.1(-ci) - - run: - name: Tag images - command: | - docker images fundocker/jitsi-magnify - DOCKER_TAG=$([[ -z "$CIRCLE_TAG" ]] && echo $CIRCLE_BRANCH || echo ${CIRCLE_TAG} | sed 's/^v//') - RELEASE_TYPE=$([[ -z "$CIRCLE_TAG" ]] && echo "branch" || echo "tag ") - # Display either: - # - DOCKER_TAG: master (Git branch) - # or - # - DOCKER_TAG: 1.1.2 (Git tag v1.1.2) - echo "DOCKER_TAG: ${DOCKER_TAG} (Git ${RELEASE_TYPE}${CIRCLE_TAG})" - docker tag magnify:${CIRCLE_SHA1} fundocker/jitsi-magnify:${DOCKER_TAG} - if [[ -n "$CIRCLE_TAG" ]]; then - docker tag magnify:${CIRCLE_SHA1} fundocker/jitsi-magnify:latest - fi - docker images | grep -E "^fundocker/jitsi-magnify\s*(${DOCKER_TAG}.*|latest|master)" - - # Publish images to DockerHub - # - # Nota bene: logged user (see "Login to DockerHub" step) must have write - # permission for the project's repository; this also implies that the - # DockerHub repository already exists. - - run: - name: Publish images - command: | - DOCKER_TAG=$([[ -z "$CIRCLE_TAG" ]] && echo $CIRCLE_BRANCH || echo ${CIRCLE_TAG} | sed 's/^v//') - RELEASE_TYPE=$([[ -z "$CIRCLE_TAG" ]] && echo "branch" || echo "tag ") - # Display either: - # - DOCKER_TAG: master (Git branch) - # or - # - DOCKER_TAG: 1.1.2 (Git tag v1.1.2) - echo "DOCKER_TAG: ${DOCKER_TAG} (Git ${RELEASE_TYPE}${CIRCLE_TAG})" - docker push fundocker/jitsi-magnify:${DOCKER_TAG} - if [[ -n "$CIRCLE_TAG" ]]; then - docker push fundocker/jitsi-magnify:latest - fi - -workflows: - version: 2 - - magnify: - jobs: - # Front-end jobs - # - # Build, lint and test the front-end apps - - build-front: - filters: - tags: - only: /.*/ - - lint-front: - requires: - - build-front - filters: - tags: - only: /.*/ - - test-front: - requires: - - lint-front - filters: - tags: - only: /.*/ - - # Git jobs - # - # Check validity of git history - - lint-git: - filters: - tags: - only: /.*/ - # Check CHANGELOG update - - check-changelog: - filters: - branches: - ignore: main - tags: - only: /(?!^v).*/ - - lint-changelog: - filters: - branches: - ignore: main - tags: - only: /.*/ - # Check Renovate configuration - - check-renovate-configuration: - filters: - tags: - only: /.*/ - # Check on each PR if the last magnify version is present everywhere it should be. - # If not the build will fail before publishing a new release. - - check-versions: - filters: - tags: - only: /.*/ - - # Docker jobs - # - # Build images - - build-docker: - filters: - tags: - only: /.*/ - - # Backend jobs - # - # Build, lint and test production and development Docker images - # (debian-based) - - build-back: - filters: - tags: - only: /.*/ - - lint-back: - requires: - - build-back - filters: - tags: - only: /.*/ - - test-back-mysql-8: - requires: - - build-back - filters: - tags: - only: /.*/ - - test-back-postgresql: - requires: - - build-back - filters: - tags: - only: /.*/ - - # i18n jobs - # - # Extract strings and upload them to our translation management SaaS - - build-back-i18n: - requires: - - build-back - filters: - tags: - only: /.*/ - - upload-i18n-strings: - requires: - - build-front - - build-back-i18n - filters: - branches: - only: main - - # Packaging: python - # - # Build the python package - - package-back: - requires: - - test-front - - test-back-mysql-8 - - test-back-postgresql - - build-front - filters: - tags: - only: /.*/ - - # PyPI publication. - # - # Publish python package to PYPI only if all build, lint and test jobs - # succeed and it has been tagged with a tag starting with the letter v - - pypi: - requires: - - check-versions - - package-back - filters: - branches: - ignore: /.*/ - tags: - only: /^v.*/ - - # NPM publication. - # - # Publish frontend package to NPM only if all build, lint and test jobs - # succeed, and it has been tagged with a tag starting with the letter v - - npm: - requires: - - check-versions - - build-front - - lint-front - - test-front - filters: - branches: - ignore: /.*/ - tags: - only: /^v.*/ - - # DockerHub publication. - # - # Publish docker images only if all build, lint and test jobs succeed - # and it has been tagged with a tag starting with the letter v or is on - # the main branch - - hub: - requires: - - build-docker - - test-back-mysql-8 - - test-back-postgresql - filters: - branches: - only: main - tags: - only: /^v.*/ +## Configuration file anchors +#generate-version-file: &generate-version-file +# run: +# name: Create a version.json +# command: | +# # Create a version.json à-la-mozilla +# # https://github.com/mozilla-services/Dockerflow/blob/master/docs/version_object.md +# printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ +# "$CIRCLE_SHA1" \ +# "$CIRCLE_TAG" \ +# "$CIRCLE_PROJECT_USERNAME" \ +# "$CIRCLE_PROJECT_REPONAME" \ +# "$CIRCLE_BUILD_URL" > sandbox/version.json +# +#version: 2 +# +#aliases: +# - &checkout_fun +# checkout: +# path: ~/fun +# +# - &restore_node_modules +# restore_cache: +# name: Restore node_modules cache +# keys: +# - v18-front-dependencies-{{ checksum "~/fun/src/frontend/yarn.lock" }} +# - v18-front-dependencies- +# +#jobs: +# # Git jobs +# # Check that the git history is clean and complies with our expectations +# lint-git: +# docker: +# - image: cimg/python:3.10 +# auth: +# username: $DOCKER_USER +# password: $DOCKER_PASS +# working_directory: ~/fun +# steps: +# - checkout +# # Make sure the changes don't add a "print" statement to the code base. +# # We should exclude the ".circleci" folder from the search as the very command that checks +# # the absence of "print" is including a "print(" itself. +# - run: +# name: enforce absence of print statements in code +# command: | +# ! git diff origin/main..HEAD -- ":(exclude)*.circleci/*" | grep "print(" +# - run: +# name: Check absence of fixup commits +# command: | +# ! git log | grep 'fixup!' +# - run: +# name: Install gitlint +# command: | +# pip install --user requests gitlint +# - run: +# name: lint commit messages added to main +# command: | +# ~/.local/bin/gitlint --commits origin/main..HEAD +# +# # Check that the CHANGELOG has been updated in the current branch +# check-changelog: +# docker: +# - image: circleci/buildpack-deps:stretch-scm +# auth: +# username: $DOCKER_USER +# password: $DOCKER_PASS +# working_directory: ~/fun +# steps: +# - checkout +# - run: +# name: Check that the CHANGELOG has been modified in the current branch +# command: | +# git whatchanged --name-only --pretty="" origin..HEAD | grep CHANGELOG +# +# # Check that the CHANGELOG max line length does not exceed 80 characters +# lint-changelog: +# docker: +# - image: debian:stretch +# auth: +# username: $DOCKER_USER +# password: $DOCKER_PASS +# working_directory: ~/fun +# steps: +# - checkout +# - run: +# name: Check CHANGELOG max line length +# command: | +# # Get the longuest line width (ignoring release links) +# test $(cat CHANGELOG.md | grep -Ev "^\[.*\]: https://github.com/openfun" | wc -L) -le 80 +# +# # Check that renovate configuration file is valid +# check-renovate-configuration: +# docker: +# - image: renovate/renovate +# auth: +# username: $DOCKER_USER +# password: $DOCKER_PASS +# working_directory: ~/fun +# steps: +# - checkout +# - run: +# name: Run renovate-config-validator command +# command: renovate-config-validator +# +# # Check that all versions (backend, frontend) are up-to-date +# check-versions: +# docker: +# - image: cimg/base:2022.04 +# working_directory: ~/fun +# steps: +# - checkout +# - run: +# name: Check that all versions are the same +# command: | +# BACKEND_VERSION=$(cat setup.cfg | grep "version" | cut -d" " -f3) +# echo "magnify version: ${BACKEND_VERSION}" +# # In the frontend workspace +# cat src/frontend/package.json | grep "\"version\": \"${BACKEND_VERSION}\",$" +# # In the "@openfun/jitsi-magnify" frontend library +# cat src/frontend/packages/core/package.json | grep "\"version\": \"${BACKEND_VERSION}\",$" +# # In the "sandbox" frontend application +# cat src/frontend/sandbox/package.json | grep "\"version\": \"${BACKEND_VERSION}\",$" +# +# # ---- Docker jobs ---- +# # Build the Docker image ready for production +# build-docker: +# docker: +# - image: circleci/buildpack-deps:stretch +# auth: +# username: $DOCKER_USER +# password: $DOCKER_PASS +# working_directory: ~/fun +# steps: +# # Checkout repository sources +# - checkout +# # Generate a version.json file describing app release +# - <<: *generate-version-file +# # Activate docker-in-docker +# - setup_remote_docker: +# version: 19.03.13 +# +# # Login to Docker Hub with encrypted credentials stored as secret +# # environment variables (set in CircleCI project settings) if the expected +# # environment variable is set; switch to anonymous mode otherwise. +# - run: +# name: Login to DockerHub +# command: > +# test -n "$DOCKER_USER" && +# echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin || +# echo "Docker Hub anonymous mode" +# +# # Each image is tagged with the current git commit sha1 to avoid +# # collisions in parallel builds. +# - run: +# name: Build production image +# command: docker build -t magnify:${CIRCLE_SHA1} --target production . +# - run: +# name: Check built image availability +# command: docker images "magnify:${CIRCLE_SHA1}*" +# +# # ---- Backend jobs ---- +# # Build backend development environment +# build-back: +# docker: +# - image: cimg/python:3.10 +# auth: +# username: $DOCKER_USER +# password: $DOCKER_PASS +# working_directory: ~/fun +# steps: +# - checkout +# - restore_cache: +# keys: +# - v2-back-dependencies-{{ .Revision }} +# - run: +# name: Install development dependencies +# command: pip install --user .[dev,sandbox] +# - save_cache: +# paths: +# - ~/.local +# key: v2-back-dependencies-{{ .Revision }} +# +# # Build backend translations +# build-back-i18n: +# docker: +# - image: cimg/python:3.10 +# auth: +# username: $DOCKER_USER +# password: $DOCKER_PASS +# working_directory: ~/fun/src/magnify +# steps: +# - *checkout_fun +# - restore_cache: +# keys: +# - v2-back-dependencies-{{ .Revision }} +# - run: +# name: Install gettext (required to make messages) +# command: | +# sudo apt-get update +# sudo apt-get install -y gettext +# # Generate and persist the translations base file +# - run: +# name: Generate a POT file from strings extracted from the project +# command: ~/.local/bin/django-admin makemessages --keep-pot --all +# - persist_to_workspace: +# root: ~/fun +# paths: +# - src/magnify/locale/django.pot +# +# lint-back: +# docker: +# - image: cimg/python:3.10 +# auth: +# username: $DOCKER_USER +# password: $DOCKER_PASS +# working_directory: ~/fun +# steps: +# - checkout +# - restore_cache: +# keys: +# - v2-back-dependencies-{{ .Revision }} +# - run: +# name: Lint code with flake8 +# command: ~/.local/bin/flake8 src/magnify/apps sandbox tests +# - run: +# name: Lint code with isort +# command: ~/.local/bin/isort --check-only src/magnify/apps sandbox tests +# - run: +# name: Lint code with black +# command: ~/.local/bin/black . --check +# - run: +# name: Lint code with pylint +# command: ~/.local/bin/pylint src/magnify/apps sandbox tests +# - run: +# name: Lint code with bandit +# command: ~/.local/bin/bandit -qr src/magnify/apps sandbox +# +# # Restore back POT & front json files containing strings to translate and upload them to our +# # translation management tool +# upload-i18n-strings: +# docker: +# - image: crowdin/cli:3.3.0 +# auth: +# username: $DOCKER_USER +# password: $DOCKER_PASS +# working_directory: ~/fun +# steps: +# - checkout +# - attach_workspace: +# at: ~/fun +# - run: +# name: upload files to crowdin +# command: crowdin upload sources -c crowdin/config.yml +# +# test-back-mysql-8: +# docker: +# - image: cimg/python:3.10 +# auth: +# username: $DOCKER_USER +# password: $DOCKER_PASS +# environment: +# DJANGO_SETTINGS_MODULE: settings +# DJANGO_CONFIGURATION: Test +# DJANGO_SECRET_KEY: ThisIsAnExampleKeyForTestPurposeOnly +# JWT_JITSI_SECRET_KEY: ThisIsAnExampleKeyForDevPurposeOnly +# MAGNIFY_API_URL: http://localhost:8070/api +# KEYCLOAK_URL: http://localhost:8080 +# PYTHONPATH: /home/circleci/fun/sandbox +# magnify_ES_HOST: localhost +# DB_ENGINE: django.db.backends.mysql +# # The DB_HOST should match the host name and cannot be set from here +# # where it will be escaped. See the test command instead: +# # DB_HOST=${HOSTNAME} +# DB_HOST: +# DB_NAME: magnify +# DB_USER: fun +# DB_PASSWORD: pass +# DB_PORT: 3306 +# # services +# - image: circleci/mysql:8.0-ram +# auth: +# username: $DOCKER_USER +# password: $DOCKER_PASS +# environment: +# MYSQL_ROOT_PASSWORD: +# MYSQL_DATABASE: test_magnify +# MYSQL_USER: fun +# MYSQL_PASSWORD: pass +# command: --default-authentication-plugin=mysql_native_password +# - image: docker.io/bitnami/redis:6.0-debian-10 +# auth: +# username: $DOCKER_USER +# password: $DOCKER_PASS +# name: redis-primary +# environment: +# ALLOW_EMPTY_PASSWORD: yes +# REDIS_REPLICATION_MODE: master +# - image: docker.io/bitnami/redis-sentinel:6.0-debian-10 +# auth: +# username: $DOCKER_USER +# password: $DOCKER_PASS +# name: redis-sentinel +# environment: +# REDIS_MASTER_HOST: redis-primary +# working_directory: ~/fun +# steps: +# - checkout +# - restore_cache: +# keys: +# - v2-back-dependencies-{{ .Revision }} +# # Attach the frontend production build +# - attach_workspace: +# at: ~/fun +# # While running tests, we need to make the /data directory writable for +# # the circleci user +# - run: +# name: Create writable /data +# command: | +# sudo mkdir /data && \ +# sudo chown circleci:circleci /data +# # Run back-end (Django) test suite +# # +# # Nota bene: +# # +# # 1. to run the django test suite, we need to ensure that the MySQL +# # service is up and ready. To achieve this, we wrap the +# # pytest command execution with dockerize, a tiny tool installed in the +# # CircleCI image. In our case, dockerize will wait up to one minute +# # that the database container opened its expected tcp port (3306). +# # 2. We should avoid using localhost for the DB_HOST with MySQL as the +# # client will try to use a local socket (_e.g._ +# # `/var/run/mysqld/mysqld.sock`) instead of the database host and port +# # ¯\_(ツ)_/¯. +# - run: +# name: Run tests +# command: | +# DB_HOST=${HOSTNAME} dockerize \ +# -wait tcp://${HOSTNAME}:3306 \ +# -timeout 60s \ +# ~/.local/bin/pytest +# +# test-back-postgresql: +# docker: +# - image: cimg/python:3.10 +# auth: +# username: $DOCKER_USER +# password: $DOCKER_PASS +# environment: +# DJANGO_SETTINGS_MODULE: settings +# DJANGO_CONFIGURATION: Test +# DJANGO_SECRET_KEY: ThisIsAnExampleKeyForTestPurposeOnly +# JWT_JITSI_SECRET_KEY: ThisIsAnExampleKeyForDevPurposeOnly +# MAGNIFY_API_URL: http://localhost:8070/api +# KEYCLOAK_URL: http://localhost:8080 +# PYTHONPATH: /home/circleci/fun/sandbox +# DB_HOST: localhost +# DB_NAME: magnify +# DB_USER: fun +# DB_PASSWORD: pass +# DB_PORT: 5432 +# # services +# - image: cimg/postgres:14.2 +# auth: +# username: $DOCKER_USER +# password: $DOCKER_PASS +# environment: +# POSTGRES_DB: test_magnify +# POSTGRES_USER: fun +# POSTGRES_PASSWORD: pass +# - image: docker.io/bitnami/redis:6.0-debian-10 +# auth: +# username: $DOCKER_USER +# password: $DOCKER_PASS +# name: redis-primary +# environment: +# ALLOW_EMPTY_PASSWORD: yes +# REDIS_REPLICATION_MODE: master +# - image: docker.io/bitnami/redis-sentinel:6.0-debian-10 +# auth: +# username: $DOCKER_USER +# password: $DOCKER_PASS +# name: redis-sentinel +# environment: +# REDIS_MASTER_HOST: redis-primary +# working_directory: ~/fun +# steps: +# - checkout +# - restore_cache: +# keys: +# - v2-back-dependencies-{{ .Revision }} +# # Attach the frontend production build +# - attach_workspace: +# at: ~/fun +# # While running tests, we need to make the /data directory writable for +# # the circleci user +# - run: +# name: Create writable /data +# command: | +# sudo mkdir /data && \ +# sudo chown circleci:circleci /data +# # Run back-end (Django) test suite +# # +# # Nota bene: to run the django test suite, we need to ensure that the +# # MySQL service is up and ready. To achieve this, we wrap the pytest +# # command execution with dockerize, a tiny tool installed +# # in the CircleCI image. In our case, dockerize will wait up to one minute +# # that the database container opened its expected tcp port (3306). +# - run: +# name: Run tests +# command: | +# dockerize \ +# -wait tcp://localhost:5432 \ +# -timeout 60s \ +# ~/.local/bin/pytest +# +# # ---- Packaging jobs ---- +# package-back: +# docker: +# - image: cimg/python:3.10 +# auth: +# username: $DOCKER_USER +# password: $DOCKER_PASS +# working_directory: ~/fun +# steps: +# - checkout +# # Ensure we restore frontend production builds in magnify's static +# # directory +# - attach_workspace: +# at: ~/fun +# - run: +# name: Build python package +# command: python setup.py sdist bdist_wheel +# # Persist build packages to the workspace +# - persist_to_workspace: +# root: ~/fun +# paths: +# - dist +# # Store packages as artifacts to download/test them +# - store_artifacts: +# path: ~/fun/dist +# +# # Publishing to PyPI requires that: +# # * you already registered to pypi.org +# # * you have define both the TWINE_USERNAME & TWINE_PASSWORD secret +# # environment variables in CircleCI UI (with your PyPI credentials) +# pypi: +# docker: +# - image: cimg/python:3.10 +# auth: +# username: $DOCKER_USER +# password: $DOCKER_PASS +# working_directory: ~/fun +# steps: +# - checkout +# # Restore built python packages +# - attach_workspace: +# at: ~/fun +# - run: +# name: List built packages +# command: ls dist/* +# - run: +# name: Install base requirements (twine) +# command: pip install --user .[ci] +# - run: +# name: Upload built packages to PyPI +# command: ~/.local/bin/twine upload dist/* +# +# # ---- Front-end jobs ---- +# build-front: +# docker: +# - image: cimg/node:16.15 +# auth: +# username: $DOCKER_USER +# password: $DOCKER_PASS +# working_directory: ~/fun/src/frontend +# steps: +# - *checkout_fun +# - *restore_node_modules +# # If the yarn.lock file is not up-to-date with the package.json file, +# # using the --frozen-lockfile should fail. +# - run: +# name: Install front-end dependencies +# command: yarn install --frozen-lockfile +# - run: +# name: Build front-end +# command: yarn build +# - run: +# name: Use formatjs-cli to generate frontend.json files +# command: yarn extract-translations +# - persist_to_workspace: +# root: ~/fun +# paths: +# - src/frontend/sandbox/i18n/frontend.json +# - src/frontend/packages/core/i18n/frontend.json +# - src/frontend/packages/core/dist +# - save_cache: +# paths: +# - ./node_modules +# - ./sandbox/node_modules +# - ./packages/core/node_modules +# key: v18-front-dependencies-{{ checksum "yarn.lock" }} +# +# lint-front: +# docker: +# - image: cimg/node:16.15 +# auth: +# username: $DOCKER_USER +# password: $DOCKER_PASS +# working_directory: ~/fun/src/frontend +# steps: +# - *checkout_fun +# - *restore_node_modules +# - run: +# name: Lint code with eslint +# command: yarn lint +# - run: +# name: Lint code with prettier +# command: yarn format:check +# +# test-front: +# docker: +# - image: cimg/node:16.15 +# auth: +# username: $DOCKER_USER +# password: $DOCKER_PASS +# parallelism: 5 +# resource_class: large +# working_directory: ~/fun/src/frontend +# steps: +# - *checkout_fun +# - *restore_node_modules +# - run: +# path: packages/core +# name: Test @openfun/jitsi-magnify +# command: | +# TEST=$(circleci tests glob "src/**/*.test.tsx" | circleci tests split --split-by=timings) +# yarn test $TEST +# - run: +# name: Test frontend demo +# command: yarn test-demo +# +# # Publishing to npm requires that you have define the NPM_TOKEN secret +# # environment variables in CircleCI UI (with your PyPI credentials) +# npm: +# docker: +# - image: cimg/node:16.15 +# auth: +# username: $DOCKER_USER +# password: $DOCKER_PASS +# working_directory: ~/fun +# steps: +# - *checkout_fun +# - attach_workspace: +# at: ~/fun +# - run: +# name: Authenticate with registry +# command: echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/fun/.npmrc +# - run: +# name: Publish @openfun/jitsi-magnify package +# command: npm publish src/frontend/packages/core +# +# # ---- DockerHub publication job ---- +# hub: +# docker: +# - image: cimg/base:2022.05 +# working_directory: ~/fun +# steps: +# - *checkout_fun +# # Generate a version.json file describing app release +# - <<: *generate-version-file +# # Activate docker-in-docker +# - setup_remote_docker: +# version: 19.03.13 +# - run: +# name: Build production image +# command: docker build -t magnify:${CIRCLE_SHA1} --target production . +# - run: +# name: Check built images availability +# command: docker images "magnify:${CIRCLE_SHA1}*" +# # Login to DockerHub to Publish new images +# # +# # Nota bene: you'll need to define the following secrets environment vars +# # in CircleCI interface: +# # +# # - DOCKER_HUB_USER +# # - DOCKER_HUB_PASSWORD +# - run: +# name: Login to DockerHub +# command: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin +# # Tag docker images with the same pattern used in Git (Semantic Versioning) +# # +# # Git tag: v1.0.1 +# # Docker tag: 1.0.1(-ci) +# - run: +# name: Tag images +# command: | +# docker images fundocker/jitsi-magnify +# DOCKER_TAG=$([[ -z "$CIRCLE_TAG" ]] && echo $CIRCLE_BRANCH || echo ${CIRCLE_TAG} | sed 's/^v//') +# RELEASE_TYPE=$([[ -z "$CIRCLE_TAG" ]] && echo "branch" || echo "tag ") +# # Display either: +# # - DOCKER_TAG: master (Git branch) +# # or +# # - DOCKER_TAG: 1.1.2 (Git tag v1.1.2) +# echo "DOCKER_TAG: ${DOCKER_TAG} (Git ${RELEASE_TYPE}${CIRCLE_TAG})" +# docker tag magnify:${CIRCLE_SHA1} fundocker/jitsi-magnify:${DOCKER_TAG} +# if [[ -n "$CIRCLE_TAG" ]]; then +# docker tag magnify:${CIRCLE_SHA1} fundocker/jitsi-magnify:latest +# fi +# docker images | grep -E "^fundocker/jitsi-magnify\s*(${DOCKER_TAG}.*|latest|master)" +# +# # Publish images to DockerHub +# # +# # Nota bene: logged user (see "Login to DockerHub" step) must have write +# # permission for the project's repository; this also implies that the +# # DockerHub repository already exists. +# - run: +# name: Publish images +# command: | +# DOCKER_TAG=$([[ -z "$CIRCLE_TAG" ]] && echo $CIRCLE_BRANCH || echo ${CIRCLE_TAG} | sed 's/^v//') +# RELEASE_TYPE=$([[ -z "$CIRCLE_TAG" ]] && echo "branch" || echo "tag ") +# # Display either: +# # - DOCKER_TAG: master (Git branch) +# # or +# # - DOCKER_TAG: 1.1.2 (Git tag v1.1.2) +# echo "DOCKER_TAG: ${DOCKER_TAG} (Git ${RELEASE_TYPE}${CIRCLE_TAG})" +# docker push fundocker/jitsi-magnify:${DOCKER_TAG} +# if [[ -n "$CIRCLE_TAG" ]]; then +# docker push fundocker/jitsi-magnify:latest +# fi +# +#workflows: +# version: 2 +# +# magnify: +# jobs: +# # Front-end jobs +# # +# # Build, lint and test the front-end apps +# - build-front: +# filters: +# tags: +# only: /.*/ +# - lint-front: +# requires: +# - build-front +# filters: +# tags: +# only: /.*/ +# - test-front: +# requires: +# - lint-front +# filters: +# tags: +# only: /.*/ +# +# # Git jobs +# # +# # Check validity of git history +# - lint-git: +# filters: +# tags: +# only: /.*/ +# # Check CHANGELOG update +# - check-changelog: +# filters: +# branches: +# ignore: main +# tags: +# only: /(?!^v).*/ +# - lint-changelog: +# filters: +# branches: +# ignore: main +# tags: +# only: /.*/ +# # Check Renovate configuration +# - check-renovate-configuration: +# filters: +# tags: +# only: /.*/ +# # Check on each PR if the last magnify version is present everywhere it should be. +# # If not the build will fail before publishing a new release. +# - check-versions: +# filters: +# tags: +# only: /.*/ +# +# # Docker jobs +# # +# # Build images +# - build-docker: +# filters: +# tags: +# only: /.*/ +# +# # Backend jobs +# # +# # Build, lint and test production and development Docker images +# # (debian-based) +# - build-back: +# filters: +# tags: +# only: /.*/ +# - lint-back: +# requires: +# - build-back +# filters: +# tags: +# only: /.*/ +# - test-back-mysql-8: +# requires: +# - build-back +# filters: +# tags: +# only: /.*/ +# - test-back-postgresql: +# requires: +# - build-back +# filters: +# tags: +# only: /.*/ +# +# # i18n jobs +# # +# # Extract strings and upload them to our translation management SaaS +# - build-back-i18n: +# requires: +# - build-back +# filters: +# tags: +# only: /.*/ +# - upload-i18n-strings: +# requires: +# - build-front +# - build-back-i18n +# filters: +# branches: +# only: main +# +# # Packaging: python +# # +# # Build the python package +# - package-back: +# requires: +# - test-front +# - test-back-mysql-8 +# - test-back-postgresql +# - build-front +# filters: +# tags: +# only: /.*/ +# +# # PyPI publication. +# # +# # Publish python package to PYPI only if all build, lint and test jobs +# # succeed and it has been tagged with a tag starting with the letter v +# - pypi: +# requires: +# - check-versions +# - package-back +# filters: +# branches: +# ignore: /.*/ +# tags: +# only: /^v.*/ +# +# # NPM publication. +# # +# # Publish frontend package to NPM only if all build, lint and test jobs +# # succeed, and it has been tagged with a tag starting with the letter v +# - npm: +# requires: +# - check-versions +# - build-front +# - lint-front +# - test-front +# filters: +# branches: +# ignore: /.*/ +# tags: +# only: /^v.*/ +# +# # DockerHub publication. +# # +# # Publish docker images only if all build, lint and test jobs succeed +# # and it has been tagged with a tag starting with the letter v or is on +# # the main branch +# - hub: +# requires: +# - build-docker +# - test-back-mysql-8 +# - test-back-postgresql +# filters: +# branches: +# only: main +# tags: +# only: /^v.*/ From 652170d3a966590f2093be61f45f60bb5d608c12 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Wed, 10 Apr 2024 17:47:05 +0200 Subject: [PATCH 04/44] jobbb --- .circleci/config.yml | 1596 +++++++++++++++++++++--------------------- 1 file changed, 798 insertions(+), 798 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a3c32133..4b1d1b20b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,798 +1,798 @@ -## Configuration file anchors -#generate-version-file: &generate-version-file -# run: -# name: Create a version.json -# command: | -# # Create a version.json à-la-mozilla -# # https://github.com/mozilla-services/Dockerflow/blob/master/docs/version_object.md -# printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ -# "$CIRCLE_SHA1" \ -# "$CIRCLE_TAG" \ -# "$CIRCLE_PROJECT_USERNAME" \ -# "$CIRCLE_PROJECT_REPONAME" \ -# "$CIRCLE_BUILD_URL" > sandbox/version.json -# -#version: 2 -# -#aliases: -# - &checkout_fun -# checkout: -# path: ~/fun -# -# - &restore_node_modules -# restore_cache: -# name: Restore node_modules cache -# keys: -# - v18-front-dependencies-{{ checksum "~/fun/src/frontend/yarn.lock" }} -# - v18-front-dependencies- -# -#jobs: -# # Git jobs -# # Check that the git history is clean and complies with our expectations -# lint-git: -# docker: -# - image: cimg/python:3.10 -# auth: -# username: $DOCKER_USER -# password: $DOCKER_PASS -# working_directory: ~/fun -# steps: -# - checkout -# # Make sure the changes don't add a "print" statement to the code base. -# # We should exclude the ".circleci" folder from the search as the very command that checks -# # the absence of "print" is including a "print(" itself. -# - run: -# name: enforce absence of print statements in code -# command: | -# ! git diff origin/main..HEAD -- ":(exclude)*.circleci/*" | grep "print(" -# - run: -# name: Check absence of fixup commits -# command: | -# ! git log | grep 'fixup!' -# - run: -# name: Install gitlint -# command: | -# pip install --user requests gitlint -# - run: -# name: lint commit messages added to main -# command: | -# ~/.local/bin/gitlint --commits origin/main..HEAD -# -# # Check that the CHANGELOG has been updated in the current branch -# check-changelog: -# docker: -# - image: circleci/buildpack-deps:stretch-scm -# auth: -# username: $DOCKER_USER -# password: $DOCKER_PASS -# working_directory: ~/fun -# steps: -# - checkout -# - run: -# name: Check that the CHANGELOG has been modified in the current branch -# command: | -# git whatchanged --name-only --pretty="" origin..HEAD | grep CHANGELOG -# -# # Check that the CHANGELOG max line length does not exceed 80 characters -# lint-changelog: -# docker: -# - image: debian:stretch -# auth: -# username: $DOCKER_USER -# password: $DOCKER_PASS -# working_directory: ~/fun -# steps: -# - checkout -# - run: -# name: Check CHANGELOG max line length -# command: | -# # Get the longuest line width (ignoring release links) -# test $(cat CHANGELOG.md | grep -Ev "^\[.*\]: https://github.com/openfun" | wc -L) -le 80 -# -# # Check that renovate configuration file is valid -# check-renovate-configuration: -# docker: -# - image: renovate/renovate -# auth: -# username: $DOCKER_USER -# password: $DOCKER_PASS -# working_directory: ~/fun -# steps: -# - checkout -# - run: -# name: Run renovate-config-validator command -# command: renovate-config-validator -# -# # Check that all versions (backend, frontend) are up-to-date -# check-versions: -# docker: -# - image: cimg/base:2022.04 -# working_directory: ~/fun -# steps: -# - checkout -# - run: -# name: Check that all versions are the same -# command: | -# BACKEND_VERSION=$(cat setup.cfg | grep "version" | cut -d" " -f3) -# echo "magnify version: ${BACKEND_VERSION}" -# # In the frontend workspace -# cat src/frontend/package.json | grep "\"version\": \"${BACKEND_VERSION}\",$" -# # In the "@openfun/jitsi-magnify" frontend library -# cat src/frontend/packages/core/package.json | grep "\"version\": \"${BACKEND_VERSION}\",$" -# # In the "sandbox" frontend application -# cat src/frontend/sandbox/package.json | grep "\"version\": \"${BACKEND_VERSION}\",$" -# -# # ---- Docker jobs ---- -# # Build the Docker image ready for production -# build-docker: -# docker: -# - image: circleci/buildpack-deps:stretch -# auth: -# username: $DOCKER_USER -# password: $DOCKER_PASS -# working_directory: ~/fun -# steps: -# # Checkout repository sources -# - checkout -# # Generate a version.json file describing app release -# - <<: *generate-version-file -# # Activate docker-in-docker -# - setup_remote_docker: -# version: 19.03.13 -# -# # Login to Docker Hub with encrypted credentials stored as secret -# # environment variables (set in CircleCI project settings) if the expected -# # environment variable is set; switch to anonymous mode otherwise. -# - run: -# name: Login to DockerHub -# command: > -# test -n "$DOCKER_USER" && -# echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin || -# echo "Docker Hub anonymous mode" -# -# # Each image is tagged with the current git commit sha1 to avoid -# # collisions in parallel builds. -# - run: -# name: Build production image -# command: docker build -t magnify:${CIRCLE_SHA1} --target production . -# - run: -# name: Check built image availability -# command: docker images "magnify:${CIRCLE_SHA1}*" -# -# # ---- Backend jobs ---- -# # Build backend development environment -# build-back: -# docker: -# - image: cimg/python:3.10 -# auth: -# username: $DOCKER_USER -# password: $DOCKER_PASS -# working_directory: ~/fun -# steps: -# - checkout -# - restore_cache: -# keys: -# - v2-back-dependencies-{{ .Revision }} -# - run: -# name: Install development dependencies -# command: pip install --user .[dev,sandbox] -# - save_cache: -# paths: -# - ~/.local -# key: v2-back-dependencies-{{ .Revision }} -# -# # Build backend translations -# build-back-i18n: -# docker: -# - image: cimg/python:3.10 -# auth: -# username: $DOCKER_USER -# password: $DOCKER_PASS -# working_directory: ~/fun/src/magnify -# steps: -# - *checkout_fun -# - restore_cache: -# keys: -# - v2-back-dependencies-{{ .Revision }} -# - run: -# name: Install gettext (required to make messages) -# command: | -# sudo apt-get update -# sudo apt-get install -y gettext -# # Generate and persist the translations base file -# - run: -# name: Generate a POT file from strings extracted from the project -# command: ~/.local/bin/django-admin makemessages --keep-pot --all -# - persist_to_workspace: -# root: ~/fun -# paths: -# - src/magnify/locale/django.pot -# -# lint-back: -# docker: -# - image: cimg/python:3.10 -# auth: -# username: $DOCKER_USER -# password: $DOCKER_PASS -# working_directory: ~/fun -# steps: -# - checkout -# - restore_cache: -# keys: -# - v2-back-dependencies-{{ .Revision }} -# - run: -# name: Lint code with flake8 -# command: ~/.local/bin/flake8 src/magnify/apps sandbox tests -# - run: -# name: Lint code with isort -# command: ~/.local/bin/isort --check-only src/magnify/apps sandbox tests -# - run: -# name: Lint code with black -# command: ~/.local/bin/black . --check -# - run: -# name: Lint code with pylint -# command: ~/.local/bin/pylint src/magnify/apps sandbox tests -# - run: -# name: Lint code with bandit -# command: ~/.local/bin/bandit -qr src/magnify/apps sandbox -# -# # Restore back POT & front json files containing strings to translate and upload them to our -# # translation management tool -# upload-i18n-strings: -# docker: -# - image: crowdin/cli:3.3.0 -# auth: -# username: $DOCKER_USER -# password: $DOCKER_PASS -# working_directory: ~/fun -# steps: -# - checkout -# - attach_workspace: -# at: ~/fun -# - run: -# name: upload files to crowdin -# command: crowdin upload sources -c crowdin/config.yml -# -# test-back-mysql-8: -# docker: -# - image: cimg/python:3.10 -# auth: -# username: $DOCKER_USER -# password: $DOCKER_PASS -# environment: -# DJANGO_SETTINGS_MODULE: settings -# DJANGO_CONFIGURATION: Test -# DJANGO_SECRET_KEY: ThisIsAnExampleKeyForTestPurposeOnly -# JWT_JITSI_SECRET_KEY: ThisIsAnExampleKeyForDevPurposeOnly -# MAGNIFY_API_URL: http://localhost:8070/api -# KEYCLOAK_URL: http://localhost:8080 -# PYTHONPATH: /home/circleci/fun/sandbox -# magnify_ES_HOST: localhost -# DB_ENGINE: django.db.backends.mysql -# # The DB_HOST should match the host name and cannot be set from here -# # where it will be escaped. See the test command instead: -# # DB_HOST=${HOSTNAME} -# DB_HOST: -# DB_NAME: magnify -# DB_USER: fun -# DB_PASSWORD: pass -# DB_PORT: 3306 -# # services -# - image: circleci/mysql:8.0-ram -# auth: -# username: $DOCKER_USER -# password: $DOCKER_PASS -# environment: -# MYSQL_ROOT_PASSWORD: -# MYSQL_DATABASE: test_magnify -# MYSQL_USER: fun -# MYSQL_PASSWORD: pass -# command: --default-authentication-plugin=mysql_native_password -# - image: docker.io/bitnami/redis:6.0-debian-10 -# auth: -# username: $DOCKER_USER -# password: $DOCKER_PASS -# name: redis-primary -# environment: -# ALLOW_EMPTY_PASSWORD: yes -# REDIS_REPLICATION_MODE: master -# - image: docker.io/bitnami/redis-sentinel:6.0-debian-10 -# auth: -# username: $DOCKER_USER -# password: $DOCKER_PASS -# name: redis-sentinel -# environment: -# REDIS_MASTER_HOST: redis-primary -# working_directory: ~/fun -# steps: -# - checkout -# - restore_cache: -# keys: -# - v2-back-dependencies-{{ .Revision }} -# # Attach the frontend production build -# - attach_workspace: -# at: ~/fun -# # While running tests, we need to make the /data directory writable for -# # the circleci user -# - run: -# name: Create writable /data -# command: | -# sudo mkdir /data && \ -# sudo chown circleci:circleci /data -# # Run back-end (Django) test suite -# # -# # Nota bene: -# # -# # 1. to run the django test suite, we need to ensure that the MySQL -# # service is up and ready. To achieve this, we wrap the -# # pytest command execution with dockerize, a tiny tool installed in the -# # CircleCI image. In our case, dockerize will wait up to one minute -# # that the database container opened its expected tcp port (3306). -# # 2. We should avoid using localhost for the DB_HOST with MySQL as the -# # client will try to use a local socket (_e.g._ -# # `/var/run/mysqld/mysqld.sock`) instead of the database host and port -# # ¯\_(ツ)_/¯. -# - run: -# name: Run tests -# command: | -# DB_HOST=${HOSTNAME} dockerize \ -# -wait tcp://${HOSTNAME}:3306 \ -# -timeout 60s \ -# ~/.local/bin/pytest -# -# test-back-postgresql: -# docker: -# - image: cimg/python:3.10 -# auth: -# username: $DOCKER_USER -# password: $DOCKER_PASS -# environment: -# DJANGO_SETTINGS_MODULE: settings -# DJANGO_CONFIGURATION: Test -# DJANGO_SECRET_KEY: ThisIsAnExampleKeyForTestPurposeOnly -# JWT_JITSI_SECRET_KEY: ThisIsAnExampleKeyForDevPurposeOnly -# MAGNIFY_API_URL: http://localhost:8070/api -# KEYCLOAK_URL: http://localhost:8080 -# PYTHONPATH: /home/circleci/fun/sandbox -# DB_HOST: localhost -# DB_NAME: magnify -# DB_USER: fun -# DB_PASSWORD: pass -# DB_PORT: 5432 -# # services -# - image: cimg/postgres:14.2 -# auth: -# username: $DOCKER_USER -# password: $DOCKER_PASS -# environment: -# POSTGRES_DB: test_magnify -# POSTGRES_USER: fun -# POSTGRES_PASSWORD: pass -# - image: docker.io/bitnami/redis:6.0-debian-10 -# auth: -# username: $DOCKER_USER -# password: $DOCKER_PASS -# name: redis-primary -# environment: -# ALLOW_EMPTY_PASSWORD: yes -# REDIS_REPLICATION_MODE: master -# - image: docker.io/bitnami/redis-sentinel:6.0-debian-10 -# auth: -# username: $DOCKER_USER -# password: $DOCKER_PASS -# name: redis-sentinel -# environment: -# REDIS_MASTER_HOST: redis-primary -# working_directory: ~/fun -# steps: -# - checkout -# - restore_cache: -# keys: -# - v2-back-dependencies-{{ .Revision }} -# # Attach the frontend production build -# - attach_workspace: -# at: ~/fun -# # While running tests, we need to make the /data directory writable for -# # the circleci user -# - run: -# name: Create writable /data -# command: | -# sudo mkdir /data && \ -# sudo chown circleci:circleci /data -# # Run back-end (Django) test suite -# # -# # Nota bene: to run the django test suite, we need to ensure that the -# # MySQL service is up and ready. To achieve this, we wrap the pytest -# # command execution with dockerize, a tiny tool installed -# # in the CircleCI image. In our case, dockerize will wait up to one minute -# # that the database container opened its expected tcp port (3306). -# - run: -# name: Run tests -# command: | -# dockerize \ -# -wait tcp://localhost:5432 \ -# -timeout 60s \ -# ~/.local/bin/pytest -# -# # ---- Packaging jobs ---- -# package-back: -# docker: -# - image: cimg/python:3.10 -# auth: -# username: $DOCKER_USER -# password: $DOCKER_PASS -# working_directory: ~/fun -# steps: -# - checkout -# # Ensure we restore frontend production builds in magnify's static -# # directory -# - attach_workspace: -# at: ~/fun -# - run: -# name: Build python package -# command: python setup.py sdist bdist_wheel -# # Persist build packages to the workspace -# - persist_to_workspace: -# root: ~/fun -# paths: -# - dist -# # Store packages as artifacts to download/test them -# - store_artifacts: -# path: ~/fun/dist -# -# # Publishing to PyPI requires that: -# # * you already registered to pypi.org -# # * you have define both the TWINE_USERNAME & TWINE_PASSWORD secret -# # environment variables in CircleCI UI (with your PyPI credentials) -# pypi: -# docker: -# - image: cimg/python:3.10 -# auth: -# username: $DOCKER_USER -# password: $DOCKER_PASS -# working_directory: ~/fun -# steps: -# - checkout -# # Restore built python packages -# - attach_workspace: -# at: ~/fun -# - run: -# name: List built packages -# command: ls dist/* -# - run: -# name: Install base requirements (twine) -# command: pip install --user .[ci] -# - run: -# name: Upload built packages to PyPI -# command: ~/.local/bin/twine upload dist/* -# -# # ---- Front-end jobs ---- -# build-front: -# docker: -# - image: cimg/node:16.15 -# auth: -# username: $DOCKER_USER -# password: $DOCKER_PASS -# working_directory: ~/fun/src/frontend -# steps: -# - *checkout_fun -# - *restore_node_modules -# # If the yarn.lock file is not up-to-date with the package.json file, -# # using the --frozen-lockfile should fail. -# - run: -# name: Install front-end dependencies -# command: yarn install --frozen-lockfile -# - run: -# name: Build front-end -# command: yarn build -# - run: -# name: Use formatjs-cli to generate frontend.json files -# command: yarn extract-translations -# - persist_to_workspace: -# root: ~/fun -# paths: -# - src/frontend/sandbox/i18n/frontend.json -# - src/frontend/packages/core/i18n/frontend.json -# - src/frontend/packages/core/dist -# - save_cache: -# paths: -# - ./node_modules -# - ./sandbox/node_modules -# - ./packages/core/node_modules -# key: v18-front-dependencies-{{ checksum "yarn.lock" }} -# -# lint-front: -# docker: -# - image: cimg/node:16.15 -# auth: -# username: $DOCKER_USER -# password: $DOCKER_PASS -# working_directory: ~/fun/src/frontend -# steps: -# - *checkout_fun -# - *restore_node_modules -# - run: -# name: Lint code with eslint -# command: yarn lint -# - run: -# name: Lint code with prettier -# command: yarn format:check -# -# test-front: -# docker: -# - image: cimg/node:16.15 -# auth: -# username: $DOCKER_USER -# password: $DOCKER_PASS -# parallelism: 5 -# resource_class: large -# working_directory: ~/fun/src/frontend -# steps: -# - *checkout_fun -# - *restore_node_modules -# - run: -# path: packages/core -# name: Test @openfun/jitsi-magnify -# command: | -# TEST=$(circleci tests glob "src/**/*.test.tsx" | circleci tests split --split-by=timings) -# yarn test $TEST -# - run: -# name: Test frontend demo -# command: yarn test-demo -# -# # Publishing to npm requires that you have define the NPM_TOKEN secret -# # environment variables in CircleCI UI (with your PyPI credentials) -# npm: -# docker: -# - image: cimg/node:16.15 -# auth: -# username: $DOCKER_USER -# password: $DOCKER_PASS -# working_directory: ~/fun -# steps: -# - *checkout_fun -# - attach_workspace: -# at: ~/fun -# - run: -# name: Authenticate with registry -# command: echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/fun/.npmrc -# - run: -# name: Publish @openfun/jitsi-magnify package -# command: npm publish src/frontend/packages/core -# -# # ---- DockerHub publication job ---- -# hub: -# docker: -# - image: cimg/base:2022.05 -# working_directory: ~/fun -# steps: -# - *checkout_fun -# # Generate a version.json file describing app release -# - <<: *generate-version-file -# # Activate docker-in-docker -# - setup_remote_docker: -# version: 19.03.13 -# - run: -# name: Build production image -# command: docker build -t magnify:${CIRCLE_SHA1} --target production . -# - run: -# name: Check built images availability -# command: docker images "magnify:${CIRCLE_SHA1}*" -# # Login to DockerHub to Publish new images -# # -# # Nota bene: you'll need to define the following secrets environment vars -# # in CircleCI interface: -# # -# # - DOCKER_HUB_USER -# # - DOCKER_HUB_PASSWORD -# - run: -# name: Login to DockerHub -# command: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin -# # Tag docker images with the same pattern used in Git (Semantic Versioning) -# # -# # Git tag: v1.0.1 -# # Docker tag: 1.0.1(-ci) -# - run: -# name: Tag images -# command: | -# docker images fundocker/jitsi-magnify -# DOCKER_TAG=$([[ -z "$CIRCLE_TAG" ]] && echo $CIRCLE_BRANCH || echo ${CIRCLE_TAG} | sed 's/^v//') -# RELEASE_TYPE=$([[ -z "$CIRCLE_TAG" ]] && echo "branch" || echo "tag ") -# # Display either: -# # - DOCKER_TAG: master (Git branch) -# # or -# # - DOCKER_TAG: 1.1.2 (Git tag v1.1.2) -# echo "DOCKER_TAG: ${DOCKER_TAG} (Git ${RELEASE_TYPE}${CIRCLE_TAG})" -# docker tag magnify:${CIRCLE_SHA1} fundocker/jitsi-magnify:${DOCKER_TAG} -# if [[ -n "$CIRCLE_TAG" ]]; then -# docker tag magnify:${CIRCLE_SHA1} fundocker/jitsi-magnify:latest -# fi -# docker images | grep -E "^fundocker/jitsi-magnify\s*(${DOCKER_TAG}.*|latest|master)" -# -# # Publish images to DockerHub -# # -# # Nota bene: logged user (see "Login to DockerHub" step) must have write -# # permission for the project's repository; this also implies that the -# # DockerHub repository already exists. -# - run: -# name: Publish images -# command: | -# DOCKER_TAG=$([[ -z "$CIRCLE_TAG" ]] && echo $CIRCLE_BRANCH || echo ${CIRCLE_TAG} | sed 's/^v//') -# RELEASE_TYPE=$([[ -z "$CIRCLE_TAG" ]] && echo "branch" || echo "tag ") -# # Display either: -# # - DOCKER_TAG: master (Git branch) -# # or -# # - DOCKER_TAG: 1.1.2 (Git tag v1.1.2) -# echo "DOCKER_TAG: ${DOCKER_TAG} (Git ${RELEASE_TYPE}${CIRCLE_TAG})" -# docker push fundocker/jitsi-magnify:${DOCKER_TAG} -# if [[ -n "$CIRCLE_TAG" ]]; then -# docker push fundocker/jitsi-magnify:latest -# fi -# -#workflows: -# version: 2 -# -# magnify: -# jobs: -# # Front-end jobs -# # -# # Build, lint and test the front-end apps -# - build-front: -# filters: -# tags: -# only: /.*/ -# - lint-front: -# requires: -# - build-front -# filters: -# tags: -# only: /.*/ -# - test-front: -# requires: -# - lint-front -# filters: -# tags: -# only: /.*/ -# -# # Git jobs -# # -# # Check validity of git history -# - lint-git: -# filters: -# tags: -# only: /.*/ -# # Check CHANGELOG update -# - check-changelog: -# filters: -# branches: -# ignore: main -# tags: -# only: /(?!^v).*/ -# - lint-changelog: -# filters: -# branches: -# ignore: main -# tags: -# only: /.*/ -# # Check Renovate configuration -# - check-renovate-configuration: -# filters: -# tags: -# only: /.*/ -# # Check on each PR if the last magnify version is present everywhere it should be. -# # If not the build will fail before publishing a new release. -# - check-versions: -# filters: -# tags: -# only: /.*/ -# -# # Docker jobs -# # -# # Build images -# - build-docker: -# filters: -# tags: -# only: /.*/ -# -# # Backend jobs -# # -# # Build, lint and test production and development Docker images -# # (debian-based) -# - build-back: -# filters: -# tags: -# only: /.*/ -# - lint-back: -# requires: -# - build-back -# filters: -# tags: -# only: /.*/ -# - test-back-mysql-8: -# requires: -# - build-back -# filters: -# tags: -# only: /.*/ -# - test-back-postgresql: -# requires: -# - build-back -# filters: -# tags: -# only: /.*/ -# -# # i18n jobs -# # -# # Extract strings and upload them to our translation management SaaS -# - build-back-i18n: -# requires: -# - build-back -# filters: -# tags: -# only: /.*/ -# - upload-i18n-strings: -# requires: -# - build-front -# - build-back-i18n -# filters: -# branches: -# only: main -# -# # Packaging: python -# # -# # Build the python package -# - package-back: -# requires: -# - test-front -# - test-back-mysql-8 -# - test-back-postgresql -# - build-front -# filters: -# tags: -# only: /.*/ -# -# # PyPI publication. -# # -# # Publish python package to PYPI only if all build, lint and test jobs -# # succeed and it has been tagged with a tag starting with the letter v -# - pypi: -# requires: -# - check-versions -# - package-back -# filters: -# branches: -# ignore: /.*/ -# tags: -# only: /^v.*/ -# -# # NPM publication. -# # -# # Publish frontend package to NPM only if all build, lint and test jobs -# # succeed, and it has been tagged with a tag starting with the letter v -# - npm: -# requires: -# - check-versions -# - build-front -# - lint-front -# - test-front -# filters: -# branches: -# ignore: /.*/ -# tags: -# only: /^v.*/ -# -# # DockerHub publication. -# # -# # Publish docker images only if all build, lint and test jobs succeed -# # and it has been tagged with a tag starting with the letter v or is on -# # the main branch -# - hub: -# requires: -# - build-docker -# - test-back-mysql-8 -# - test-back-postgresql -# filters: -# branches: -# only: main -# tags: -# only: /^v.*/ +# Configuration file anchors +generate-version-file: &generate-version-file + run: + name: Create a version.json + command: | + # Create a version.json à-la-mozilla + # https://github.com/mozilla-services/Dockerflow/blob/master/docs/version_object.md + printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ + "$CIRCLE_SHA1" \ + "$CIRCLE_TAG" \ + "$CIRCLE_PROJECT_USERNAME" \ + "$CIRCLE_PROJECT_REPONAME" \ + "$CIRCLE_BUILD_URL" > sandbox/version.json + +version: 2 + +aliases: + - &checkout_fun + checkout: + path: ~/fun + + - &restore_node_modules + restore_cache: + name: Restore node_modules cache + keys: + - v18-front-dependencies-{{ checksum "~/fun/src/frontend/yarn.lock" }} + - v18-front-dependencies- + +jobs: + # Git jobs + # Check that the git history is clean and complies with our expectations + lint-git: + docker: + - image: cimg/python:3.10 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + working_directory: ~/fun + steps: + - checkout + # Make sure the changes don't add a "print" statement to the code base. + # We should exclude the ".circleci" folder from the search as the very command that checks + # the absence of "print" is including a "print(" itself. + - run: + name: enforce absence of print statements in code + command: | + ! git diff origin/main..HEAD -- ":(exclude)*.circleci/*" | grep "print(" + - run: + name: Check absence of fixup commits + command: | + ! git log | grep 'fixup!' + - run: + name: Install gitlint + command: | + pip install --user requests gitlint + - run: + name: lint commit messages added to main + command: | + ~/.local/bin/gitlint --commits origin/main..HEAD + + # Check that the CHANGELOG has been updated in the current branch + check-changelog: + docker: + - image: circleci/buildpack-deps:stretch-scm + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + working_directory: ~/fun + steps: + - checkout + - run: + name: Check that the CHANGELOG has been modified in the current branch + command: | + git whatchanged --name-only --pretty="" origin..HEAD | grep CHANGELOG + + # Check that the CHANGELOG max line length does not exceed 80 characters + lint-changelog: + docker: + - image: debian:stretch + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + working_directory: ~/fun + steps: + - checkout + - run: + name: Check CHANGELOG max line length + command: | + # Get the longuest line width (ignoring release links) + test $(cat CHANGELOG.md | grep -Ev "^\[.*\]: https://github.com/openfun" | wc -L) -le 80 + + # Check that renovate configuration file is valid + check-renovate-configuration: + docker: + - image: renovate/renovate + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + working_directory: ~/fun + steps: + - checkout + - run: + name: Run renovate-config-validator command + command: renovate-config-validator + + # Check that all versions (backend, frontend) are up-to-date + check-versions: + docker: + - image: cimg/base:2022.04 + working_directory: ~/fun + steps: + - checkout + - run: + name: Check that all versions are the same + command: | + BACKEND_VERSION=$(cat setup.cfg | grep "version" | cut -d" " -f3) + echo "magnify version: ${BACKEND_VERSION}" + # In the frontend workspace + cat src/frontend/package.json | grep "\"version\": \"${BACKEND_VERSION}\",$" + # In the "@openfun/jitsi-magnify" frontend library + cat src/frontend/packages/core/package.json | grep "\"version\": \"${BACKEND_VERSION}\",$" + # In the "sandbox" frontend application + cat src/frontend/sandbox/package.json | grep "\"version\": \"${BACKEND_VERSION}\",$" + + # ---- Docker jobs ---- + # Build the Docker image ready for production + build-docker: + docker: + - image: circleci/buildpack-deps:stretch + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + working_directory: ~/fun + steps: + # Checkout repository sources + - checkout + # Generate a version.json file describing app release + - <<: *generate-version-file + # Activate docker-in-docker + - setup_remote_docker: + version: 19.03.13 + + # Login to Docker Hub with encrypted credentials stored as secret + # environment variables (set in CircleCI project settings) if the expected + # environment variable is set; switch to anonymous mode otherwise. + - run: + name: Login to DockerHub + command: > + test -n "$DOCKER_USER" && + echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin || + echo "Docker Hub anonymous mode" + + # Each image is tagged with the current git commit sha1 to avoid + # collisions in parallel builds. + - run: + name: Build production image + command: docker build -t magnify:${CIRCLE_SHA1} --target production . + - run: + name: Check built image availability + command: docker images "magnify:${CIRCLE_SHA1}*" + + # ---- Backend jobs ---- + # Build backend development environment + build-back: + docker: + - image: cimg/python:3.10 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + working_directory: ~/fun + steps: + - checkout + - restore_cache: + keys: + - v2-back-dependencies-{{ .Revision }} + - run: + name: Install development dependencies + command: pip install --user .[dev,sandbox] + - save_cache: + paths: + - ~/.local + key: v2-back-dependencies-{{ .Revision }} + + # Build backend translations + build-back-i18n: + docker: + - image: cimg/python:3.10 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + working_directory: ~/fun/src/magnify + steps: + - *checkout_fun + - restore_cache: + keys: + - v2-back-dependencies-{{ .Revision }} + - run: + name: Install gettext (required to make messages) + command: | + sudo apt-get update + sudo apt-get install -y gettext + # Generate and persist the translations base file + - run: + name: Generate a POT file from strings extracted from the project + command: ~/.local/bin/django-admin makemessages --keep-pot --all + - persist_to_workspace: + root: ~/fun + paths: + - src/magnify/locale/django.pot + + lint-back: + docker: + - image: cimg/python:3.10 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + working_directory: ~/fun + steps: + - checkout + - restore_cache: + keys: + - v2-back-dependencies-{{ .Revision }} + - run: + name: Lint code with flake8 + command: ~/.local/bin/flake8 src/magnify/apps sandbox tests + - run: + name: Lint code with isort + command: ~/.local/bin/isort --check-only src/magnify/apps sandbox tests + - run: + name: Lint code with black + command: ~/.local/bin/black . --check + - run: + name: Lint code with pylint + command: ~/.local/bin/pylint src/magnify/apps sandbox tests + - run: + name: Lint code with bandit + command: ~/.local/bin/bandit -qr src/magnify/apps sandbox + + # Restore back POT & front json files containing strings to translate and upload them to our + # translation management tool + upload-i18n-strings: + docker: + - image: crowdin/cli:3.3.0 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + working_directory: ~/fun + steps: + - checkout + - attach_workspace: + at: ~/fun + - run: + name: upload files to crowdin + command: crowdin upload sources -c crowdin/config.yml + + test-back-mysql-8: + docker: + - image: cimg/python:3.10 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + environment: + DJANGO_SETTINGS_MODULE: settings + DJANGO_CONFIGURATION: Test + DJANGO_SECRET_KEY: ThisIsAnExampleKeyForTestPurposeOnly + JWT_JITSI_SECRET_KEY: ThisIsAnExampleKeyForDevPurposeOnly + MAGNIFY_API_URL: http://localhost:8070/api + KEYCLOAK_URL: http://localhost:8080 + PYTHONPATH: /home/circleci/fun/sandbox + magnify_ES_HOST: localhost + DB_ENGINE: django.db.backends.mysql + # The DB_HOST should match the host name and cannot be set from here + # where it will be escaped. See the test command instead: + # DB_HOST=${HOSTNAME} + DB_HOST: + DB_NAME: magnify + DB_USER: fun + DB_PASSWORD: pass + DB_PORT: 3306 + # services + - image: circleci/mysql:8.0-ram + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + environment: + MYSQL_ROOT_PASSWORD: + MYSQL_DATABASE: test_magnify + MYSQL_USER: fun + MYSQL_PASSWORD: pass + command: --default-authentication-plugin=mysql_native_password + - image: docker.io/bitnami/redis:6.0-debian-10 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + name: redis-primary + environment: + ALLOW_EMPTY_PASSWORD: yes + REDIS_REPLICATION_MODE: master + - image: docker.io/bitnami/redis-sentinel:6.0-debian-10 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + name: redis-sentinel + environment: + REDIS_MASTER_HOST: redis-primary + working_directory: ~/fun + steps: + - checkout + - restore_cache: + keys: + - v2-back-dependencies-{{ .Revision }} + # Attach the frontend production build + - attach_workspace: + at: ~/fun + # While running tests, we need to make the /data directory writable for + # the circleci user + - run: + name: Create writable /data + command: | + sudo mkdir /data && \ + sudo chown circleci:circleci /data + # Run back-end (Django) test suite + # + # Nota bene: + # + # 1. to run the django test suite, we need to ensure that the MySQL + # service is up and ready. To achieve this, we wrap the + # pytest command execution with dockerize, a tiny tool installed in the + # CircleCI image. In our case, dockerize will wait up to one minute + # that the database container opened its expected tcp port (3306). + # 2. We should avoid using localhost for the DB_HOST with MySQL as the + # client will try to use a local socket (_e.g._ + # `/var/run/mysqld/mysqld.sock`) instead of the database host and port + # ¯\_(ツ)_/¯. + - run: + name: Run tests + command: | + DB_HOST=${HOSTNAME} dockerize \ + -wait tcp://${HOSTNAME}:3306 \ + -timeout 60s \ + ~/.local/bin/pytest + + test-back-postgresql: + docker: + - image: cimg/python:3.10 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + environment: + DJANGO_SETTINGS_MODULE: settings + DJANGO_CONFIGURATION: Test + DJANGO_SECRET_KEY: ThisIsAnExampleKeyForTestPurposeOnly + JWT_JITSI_SECRET_KEY: ThisIsAnExampleKeyForDevPurposeOnly + MAGNIFY_API_URL: http://localhost:8070/api + KEYCLOAK_URL: http://localhost:8080 + PYTHONPATH: /home/circleci/fun/sandbox + DB_HOST: localhost + DB_NAME: magnify + DB_USER: fun + DB_PASSWORD: pass + DB_PORT: 5432 + # services + - image: cimg/postgres:14.2 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + environment: + POSTGRES_DB: test_magnify + POSTGRES_USER: fun + POSTGRES_PASSWORD: pass + - image: docker.io/bitnami/redis:6.0-debian-10 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + name: redis-primary + environment: + ALLOW_EMPTY_PASSWORD: yes + REDIS_REPLICATION_MODE: master + - image: docker.io/bitnami/redis-sentinel:6.0-debian-10 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + name: redis-sentinel + environment: + REDIS_MASTER_HOST: redis-primary + working_directory: ~/fun + steps: + - checkout + - restore_cache: + keys: + - v2-back-dependencies-{{ .Revision }} + # Attach the frontend production build + - attach_workspace: + at: ~/fun + # While running tests, we need to make the /data directory writable for + # the circleci user + - run: + name: Create writable /data + command: | + sudo mkdir /data && \ + sudo chown circleci:circleci /data + # Run back-end (Django) test suite + # + # Nota bene: to run the django test suite, we need to ensure that the + # MySQL service is up and ready. To achieve this, we wrap the pytest + # command execution with dockerize, a tiny tool installed + # in the CircleCI image. In our case, dockerize will wait up to one minute + # that the database container opened its expected tcp port (3306). + - run: + name: Run tests + command: | + dockerize \ + -wait tcp://localhost:5432 \ + -timeout 60s \ + ~/.local/bin/pytest + + # ---- Packaging jobs ---- + package-back: + docker: + - image: cimg/python:3.10 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + working_directory: ~/fun + steps: + - checkout + # Ensure we restore frontend production builds in magnify's static + # directory + - attach_workspace: + at: ~/fun + - run: + name: Build python package + command: python setup.py sdist bdist_wheel + # Persist build packages to the workspace + - persist_to_workspace: + root: ~/fun + paths: + - dist + # Store packages as artifacts to download/test them + - store_artifacts: + path: ~/fun/dist + + # Publishing to PyPI requires that: + # * you already registered to pypi.org + # * you have define both the TWINE_USERNAME & TWINE_PASSWORD secret + # environment variables in CircleCI UI (with your PyPI credentials) + pypi: + docker: + - image: cimg/python:3.10 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + working_directory: ~/fun + steps: + - checkout + # Restore built python packages + - attach_workspace: + at: ~/fun + - run: + name: List built packages + command: ls dist/* + - run: + name: Install base requirements (twine) + command: pip install --user .[ci] + - run: + name: Upload built packages to PyPI + command: ~/.local/bin/twine upload dist/* + + # ---- Front-end jobs ---- + build-front: + docker: + - image: cimg/node:16.15 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + working_directory: ~/fun/src/frontend + steps: + - *checkout_fun + - *restore_node_modules + # If the yarn.lock file is not up-to-date with the package.json file, + # using the --frozen-lockfile should fail. + - run: + name: Install front-end dependencies + command: yarn install --frozen-lockfile + - run: + name: Build front-end + command: yarn build + - run: + name: Use formatjs-cli to generate frontend.json files + command: yarn extract-translations + - persist_to_workspace: + root: ~/fun + paths: + - src/frontend/sandbox/i18n/frontend.json + - src/frontend/packages/core/i18n/frontend.json + - src/frontend/packages/core/dist + - save_cache: + paths: + - ./node_modules + - ./sandbox/node_modules + - ./packages/core/node_modules + key: v18-front-dependencies-{{ checksum "yarn.lock" }} + + lint-front: + docker: + - image: cimg/node:16.15 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + working_directory: ~/fun/src/frontend + steps: + - *checkout_fun + - *restore_node_modules + - run: + name: Lint code with eslint + command: yarn lint + - run: + name: Lint code with prettier + command: yarn format:check + + test-front: + docker: + - image: cimg/node:16.15 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + parallelism: 5 + resource_class: large + working_directory: ~/fun/src/frontend + steps: + - *checkout_fun + - *restore_node_modules + - run: + path: packages/core + name: Test @openfun/jitsi-magnify + command: | + TEST=$(circleci tests glob "src/**/*.test.tsx" | circleci tests split --split-by=timings) + yarn test $TEST + - run: + name: Test frontend demo + command: yarn test-demo + + # Publishing to npm requires that you have define the NPM_TOKEN secret + # environment variables in CircleCI UI (with your PyPI credentials) + npm: + docker: + - image: cimg/node:16.15 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + working_directory: ~/fun + steps: + - *checkout_fun + - attach_workspace: + at: ~/fun + - run: + name: Authenticate with registry + command: echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/fun/.npmrc + - run: + name: Publish @openfun/jitsi-magnify package + command: npm publish src/frontend/packages/core + + # ---- DockerHub publication job ---- + hub: + docker: + - image: cimg/base:2022.05 + working_directory: ~/fun + steps: + - *checkout_fun + # Generate a version.json file describing app release + - <<: *generate-version-file + # Activate docker-in-docker + - setup_remote_docker: + version: 19.03.13 + - run: + name: Build production image + command: docker build -t magnify:${CIRCLE_SHA1} --target production . + - run: + name: Check built images availability + command: docker images "magnify:${CIRCLE_SHA1}*" + # Login to DockerHub to Publish new images + # + # Nota bene: you'll need to define the following secrets environment vars + # in CircleCI interface: + # + # - DOCKER_HUB_USER + # - DOCKER_HUB_PASSWORD + - run: + name: Login to DockerHub + command: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin + # Tag docker images with the same pattern used in Git (Semantic Versioning) + # + # Git tag: v1.0.1 + # Docker tag: 1.0.1(-ci) + - run: + name: Tag images + command: | + docker images fundocker/jitsi-magnify + DOCKER_TAG=$([[ -z "$CIRCLE_TAG" ]] && echo $CIRCLE_BRANCH || echo ${CIRCLE_TAG} | sed 's/^v//') + RELEASE_TYPE=$([[ -z "$CIRCLE_TAG" ]] && echo "branch" || echo "tag ") + # Display either: + # - DOCKER_TAG: master (Git branch) + # or + # - DOCKER_TAG: 1.1.2 (Git tag v1.1.2) + echo "DOCKER_TAG: ${DOCKER_TAG} (Git ${RELEASE_TYPE}${CIRCLE_TAG})" + docker tag magnify:${CIRCLE_SHA1} fundocker/jitsi-magnify:${DOCKER_TAG} + if [[ -n "$CIRCLE_TAG" ]]; then + docker tag magnify:${CIRCLE_SHA1} fundocker/jitsi-magnify:latest + fi + docker images | grep -E "^fundocker/jitsi-magnify\s*(${DOCKER_TAG}.*|latest|master)" + + # Publish images to DockerHub + # + # Nota bene: logged user (see "Login to DockerHub" step) must have write + # permission for the project's repository; this also implies that the + # DockerHub repository already exists. + - run: + name: Publish images + command: | + DOCKER_TAG=$([[ -z "$CIRCLE_TAG" ]] && echo $CIRCLE_BRANCH || echo ${CIRCLE_TAG} | sed 's/^v//') + RELEASE_TYPE=$([[ -z "$CIRCLE_TAG" ]] && echo "branch" || echo "tag ") + # Display either: + # - DOCKER_TAG: master (Git branch) + # or + # - DOCKER_TAG: 1.1.2 (Git tag v1.1.2) + echo "DOCKER_TAG: ${DOCKER_TAG} (Git ${RELEASE_TYPE}${CIRCLE_TAG})" + docker push fundocker/jitsi-magnify:${DOCKER_TAG} + if [[ -n "$CIRCLE_TAG" ]]; then + docker push fundocker/jitsi-magnify:latest + fi + +workflows: + version: 2 + + magnify: + jobs: + # Front-end jobs + # + # Build, lint and test the front-end apps + - build-front: + filters: + tags: + only: /.*/ + - lint-front: + requires: + - build-front + filters: + tags: + only: /.*/ + - test-front: + requires: + - lint-front + filters: + tags: + only: /.*/ + + # Git jobs + # + # Check validity of git history + - lint-git: + filters: + tags: + only: /.*/ + # Check CHANGELOG update + - check-changelog: + filters: + branches: + ignore: main + tags: + only: /(?!^v).*/ + - lint-changelog: + filters: + branches: + ignore: main + tags: + only: /.*/ + # Check Renovate configuration + - check-renovate-configuration: + filters: + tags: + only: /.*/ + # Check on each PR if the last magnify version is present everywhere it should be. + # If not the build will fail before publishing a new release. + - check-versions: + filters: + tags: + only: /.*/ + + # Docker jobs + # + # Build images + - build-docker: + filters: + tags: + only: /.*/ + + # Backend jobs + # + # Build, lint and test production and development Docker images + # (debian-based) + - build-back: + filters: + tags: + only: /.*/ + - lint-back: + requires: + - build-back + filters: + tags: + only: /.*/ + - test-back-mysql-8: + requires: + - build-back + filters: + tags: + only: /.*/ + - test-back-postgresql: + requires: + - build-back + filters: + tags: + only: /.*/ + + # i18n jobs + # + # Extract strings and upload them to our translation management SaaS + - build-back-i18n: + requires: + - build-back + filters: + tags: + only: /.*/ + - upload-i18n-strings: + requires: + - build-front + - build-back-i18n + filters: + branches: + only: main + + # Packaging: python + # + # Build the python package + - package-back: + requires: + - test-front + - test-back-mysql-8 + - test-back-postgresql + - build-front + filters: + tags: + only: /.*/ + + # PyPI publication. + # + # Publish python package to PYPI only if all build, lint and test jobs + # succeed and it has been tagged with a tag starting with the letter v + - pypi: + requires: + - check-versions + - package-back + filters: + branches: + ignore: /.*/ + tags: + only: /^v.*/ + + # NPM publication. + # + # Publish frontend package to NPM only if all build, lint and test jobs + # succeed, and it has been tagged with a tag starting with the letter v + - npm: + requires: + - check-versions + - build-front + - lint-front + - test-front + filters: + branches: + ignore: /.*/ + tags: + only: /^v.*/ + + # DockerHub publication. + # + # Publish docker images only if all build, lint and test jobs succeed + # and it has been tagged with a tag starting with the letter v or is on + # the main branch + - hub: + requires: + - build-docker + - test-back-mysql-8 + - test-back-postgresql + filters: + branches: + only: main + tags: + only: /^v.*/ From 01a7c81e108fc92da80012c31f64ed887a18bb92 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Thu, 11 Apr 2024 09:56:02 +0200 Subject: [PATCH 05/44] fix 4 job --- .circleci/config.yml | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4b1d1b20b..081af48a8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -657,17 +657,17 @@ workflows: # Git jobs # # Check validity of git history - - lint-git: - filters: - tags: - only: /.*/ +# - lint-git: +# filters: +# tags: +# only: /.*/ # Check CHANGELOG update - - check-changelog: - filters: - branches: - ignore: main - tags: - only: /(?!^v).*/ +# - check-changelog: +# filters: +# branches: +# ignore: main +# tags: +# only: /(?!^v).*/ - lint-changelog: filters: branches: @@ -708,18 +708,18 @@ workflows: filters: tags: only: /.*/ - - test-back-mysql-8: - requires: - - build-back - filters: - tags: - only: /.*/ - - test-back-postgresql: - requires: - - build-back - filters: - tags: - only: /.*/ +# - test-back-mysql-8: +# requires: +# - build-back +# filters: +# tags: +# only: /.*/ +# - test-back-postgresql: +# requires: +# - build-back +# filters: +# tags: +# only: /.*/ # i18n jobs # From 07f41de5c7ee809176356c1cd6841297e707ab8a Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Thu, 11 Apr 2024 09:58:09 +0200 Subject: [PATCH 06/44] fix 4 job --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 081af48a8..59d5d94d4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -744,8 +744,8 @@ workflows: - package-back: requires: - test-front - - test-back-mysql-8 - - test-back-postgresql + #- test-back-mysql-8 + #- test-back-postgresql - build-front filters: tags: @@ -789,8 +789,8 @@ workflows: - hub: requires: - build-docker - - test-back-mysql-8 - - test-back-postgresql + #- test-back-mysql-8 + #- test-back-postgresql filters: branches: only: main From 8abec03fbcad3ddbb8d300d0550c3874e05a78d3 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 13:55:19 +0200 Subject: [PATCH 07/44] build and push backend --- .github/workflows/docker-hub.yml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml index 425bb8bfa..8fed7dc40 100644 --- a/.github/workflows/docker-hub.yml +++ b/.github/workflows/docker-hub.yml @@ -18,36 +18,32 @@ jobs: build-and-push-backend: runs-on: ubuntu-latest steps: - - - name: Checkout + - name: Checkout uses: actions/checkout@v4 - - - name: Docker meta + - name: Docker meta id: meta uses: docker/metadata-action@v5 with: images: lasuite/magnify-backend - - - name: Load sops secrets + - name: Load sops secrets uses: rouja/actions-sops@main with: secret-file: .github/workflows/secrets.enc.env age-key: ${{ secrets.SOPS_PRIVATE }} - - - name: Login to DockerHub + - name: Login to DockerHub if: github.event_name != 'pull_request' run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin - - - name: Build and push + - name: Build and push uses: docker/build-push-action@v5 with: context: . target: backend-production - build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 + build-args: DOCKER_USER=${{ env.DOCKER_USER }} push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + build-and-push-frontend: runs-on: ubuntu-latest steps: From 5bc4934850560e92131d6f99c609c9ab4e69a9df Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 13:56:00 +0200 Subject: [PATCH 08/44] build and push backend --- .github/workflows/docker-hub.yml | 106 +++++++++++++++---------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml index 8fed7dc40..61e74722a 100644 --- a/.github/workflows/docker-hub.yml +++ b/.github/workflows/docker-hub.yml @@ -44,57 +44,57 @@ jobs: labels: ${{ steps.meta.outputs.labels }} - build-and-push-frontend: - runs-on: ubuntu-latest - steps: - - - name: Checkout - uses: actions/checkout@v4 - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: lasuite/magnify-frontend - - - name: Load sops secrets - uses: rouja/actions-sops@main - with: - secret-file: .github/workflows/secrets.enc.env - age-key: ${{ secrets.SOPS_PRIVATE }} - - - name: Login to DockerHub - if: github.event_name != 'pull_request' - run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin - - - name: Build and push - uses: docker/build-push-action@v5 - with: - context: . - target: frontend-production - build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} +# build-and-push-frontend: +# runs-on: ubuntu-latest +# steps: +# - +# name: Checkout +# uses: actions/checkout@v4 +# - +# name: Docker meta +# id: meta +# uses: docker/metadata-action@v5 +# with: +# images: lasuite/magnify-frontend +# - +# name: Load sops secrets +# uses: rouja/actions-sops@main +# with: +# secret-file: .github/workflows/secrets.enc.env +# age-key: ${{ secrets.SOPS_PRIVATE }} +# - +# name: Login to DockerHub +# if: github.event_name != 'pull_request' +# run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin +# - +# name: Build and push +# uses: docker/build-push-action@v5 +# with: +# context: . +# target: frontend-production +# build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 +# push: ${{ github.event_name != 'pull_request' }} +# tags: ${{ steps.meta.outputs.tags }} +# labels: ${{ steps.meta.outputs.labels }} - notify-argocd: - needs: - - build-and-push-frontend - - build-and-push-backend - runs-on: ubuntu-latest - steps: - - - name: Checkout - uses: actions/checkout@v4 - - - name: Load sops secrets - uses: rouja/actions-sops@main - with: - secret-file: .github/workflows/secrets.enc.env - age-key: ${{ secrets.SOPS_PRIVATE }} - - - name: Call argocd github webhook - run: | - data='{"ref": "'$GITHUB_REF'","repository": {"html_url":"'$GITHUB_SERVER_URL'/'$GITHUB_REPOSITORY'"}}' - sig=$(echo -n ${data} | openssl dgst -sha1 -hmac ''${ARGOCD_WEBHOOK_SECRET}'' | awk '{print "X-Hub-Signature: sha1="$2}') - curl -X POST -H 'X-GitHub-Event:push' -H "Content-Type: application/json" -H "${sig}" --data "${data}" $ARGOCD_WEBHOOK_URL +# notify-argocd: +# needs: +# - build-and-push-frontend +# - build-and-push-backend +# runs-on: ubuntu-latest +# steps: +# - +# name: Checkout +# uses: actions/checkout@v4 +# - +# name: Load sops secrets +# uses: rouja/actions-sops@main +# with: +# secret-file: .github/workflows/secrets.enc.env +# age-key: ${{ secrets.SOPS_PRIVATE }} +# - +# name: Call argocd github webhook +# run: | +# data='{"ref": "'$GITHUB_REF'","repository": {"html_url":"'$GITHUB_SERVER_URL'/'$GITHUB_REPOSITORY'"}}' +# sig=$(echo -n ${data} | openssl dgst -sha1 -hmac ''${ARGOCD_WEBHOOK_SECRET}'' | awk '{print "X-Hub-Signature: sha1="$2}') +# curl -X POST -H 'X-GitHub-Event:push' -H "Content-Type: application/json" -H "${sig}" --data "${data}" $ARGOCD_WEBHOOK_URL From 6b40e6b8a5c06d8bf0e619f3f1739c7c553ff95b Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 14:00:24 +0200 Subject: [PATCH 09/44] re faire --- .github/workflows/docker-hub.yml | 124 +++++++++++++------------ .github/workflows/magnify-frontend.yml | 8 +- 2 files changed, 68 insertions(+), 64 deletions(-) diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml index 61e74722a..425bb8bfa 100644 --- a/.github/workflows/docker-hub.yml +++ b/.github/workflows/docker-hub.yml @@ -18,83 +18,87 @@ jobs: build-and-push-backend: runs-on: ubuntu-latest steps: - - name: Checkout + - + name: Checkout uses: actions/checkout@v4 - - name: Docker meta + - + name: Docker meta id: meta uses: docker/metadata-action@v5 with: images: lasuite/magnify-backend - - name: Load sops secrets + - + name: Load sops secrets uses: rouja/actions-sops@main with: secret-file: .github/workflows/secrets.enc.env age-key: ${{ secrets.SOPS_PRIVATE }} - - name: Login to DockerHub + - + name: Login to DockerHub if: github.event_name != 'pull_request' run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin - - name: Build and push + - + name: Build and push uses: docker/build-push-action@v5 with: context: . target: backend-production - build-args: DOCKER_USER=${{ env.DOCKER_USER }} + build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + build-and-push-frontend: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v4 + - + name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: lasuite/magnify-frontend + - + name: Load sops secrets + uses: rouja/actions-sops@main + with: + secret-file: .github/workflows/secrets.enc.env + age-key: ${{ secrets.SOPS_PRIVATE }} + - + name: Login to DockerHub + if: github.event_name != 'pull_request' + run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin + - + name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + target: frontend-production + build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} -# build-and-push-frontend: -# runs-on: ubuntu-latest -# steps: -# - -# name: Checkout -# uses: actions/checkout@v4 -# - -# name: Docker meta -# id: meta -# uses: docker/metadata-action@v5 -# with: -# images: lasuite/magnify-frontend -# - -# name: Load sops secrets -# uses: rouja/actions-sops@main -# with: -# secret-file: .github/workflows/secrets.enc.env -# age-key: ${{ secrets.SOPS_PRIVATE }} -# - -# name: Login to DockerHub -# if: github.event_name != 'pull_request' -# run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin -# - -# name: Build and push -# uses: docker/build-push-action@v5 -# with: -# context: . -# target: frontend-production -# build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 -# push: ${{ github.event_name != 'pull_request' }} -# tags: ${{ steps.meta.outputs.tags }} -# labels: ${{ steps.meta.outputs.labels }} - -# notify-argocd: -# needs: -# - build-and-push-frontend -# - build-and-push-backend -# runs-on: ubuntu-latest -# steps: -# - -# name: Checkout -# uses: actions/checkout@v4 -# - -# name: Load sops secrets -# uses: rouja/actions-sops@main -# with: -# secret-file: .github/workflows/secrets.enc.env -# age-key: ${{ secrets.SOPS_PRIVATE }} -# - -# name: Call argocd github webhook -# run: | -# data='{"ref": "'$GITHUB_REF'","repository": {"html_url":"'$GITHUB_SERVER_URL'/'$GITHUB_REPOSITORY'"}}' -# sig=$(echo -n ${data} | openssl dgst -sha1 -hmac ''${ARGOCD_WEBHOOK_SECRET}'' | awk '{print "X-Hub-Signature: sha1="$2}') -# curl -X POST -H 'X-GitHub-Event:push' -H "Content-Type: application/json" -H "${sig}" --data "${data}" $ARGOCD_WEBHOOK_URL + notify-argocd: + needs: + - build-and-push-frontend + - build-and-push-backend + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v4 + - + name: Load sops secrets + uses: rouja/actions-sops@main + with: + secret-file: .github/workflows/secrets.enc.env + age-key: ${{ secrets.SOPS_PRIVATE }} + - + name: Call argocd github webhook + run: | + data='{"ref": "'$GITHUB_REF'","repository": {"html_url":"'$GITHUB_SERVER_URL'/'$GITHUB_REPOSITORY'"}}' + sig=$(echo -n ${data} | openssl dgst -sha1 -hmac ''${ARGOCD_WEBHOOK_SECRET}'' | awk '{print "X-Hub-Signature: sha1="$2}') + curl -X POST -H 'X-GitHub-Event:push' -H "Content-Type: application/json" -H "${sig}" --data "${data}" $ARGOCD_WEBHOOK_URL diff --git a/.github/workflows/magnify-frontend.yml b/.github/workflows/magnify-frontend.yml index ac408e353..5e0660fb3 100644 --- a/.github/workflows/magnify-frontend.yml +++ b/.github/workflows/magnify-frontend.yml @@ -94,7 +94,7 @@ jobs: with: path: 'src/frontend/**/node_modules' key: front-node_modules-${{ hashFiles('src/frontend/**/yarn.lock') }} - + - name: Check linting run: cd src/frontend/ && yarn lint @@ -105,7 +105,7 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 - + - name: Set services env variables run: | make create-env-files @@ -124,7 +124,7 @@ jobs: with: path: src/frontend/apps/magnify/out/ key: build-front-${{ github.run_id }} - + - name: Build and Start Docker Servers env: DOCKER_BUILDKIT: 1 @@ -132,7 +132,7 @@ jobs: run: | docker-compose build --pull --build-arg BUILDKIT_INLINE_CACHE=1 make run - + - name: Apply DRF migrations run: | make migrate From 9db6c21383eace9a32b596f301fc4ec0d0032ac9 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 14:06:34 +0200 Subject: [PATCH 10/44] re faire front --- .github/workflows/magnify-frontend.yml | 148 ++++++++++++------------- 1 file changed, 74 insertions(+), 74 deletions(-) diff --git a/.github/workflows/magnify-frontend.yml b/.github/workflows/magnify-frontend.yml index 5e0660fb3..f47d52673 100644 --- a/.github/workflows/magnify-frontend.yml +++ b/.github/workflows/magnify-frontend.yml @@ -58,28 +58,28 @@ jobs: - name: Build CI App run: cd src/frontend/ && yarn ci:build - - name: Cache build frontend - uses: actions/cache@v4 - with: - path: src/frontend/apps/magnify/out/ - key: build-front-${{ github.run_id }} - - test-front: - runs-on: ubuntu-latest - needs: install-front - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Restore the frontend cache - uses: actions/cache@v4 - id: front-node_modules - with: - path: 'src/frontend/**/node_modules' - key: front-node_modules-${{ hashFiles('src/frontend/**/yarn.lock') }} - - - name: Test App - run: cd src/frontend/ && yarn app:test +# - name: Cache build frontend +# uses: actions/cache@v4 +# with: +# path: src/frontend/apps/magnify/out/ +# key: build-front-${{ github.run_id }} + +# test-front: +# runs-on: ubuntu-latest +# needs: install-front +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# +# - name: Restore the frontend cache +# uses: actions/cache@v4 +# id: front-node_modules +# with: +# path: 'src/frontend/**/node_modules' +# key: front-node_modules-${{ hashFiles('src/frontend/**/yarn.lock') }} +# +# - name: Test App +# run: cd src/frontend/ && yarn app:test lint-front: runs-on: ubuntu-latest @@ -98,55 +98,55 @@ jobs: - name: Check linting run: cd src/frontend/ && yarn lint - test-e2e: - runs-on: ubuntu-latest - needs: build-front - timeout-minutes: 10 - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set services env variables - run: | - make create-env-files - cat env.d/development/common.e2e.dist >> env.d/development/common - - - name: Restore the frontend cache - uses: actions/cache@v4 - id: front-node_modules - with: - path: 'src/frontend/**/node_modules' - key: front-node_modules-${{ hashFiles('src/frontend/**/yarn.lock') }} - - - name: Restore the build cache - uses: actions/cache@v4 - id: cache-build - with: - path: src/frontend/apps/magnify/out/ - key: build-front-${{ github.run_id }} - - - name: Build and Start Docker Servers - env: - DOCKER_BUILDKIT: 1 - COMPOSE_DOCKER_CLI_BUILD: 1 - run: | - docker-compose build --pull --build-arg BUILDKIT_INLINE_CACHE=1 - make run - - - name: Apply DRF migrations - run: | - make migrate - - - name: Install Playwright Browsers - run: cd src/frontend/apps/e2e && yarn install - - - name: Run e2e tests - run: cd src/frontend/ && yarn e2e:test - - - uses: actions/upload-artifact@v3 - if: always() - with: - name: playwright-report - path: src/frontend/apps/e2e/report/ - retention-days: 7 - +# test-e2e: +# runs-on: ubuntu-latest +# needs: build-front +# timeout-minutes: 10 +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# +# - name: Set services env variables +# run: | +# make create-env-files +# cat env.d/development/common.e2e.dist >> env.d/development/common +# +# - name: Restore the frontend cache +# uses: actions/cache@v4 +# id: front-node_modules +# with: +# path: 'src/frontend/**/node_modules' +# key: front-node_modules-${{ hashFiles('src/frontend/**/yarn.lock') }} +# +# - name: Restore the build cache +# uses: actions/cache@v4 +# id: cache-build +# with: +# path: src/frontend/apps/magnify/out/ +# key: build-front-${{ github.run_id }} +# +# - name: Build and Start Docker Servers +# env: +# DOCKER_BUILDKIT: 1 +# COMPOSE_DOCKER_CLI_BUILD: 1 +# run: | +# docker-compose build --pull --build-arg BUILDKIT_INLINE_CACHE=1 +# make run +# +# - name: Apply DRF migrations +# run: | +# make migrate +# +# - name: Install Playwright Browsers +# run: cd src/frontend/apps/e2e && yarn install +# +# - name: Run e2e tests +# run: cd src/frontend/ && yarn e2e:test +# +# - uses: actions/upload-artifact@v3 +# if: always() +# with: +# name: playwright-report +# path: src/frontend/apps/e2e/report/ +# retention-days: 7 +# From 885337f8e7b84c30ad969d58e799874da65b7de0 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 14:11:28 +0200 Subject: [PATCH 11/44] re faire front --- .github/workflows/magnify-frontend.yml | 39 +++++++++++++++----------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/.github/workflows/magnify-frontend.yml b/.github/workflows/magnify-frontend.yml index f47d52673..5fb356dd0 100644 --- a/.github/workflows/magnify-frontend.yml +++ b/.github/workflows/magnify-frontend.yml @@ -41,22 +41,29 @@ jobs: path: 'src/frontend/**/node_modules' key: front-node_modules-${{ hashFiles('src/frontend/**/yarn.lock') }} - build-front: - runs-on: ubuntu-latest - needs: install-front - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Restore the frontend cache - uses: actions/cache@v4 - id: front-node_modules - with: - path: 'src/frontend/**/node_modules' - key: front-node_modules-${{ hashFiles('src/frontend/**/yarn.lock') }} - - - name: Build CI App - run: cd src/frontend/ && yarn ci:build +build-front: + runs-on: ubuntu-latest + needs: install-front + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Restore the frontend cache + uses: actions/cache@v2 + with: + path: src/frontend/node_modules + key: front-node_modules-${{ hashFiles('src/frontend/yarn.lock') }} + + - name: Set up Node.js + uses: actions/setup-node@v2 + with: + node-version: 14 + + - name: Install dependencies + run: cd src/frontend/ && yarn install + + - name: Build CI App + run: cd src/frontend/ && yarn build # - name: Cache build frontend # uses: actions/cache@v4 From 5db8d2f69c0ebccf9c51c640e5bb0a5fa41b8bc2 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 14:13:00 +0200 Subject: [PATCH 12/44] re faire front --- .github/workflows/magnify-frontend.yml | 39 +++++++++++--------------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/.github/workflows/magnify-frontend.yml b/.github/workflows/magnify-frontend.yml index 5fb356dd0..d8314647b 100644 --- a/.github/workflows/magnify-frontend.yml +++ b/.github/workflows/magnify-frontend.yml @@ -41,29 +41,22 @@ jobs: path: 'src/frontend/**/node_modules' key: front-node_modules-${{ hashFiles('src/frontend/**/yarn.lock') }} -build-front: - runs-on: ubuntu-latest - needs: install-front - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Restore the frontend cache - uses: actions/cache@v2 - with: - path: src/frontend/node_modules - key: front-node_modules-${{ hashFiles('src/frontend/yarn.lock') }} - - - name: Set up Node.js - uses: actions/setup-node@v2 - with: - node-version: 14 - - - name: Install dependencies - run: cd src/frontend/ && yarn install - - - name: Build CI App - run: cd src/frontend/ && yarn build + build-front: + runs-on: ubuntu-latest + needs: install-front + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Restore the frontend cache + uses: actions/cache@v4 + id: front-node_modules + with: + path: 'src/frontend/**/node_modules' + key: front-node_modules-${{ hashFiles('src/frontend/**/yarn.lock') }} + + - name: Build CI App + run: cd src/frontend/ && yarn build # - name: Cache build frontend # uses: actions/cache@v4 From d46690f683c7db64e67cc21a75e2edd4b15a6a94 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 14:15:35 +0200 Subject: [PATCH 13/44] re faire front test e2e --- .github/workflows/magnify-frontend.yml | 104 ++++++++++++------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/.github/workflows/magnify-frontend.yml b/.github/workflows/magnify-frontend.yml index d8314647b..81bc54b1c 100644 --- a/.github/workflows/magnify-frontend.yml +++ b/.github/workflows/magnify-frontend.yml @@ -98,55 +98,55 @@ jobs: - name: Check linting run: cd src/frontend/ && yarn lint -# test-e2e: -# runs-on: ubuntu-latest -# needs: build-front -# timeout-minutes: 10 -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# -# - name: Set services env variables -# run: | -# make create-env-files -# cat env.d/development/common.e2e.dist >> env.d/development/common -# -# - name: Restore the frontend cache -# uses: actions/cache@v4 -# id: front-node_modules -# with: -# path: 'src/frontend/**/node_modules' -# key: front-node_modules-${{ hashFiles('src/frontend/**/yarn.lock') }} -# -# - name: Restore the build cache -# uses: actions/cache@v4 -# id: cache-build -# with: -# path: src/frontend/apps/magnify/out/ -# key: build-front-${{ github.run_id }} -# -# - name: Build and Start Docker Servers -# env: -# DOCKER_BUILDKIT: 1 -# COMPOSE_DOCKER_CLI_BUILD: 1 -# run: | -# docker-compose build --pull --build-arg BUILDKIT_INLINE_CACHE=1 -# make run -# -# - name: Apply DRF migrations -# run: | -# make migrate -# -# - name: Install Playwright Browsers -# run: cd src/frontend/apps/e2e && yarn install -# -# - name: Run e2e tests -# run: cd src/frontend/ && yarn e2e:test -# -# - uses: actions/upload-artifact@v3 -# if: always() -# with: -# name: playwright-report -# path: src/frontend/apps/e2e/report/ -# retention-days: 7 -# + test-e2e: + runs-on: ubuntu-latest + needs: build-front + timeout-minutes: 10 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set services env variables + run: | + make create-env-files + cat env.d/development/common.e2e.dist >> env.d/development/common + + - name: Restore the frontend cache + uses: actions/cache@v4 + id: front-node_modules + with: + path: 'src/frontend/**/node_modules' + key: front-node_modules-${{ hashFiles('src/frontend/**/yarn.lock') }} + + - name: Restore the build cache + uses: actions/cache@v4 + id: cache-build + with: + path: src/frontend/apps/magnify/out/ + key: build-front-${{ github.run_id }} + + - name: Build and Start Docker Servers + env: + DOCKER_BUILDKIT: 1 + COMPOSE_DOCKER_CLI_BUILD: 1 + run: | + docker-compose build --pull --build-arg BUILDKIT_INLINE_CACHE=1 + make run + + - name: Apply DRF migrations + run: | + make migrate + + - name: Install Playwright Browsers + run: cd src/frontend/apps/e2e && yarn install + + - name: Run e2e tests + run: cd src/frontend/ && yarn e2e:test + + - uses: actions/upload-artifact@v3 + if: always() + with: + name: playwright-report + path: src/frontend/apps/e2e/report/ + retention-days: 7 + From c3bfc38ff711429ad9f2cbe4e147fe7096701ddf Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 14:19:26 +0200 Subject: [PATCH 14/44] re faire front test --- .github/workflows/magnify-frontend.yml | 64 +++----------------------- 1 file changed, 7 insertions(+), 57 deletions(-) diff --git a/.github/workflows/magnify-frontend.yml b/.github/workflows/magnify-frontend.yml index 81bc54b1c..3fafef286 100644 --- a/.github/workflows/magnify-frontend.yml +++ b/.github/workflows/magnify-frontend.yml @@ -64,24 +64,7 @@ jobs: # path: src/frontend/apps/magnify/out/ # key: build-front-${{ github.run_id }} -# test-front: -# runs-on: ubuntu-latest -# needs: install-front -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# -# - name: Restore the frontend cache -# uses: actions/cache@v4 -# id: front-node_modules -# with: -# path: 'src/frontend/**/node_modules' -# key: front-node_modules-${{ hashFiles('src/frontend/**/yarn.lock') }} -# -# - name: Test App -# run: cd src/frontend/ && yarn app:test - - lint-front: + test-front: runs-on: ubuntu-latest needs: install-front steps: @@ -95,22 +78,16 @@ jobs: path: 'src/frontend/**/node_modules' key: front-node_modules-${{ hashFiles('src/frontend/**/yarn.lock') }} - - name: Check linting - run: cd src/frontend/ && yarn lint + - name: Test App + run: cd src/frontend/ && yarn test - test-e2e: + lint-front: runs-on: ubuntu-latest - needs: build-front - timeout-minutes: 10 + needs: install-front steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Set services env variables - run: | - make create-env-files - cat env.d/development/common.e2e.dist >> env.d/development/common - - name: Restore the frontend cache uses: actions/cache@v4 id: front-node_modules @@ -118,35 +95,8 @@ jobs: path: 'src/frontend/**/node_modules' key: front-node_modules-${{ hashFiles('src/frontend/**/yarn.lock') }} - - name: Restore the build cache - uses: actions/cache@v4 - id: cache-build - with: - path: src/frontend/apps/magnify/out/ - key: build-front-${{ github.run_id }} - - - name: Build and Start Docker Servers - env: - DOCKER_BUILDKIT: 1 - COMPOSE_DOCKER_CLI_BUILD: 1 - run: | - docker-compose build --pull --build-arg BUILDKIT_INLINE_CACHE=1 - make run - - - name: Apply DRF migrations - run: | - make migrate - - - name: Install Playwright Browsers - run: cd src/frontend/apps/e2e && yarn install + - name: Check linting + run: cd src/frontend/ && yarn lint - - name: Run e2e tests - run: cd src/frontend/ && yarn e2e:test - - uses: actions/upload-artifact@v3 - if: always() - with: - name: playwright-report - path: src/frontend/apps/e2e/report/ - retention-days: 7 From e8856379973d1e220eb6ffed89eb85f5905333c0 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 14:22:04 +0200 Subject: [PATCH 15/44] re faire front cache front --- .github/workflows/magnify-frontend.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/magnify-frontend.yml b/.github/workflows/magnify-frontend.yml index 3fafef286..1a900cd75 100644 --- a/.github/workflows/magnify-frontend.yml +++ b/.github/workflows/magnify-frontend.yml @@ -58,11 +58,11 @@ jobs: - name: Build CI App run: cd src/frontend/ && yarn build -# - name: Cache build frontend -# uses: actions/cache@v4 -# with: -# path: src/frontend/apps/magnify/out/ -# key: build-front-${{ github.run_id }} + - name: Cache build frontend + uses: actions/cache@v4 + with: + path: src/frontend/apps/magnify/out/ + key: build-front-${{ github.run_id }} test-front: runs-on: ubuntu-latest From d09794a6261859e5fd752a2a6632f4f1c1f8a711 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 14:29:50 +0200 Subject: [PATCH 16/44] re faire magnify yml --- .github/workflows/docker-hub.yml | 130 ++++++------ .github/workflows/magnify.yml | 331 ++++++++++++++++--------------- 2 files changed, 231 insertions(+), 230 deletions(-) diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml index 425bb8bfa..f9494f8ac 100644 --- a/.github/workflows/docker-hub.yml +++ b/.github/workflows/docker-hub.yml @@ -15,71 +15,71 @@ env: DOCKER_USER: 1001:127 jobs: - build-and-push-backend: - runs-on: ubuntu-latest - steps: - - - name: Checkout - uses: actions/checkout@v4 - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: lasuite/magnify-backend - - - name: Load sops secrets - uses: rouja/actions-sops@main - with: - secret-file: .github/workflows/secrets.enc.env - age-key: ${{ secrets.SOPS_PRIVATE }} - - - name: Login to DockerHub - if: github.event_name != 'pull_request' - run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin - - - name: Build and push - uses: docker/build-push-action@v5 - with: - context: . - target: backend-production - build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - build-and-push-frontend: - runs-on: ubuntu-latest - steps: - - - name: Checkout - uses: actions/checkout@v4 - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: lasuite/magnify-frontend - - - name: Load sops secrets - uses: rouja/actions-sops@main - with: - secret-file: .github/workflows/secrets.enc.env - age-key: ${{ secrets.SOPS_PRIVATE }} - - - name: Login to DockerHub - if: github.event_name != 'pull_request' - run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin - - - name: Build and push - uses: docker/build-push-action@v5 - with: - context: . - target: frontend-production - build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} +# build-and-push-backend: +# runs-on: ubuntu-latest +# steps: +# - +# name: Checkout +# uses: actions/checkout@v4 +# - +# name: Docker meta +# id: meta +# uses: docker/metadata-action@v5 +# with: +# images: lasuite/magnify-backend +# - +# name: Load sops secrets +# uses: rouja/actions-sops@main +# with: +# secret-file: .github/workflows/secrets.enc.env +# age-key: ${{ secrets.SOPS_PRIVATE }} +# - +# name: Login to DockerHub +# if: github.event_name != 'pull_request' +# run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin +# - +# name: Build and push +# uses: docker/build-push-action@v5 +# with: +# context: . +# target: backend-production +# build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 +# push: ${{ github.event_name != 'pull_request' }} +# tags: ${{ steps.meta.outputs.tags }} +# labels: ${{ steps.meta.outputs.labels }} +# +# build-and-push-frontend: +# runs-on: ubuntu-latest +# steps: +# - +# name: Checkout +# uses: actions/checkout@v4 +# - +# name: Docker meta +# id: meta +# uses: docker/metadata-action@v5 +# with: +# images: lasuite/magnify-frontend +# - +# name: Load sops secrets +# uses: rouja/actions-sops@main +# with: +# secret-file: .github/workflows/secrets.enc.env +# age-key: ${{ secrets.SOPS_PRIVATE }} +# - +# name: Login to DockerHub +# if: github.event_name != 'pull_request' +# run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin +# - +# name: Build and push +# uses: docker/build-push-action@v5 +# with: +# context: . +# target: frontend-production +# build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 +# push: ${{ github.event_name != 'pull_request' }} +# tags: ${{ steps.meta.outputs.tags }} +# labels: ${{ steps.meta.outputs.labels }} notify-argocd: needs: diff --git a/.github/workflows/magnify.yml b/.github/workflows/magnify.yml index 7c6d53268..0827d1586 100644 --- a/.github/workflows/magnify.yml +++ b/.github/workflows/magnify.yml @@ -32,168 +32,169 @@ jobs: - name: Lint commit messages added to main run: ~/.local/bin/gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD - check-changelog: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Check that the CHANGELOG has been modified in the current branch - run: git whatchanged --name-only --pretty="" origin..HEAD | grep CHANGELOG - - lint-changelog: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Check CHANGELOG max line length - run: | - max_line_length=$(cat CHANGELOG.md | grep -Ev "^\[.*\]: https://github.com" | wc -L) - if [ $max_line_length -ge 80 ]; then - echo "ERROR: CHANGELOG has lines longer than 80 characters." - exit 1 - fi - - build-mails: - runs-on: ubuntu-latest - defaults: - run: - working-directory: src/mail - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Install Node.js - uses: actions/setup-node@v4 - with: - node-version: '18' - - name: Install yarn - run: npm install -g yarn - - name: Install node dependencies - run: yarn install --frozen-lockfile - - name: Build mails - run: yarn build - - lint-back: - runs-on: ubuntu-latest - defaults: - run: - working-directory: src/backend - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Install Python - uses: actions/setup-python@v3 - with: - python-version: '3.10' - - name: Install development dependencies - run: pip install --user .[dev] - - name: Check code formatting with ruff - run: ~/.local/bin/ruff format magnify --diff - - name: Lint code with ruff - run: ~/.local/bin/ruff check magnify - - name: Lint code with pylint - run: ~/.local/bin/pylint magnify - - test-back: - runs-on: ubuntu-latest - defaults: - run: - working-directory: src/backend - - services: - postgres: - image: postgres:16 - env: - POSTGRES_DB: magnify - POSTGRES_USER: dinum - POSTGRES_PASSWORD: pass - ports: - - 5432:5432 - # needed because the postgres container does not provide a healthcheck - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - - env: - DJANGO_CONFIGURATION: Test - DJANGO_SETTINGS_MODULE: magnify.settings - DJANGO_SECRET_KEY: ThisIsAnExampleKeyForTestPurposeOnly - OIDC_OP_JWKS_ENDPOINT: /endpoint-for-test-purpose-only - DB_HOST: localhost - DB_NAME: magnify - DB_USER: dinum - DB_PASSWORD: pass - DB_PORT: 5432 - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Create writable /data - run: | - sudo mkdir -p /data/media && \ - sudo mkdir -p /data/static - - name: Install Python - uses: actions/setup-python@v3 - with: - python-version: '3.10' - - name: Install development dependencies - run: pip install --user .[dev] - - name: Install gettext (required to compile messages) - run: | - sudo apt-get update - sudo apt-get install -y gettext - - name: Generate a MO file from strings extracted from the project - run: python manage.py compilemessages - - name: Run tests - run: ~/.local/bin/pytest -n 2 - - i18n-crowdin: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Install gettext (required to make messages) - run: | - sudo apt-get update - sudo apt-get install -y gettext - - - name: Install Python - uses: actions/setup-python@v3 - with: - python-version: '3.10' - - - name: Install development dependencies - working-directory: src/backend - run: pip install --user .[dev] - - - name: Generate the translation base file - run: ~/.local/bin/django-admin makemessages --keep-pot --all - - - name: Load sops secrets - uses: rouja/actions-sops@main - with: - secret-file: .github/workflows/secrets.enc.env - age-key: ${{ secrets.SOPS_PRIVATE }} - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '18.x' - cache: 'yarn' - cache-dependency-path: src/frontend/yarn.lock - - - name: Install dependencies - run: cd src/frontend/ && yarn install --frozen-lockfile - - - name: Extract the frontend translation - run: make frontend-i18n-extract - - - name: Upload files to Crowdin - run: | - docker run \ - --rm \ - -e CROWDIN_API_TOKEN=$CROWDIN_API_TOKEN \ - -e CROWDIN_PROJECT_ID=$CROWDIN_PROJECT_ID \ - -e CROWDIN_BASE_PATH=$CROWDIN_BASE_PATH \ - -v "${{ github.workspace }}:/app" \ - crowdin/cli:3.16.0 \ - crowdin upload sources -c /app/crowdin/config.yml - +# check-changelog: +# runs-on: ubuntu-latest +# steps: +# - name: Checkout repository +# uses: actions/checkout@v2 +# - name: Check that the CHANGELOG has been modified in the current branch +# run: git whatchanged --name-only --pretty="" origin..HEAD | grep CHANGELOG +# +# lint-changelog: +# runs-on: ubuntu-latest +# steps: +# - name: Checkout repository +# uses: actions/checkout@v2 +# - name: Check CHANGELOG max line length +# run: | +# max_line_length=$(cat CHANGELOG.md | grep -Ev "^\[.*\]: https://github.com" | wc -L) +# if [ $max_line_length -ge 80 ]; then +# echo "ERROR: CHANGELOG has lines longer than 80 characters." +# exit 1 +# fi +# +# +# build-mails: +# runs-on: ubuntu-latest +# defaults: +# run: +# working-directory: src/mail +# steps: +# - name: Checkout repository +# uses: actions/checkout@v2 +# - name: Install Node.js +# uses: actions/setup-node@v4 +# with: +# node-version: '18' +# - name: Install yarn +# run: npm install -g yarn +# - name: Install node dependencies +# run: yarn install --frozen-lockfile +# - name: Build mails +# run: yarn build +# +# lint-back: +# runs-on: ubuntu-latest +# defaults: +# run: +# working-directory: src/backend +# steps: +# - name: Checkout repository +# uses: actions/checkout@v2 +# - name: Install Python +# uses: actions/setup-python@v3 +# with: +# python-version: '3.10' +# - name: Install development dependencies +# run: pip install --user .[dev] +# - name: Check code formatting with ruff +# run: ~/.local/bin/ruff format magnify --diff +# - name: Lint code with ruff +# run: ~/.local/bin/ruff check magnify +# - name: Lint code with pylint +# run: ~/.local/bin/pylint magnify +# +# test-back: +# runs-on: ubuntu-latest +# defaults: +# run: +# working-directory: src/backend +# +# services: +# postgres: +# image: postgres:16 +# env: +# POSTGRES_DB: magnify +# POSTGRES_USER: dinum +# POSTGRES_PASSWORD: pass +# ports: +# - 5432:5432 +# # needed because the postgres container does not provide a healthcheck +# options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 +# +# env: +# DJANGO_CONFIGURATION: Test +# DJANGO_SETTINGS_MODULE: magnify.settings +# DJANGO_SECRET_KEY: ThisIsAnExampleKeyForTestPurposeOnly +# OIDC_OP_JWKS_ENDPOINT: /endpoint-for-test-purpose-only +# DB_HOST: localhost +# DB_NAME: magnify +# DB_USER: dinum +# DB_PASSWORD: pass +# DB_PORT: 5432 +# +# steps: +# - name: Checkout repository +# uses: actions/checkout@v2 +# - name: Create writable /data +# run: | +# sudo mkdir -p /data/media && \ +# sudo mkdir -p /data/static +# - name: Install Python +# uses: actions/setup-python@v3 +# with: +# python-version: '3.10' +# - name: Install development dependencies +# run: pip install --user .[dev] +# - name: Install gettext (required to compile messages) +# run: | +# sudo apt-get update +# sudo apt-get install -y gettext +# - name: Generate a MO file from strings extracted from the project +# run: python manage.py compilemessages +# - name: Run tests +# run: ~/.local/bin/pytest -n 2 +# +# i18n-crowdin: +# runs-on: ubuntu-latest +# steps: +# - name: Checkout repository +# uses: actions/checkout@v2 +# +# - name: Install gettext (required to make messages) +# run: | +# sudo apt-get update +# sudo apt-get install -y gettext +# +# - name: Install Python +# uses: actions/setup-python@v3 +# with: +# python-version: '3.10' +# +# - name: Install development dependencies +# working-directory: src/backend +# run: pip install --user .[dev] +# +# - name: Generate the translation base file +# run: ~/.local/bin/django-admin makemessages --keep-pot --all +# +# - name: Load sops secrets +# uses: rouja/actions-sops@main +# with: +# secret-file: .github/workflows/secrets.enc.env +# age-key: ${{ secrets.SOPS_PRIVATE }} +# +# - name: Setup Node.js +# uses: actions/setup-node@v4 +# with: +# node-version: '18.x' +# cache: 'yarn' +# cache-dependency-path: src/frontend/yarn.lock +# +# - name: Install dependencies +# run: cd src/frontend/ && yarn install --frozen-lockfile +# +# - name: Extract the frontend translation +# run: make frontend-i18n-extract +# +# - name: Upload files to Crowdin +# run: | +# docker run \ +# --rm \ +# -e CROWDIN_API_TOKEN=$CROWDIN_API_TOKEN \ +# -e CROWDIN_PROJECT_ID=$CROWDIN_PROJECT_ID \ +# -e CROWDIN_BASE_PATH=$CROWDIN_BASE_PATH \ +# -v "${{ github.workspace }}:/app" \ +# crowdin/cli:3.16.0 \ +# crowdin upload sources -c /app/crowdin/config.yml +# From e45743d391377085008e0dd9cec4dc319f684182 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 14:31:53 +0200 Subject: [PATCH 17/44] re faire magnify yml --- .github/workflows/magnify.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/magnify.yml b/.github/workflows/magnify.yml index 0827d1586..b3ca59d8c 100644 --- a/.github/workflows/magnify.yml +++ b/.github/workflows/magnify.yml @@ -13,23 +13,23 @@ on: jobs: lint-git: runs-on: ubuntu-latest - if: github.event_name == 'pull_request' # Makes sense only for pull requests + if: github.event_name == 'pull_request' # Ne s'applique que pour les pull requests steps: - - name: Checkout repository + - name: Checkout du dépôt uses: actions/checkout@v2 with: fetch-depth: 0 - - name: show + - name: Afficher les logs run: git log - - name: Enforce absence of print statements in code + - name: Assurer l'absence d'instructions print dans le code run: | ! git diff origin/${{ github.event.pull_request.base.ref }}..HEAD -- . ':(exclude)**/magnify.yml' | grep "print(" - - name: Check absence of fixup commits + - name: Vérifier l'absence de commits fixup run: | ! git log | grep 'fixup!' - - name: Install gitlint + - name: Installer gitlint run: pip install --user requests gitlint - - name: Lint commit messages added to main + - name: Vérifier les messages de commit ajoutés à la branche principale run: ~/.local/bin/gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD # check-changelog: From 71d5dca2763b5f6642bb40990dd14f72c8a24bb3 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 14:33:53 +0200 Subject: [PATCH 18/44] re faire magnify yml check-changelog --- .github/workflows/magnify.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/magnify.yml b/.github/workflows/magnify.yml index b3ca59d8c..907759ffa 100644 --- a/.github/workflows/magnify.yml +++ b/.github/workflows/magnify.yml @@ -32,14 +32,14 @@ jobs: - name: Vérifier les messages de commit ajoutés à la branche principale run: ~/.local/bin/gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD -# check-changelog: -# runs-on: ubuntu-latest -# steps: -# - name: Checkout repository -# uses: actions/checkout@v2 -# - name: Check that the CHANGELOG has been modified in the current branch -# run: git whatchanged --name-only --pretty="" origin..HEAD | grep CHANGELOG -# + check-changelog: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Check that the CHANGELOG has been modified in the current branch + run: git whatchanged --name-only --pretty="" origin..HEAD | grep CHANGELOG + # lint-changelog: # runs-on: ubuntu-latest # steps: From ae442f4dfa1ee69c14a3b705945794f3deff6d07 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 14:35:51 +0200 Subject: [PATCH 19/44] re faire magnify yml lint git --- .github/workflows/magnify.yml | 43 +++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/.github/workflows/magnify.yml b/.github/workflows/magnify.yml index 907759ffa..237172b9b 100644 --- a/.github/workflows/magnify.yml +++ b/.github/workflows/magnify.yml @@ -11,26 +11,29 @@ on: - '*' jobs: - lint-git: - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' # Ne s'applique que pour les pull requests - steps: - - name: Checkout du dépôt - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Afficher les logs - run: git log - - name: Assurer l'absence d'instructions print dans le code - run: | - ! git diff origin/${{ github.event.pull_request.base.ref }}..HEAD -- . ':(exclude)**/magnify.yml' | grep "print(" - - name: Vérifier l'absence de commits fixup - run: | - ! git log | grep 'fixup!' - - name: Installer gitlint - run: pip install --user requests gitlint - - name: Vérifier les messages de commit ajoutés à la branche principale - run: ~/.local/bin/gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD +lint-git: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' # Ne s'applique que pour les pull requests + steps: + - name: Checkout du dépôt + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Afficher les logs + run: git log + - name: Assurer l'absence d'instructions print dans le code + run: | + ! git diff origin/${{ github.event.pull_request.base.ref }}..HEAD -- . ':(exclude)**/magnify.yml' | grep "print(" + - name: Vérifier l'absence de commits fixup + run: | + ! git log | grep 'fixup!' + - name: Rebaser la branche sur la branche principale + run: git rebase origin/${{ github.event.pull_request.base.ref }} + - name: Installer gitlint + run: pip install --user requests gitlint + - name: Vérifier les messages de commit ajoutés à la branche principale + run: ~/.local/bin/gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD + check-changelog: runs-on: ubuntu-latest From 7f9f2f70d05cd4a060cb4b41ae10ae50195efb99 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 14:39:19 +0200 Subject: [PATCH 20/44] re faire magnify yml --- .github/workflows/magnify.yml | 43 ++++++++++++++++------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/.github/workflows/magnify.yml b/.github/workflows/magnify.yml index 237172b9b..907759ffa 100644 --- a/.github/workflows/magnify.yml +++ b/.github/workflows/magnify.yml @@ -11,29 +11,26 @@ on: - '*' jobs: -lint-git: - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' # Ne s'applique que pour les pull requests - steps: - - name: Checkout du dépôt - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Afficher les logs - run: git log - - name: Assurer l'absence d'instructions print dans le code - run: | - ! git diff origin/${{ github.event.pull_request.base.ref }}..HEAD -- . ':(exclude)**/magnify.yml' | grep "print(" - - name: Vérifier l'absence de commits fixup - run: | - ! git log | grep 'fixup!' - - name: Rebaser la branche sur la branche principale - run: git rebase origin/${{ github.event.pull_request.base.ref }} - - name: Installer gitlint - run: pip install --user requests gitlint - - name: Vérifier les messages de commit ajoutés à la branche principale - run: ~/.local/bin/gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD - + lint-git: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' # Ne s'applique que pour les pull requests + steps: + - name: Checkout du dépôt + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Afficher les logs + run: git log + - name: Assurer l'absence d'instructions print dans le code + run: | + ! git diff origin/${{ github.event.pull_request.base.ref }}..HEAD -- . ':(exclude)**/magnify.yml' | grep "print(" + - name: Vérifier l'absence de commits fixup + run: | + ! git log | grep 'fixup!' + - name: Installer gitlint + run: pip install --user requests gitlint + - name: Vérifier les messages de commit ajoutés à la branche principale + run: ~/.local/bin/gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD check-changelog: runs-on: ubuntu-latest From a406f4ec30dd473f1d3bec59384893156a694242 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 14:46:51 +0200 Subject: [PATCH 21/44] re faire magnify yml --- .github/workflows/magnify.yml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/magnify.yml b/.github/workflows/magnify.yml index 907759ffa..aee83bdce 100644 --- a/.github/workflows/magnify.yml +++ b/.github/workflows/magnify.yml @@ -29,16 +29,18 @@ jobs: ! git log | grep 'fixup!' - name: Installer gitlint run: pip install --user requests gitlint - - name: Vérifier les messages de commit ajoutés à la branche principale - run: ~/.local/bin/gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD +# - name: Vérifier les messages de commit ajoutés à la branche principale +# run: ~/.local/bin/gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD + - name: Run GitLint + run: gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD - check-changelog: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Check that the CHANGELOG has been modified in the current branch - run: git whatchanged --name-only --pretty="" origin..HEAD | grep CHANGELOG +check-changelog: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Check that the CHANGELOG has been modified in the current branch + run: git log --name-only --pretty="" origin..HEAD | grep CHANGELOG # lint-changelog: # runs-on: ubuntu-latest From de42da7237fb88b451efc8753b50604581b4b35b Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 14:52:26 +0200 Subject: [PATCH 22/44] re faire magnify yml check changelog --- .github/workflows/magnify.yml | 44 +++++++++++++++++------------------ 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/magnify.yml b/.github/workflows/magnify.yml index aee83bdce..9791721f1 100644 --- a/.github/workflows/magnify.yml +++ b/.github/workflows/magnify.yml @@ -11,28 +11,28 @@ on: - '*' jobs: - lint-git: - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' # Ne s'applique que pour les pull requests - steps: - - name: Checkout du dépôt - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Afficher les logs - run: git log - - name: Assurer l'absence d'instructions print dans le code - run: | - ! git diff origin/${{ github.event.pull_request.base.ref }}..HEAD -- . ':(exclude)**/magnify.yml' | grep "print(" - - name: Vérifier l'absence de commits fixup - run: | - ! git log | grep 'fixup!' - - name: Installer gitlint - run: pip install --user requests gitlint -# - name: Vérifier les messages de commit ajoutés à la branche principale -# run: ~/.local/bin/gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD - - name: Run GitLint - run: gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD +# lint-git: +# runs-on: ubuntu-latest +# if: github.event_name == 'pull_request' # Ne s'applique que pour les pull requests +# steps: +# - name: Checkout du dépôt +# uses: actions/checkout@v2 +# with: +# fetch-depth: 0 +# - name: Afficher les logs +# run: git log +# - name: Assurer l'absence d'instructions print dans le code +# run: | +# ! git diff origin/${{ github.event.pull_request.base.ref }}..HEAD -- . ':(exclude)**/magnify.yml' | grep "print(" +# - name: Vérifier l'absence de commits fixup +# run: | +# ! git log | grep 'fixup!' +# - name: Installer gitlint +# run: pip install --user requests gitlint +## - name: Vérifier les messages de commit ajoutés à la branche principale +## run: ~/.local/bin/gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD +# - name: Run GitLint +# run: gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD check-changelog: runs-on: ubuntu-latest From a509be465b5df6551d92f25564133f85893a0429 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 15:03:08 +0200 Subject: [PATCH 23/44] re faire magnify --- .circleci/config.yml | 52 ++--- .github/workflows/docker-hub.yml | 64 +++--- .github/workflows/magnify.yml | 360 +++++++++++++++---------------- 3 files changed, 238 insertions(+), 238 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 59d5d94d4..4b1d1b20b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -657,17 +657,17 @@ workflows: # Git jobs # # Check validity of git history -# - lint-git: -# filters: -# tags: -# only: /.*/ + - lint-git: + filters: + tags: + only: /.*/ # Check CHANGELOG update -# - check-changelog: -# filters: -# branches: -# ignore: main -# tags: -# only: /(?!^v).*/ + - check-changelog: + filters: + branches: + ignore: main + tags: + only: /(?!^v).*/ - lint-changelog: filters: branches: @@ -708,18 +708,18 @@ workflows: filters: tags: only: /.*/ -# - test-back-mysql-8: -# requires: -# - build-back -# filters: -# tags: -# only: /.*/ -# - test-back-postgresql: -# requires: -# - build-back -# filters: -# tags: -# only: /.*/ + - test-back-mysql-8: + requires: + - build-back + filters: + tags: + only: /.*/ + - test-back-postgresql: + requires: + - build-back + filters: + tags: + only: /.*/ # i18n jobs # @@ -744,8 +744,8 @@ workflows: - package-back: requires: - test-front - #- test-back-mysql-8 - #- test-back-postgresql + - test-back-mysql-8 + - test-back-postgresql - build-front filters: tags: @@ -789,8 +789,8 @@ workflows: - hub: requires: - build-docker - #- test-back-mysql-8 - #- test-back-postgresql + - test-back-mysql-8 + - test-back-postgresql filters: branches: only: main diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml index f9494f8ac..7bd638730 100644 --- a/.github/workflows/docker-hub.yml +++ b/.github/workflows/docker-hub.yml @@ -48,38 +48,38 @@ jobs: # tags: ${{ steps.meta.outputs.tags }} # labels: ${{ steps.meta.outputs.labels }} # -# build-and-push-frontend: -# runs-on: ubuntu-latest -# steps: -# - -# name: Checkout -# uses: actions/checkout@v4 -# - -# name: Docker meta -# id: meta -# uses: docker/metadata-action@v5 -# with: -# images: lasuite/magnify-frontend -# - -# name: Load sops secrets -# uses: rouja/actions-sops@main -# with: -# secret-file: .github/workflows/secrets.enc.env -# age-key: ${{ secrets.SOPS_PRIVATE }} -# - -# name: Login to DockerHub -# if: github.event_name != 'pull_request' -# run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin -# - -# name: Build and push -# uses: docker/build-push-action@v5 -# with: -# context: . -# target: frontend-production -# build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 -# push: ${{ github.event_name != 'pull_request' }} -# tags: ${{ steps.meta.outputs.tags }} -# labels: ${{ steps.meta.outputs.labels }} + build-and-push-frontend: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v4 + - + name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: lasuite/magnify-frontend + - + name: Load sops secrets + uses: rouja/actions-sops@main + with: + secret-file: .github/workflows/secrets.enc.env + age-key: ${{ secrets.SOPS_PRIVATE }} + - + name: Login to DockerHub + if: github.event_name != 'pull_request' + run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin + - + name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + target: frontend-production + build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} notify-argocd: needs: diff --git a/.github/workflows/magnify.yml b/.github/workflows/magnify.yml index 9791721f1..f434f2d88 100644 --- a/.github/workflows/magnify.yml +++ b/.github/workflows/magnify.yml @@ -11,28 +11,28 @@ on: - '*' jobs: -# lint-git: -# runs-on: ubuntu-latest -# if: github.event_name == 'pull_request' # Ne s'applique que pour les pull requests -# steps: -# - name: Checkout du dépôt -# uses: actions/checkout@v2 -# with: -# fetch-depth: 0 -# - name: Afficher les logs -# run: git log -# - name: Assurer l'absence d'instructions print dans le code -# run: | -# ! git diff origin/${{ github.event.pull_request.base.ref }}..HEAD -- . ':(exclude)**/magnify.yml' | grep "print(" -# - name: Vérifier l'absence de commits fixup -# run: | -# ! git log | grep 'fixup!' -# - name: Installer gitlint -# run: pip install --user requests gitlint -## - name: Vérifier les messages de commit ajoutés à la branche principale -## run: ~/.local/bin/gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD -# - name: Run GitLint -# run: gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD + lint-git: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' # Ne s'applique que pour les pull requests + steps: + - name: Checkout du dépôt + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Afficher les logs + run: git log + - name: Assurer l'absence d'instructions print dans le code + run: | + ! git diff origin/${{ github.event.pull_request.base.ref }}..HEAD -- . ':(exclude)**/magnify.yml' | grep "print(" + - name: Vérifier l'absence de commits fixup + run: | + ! git log | grep 'fixup!' + - name: Installer gitlint + run: pip install --user requests gitlint +# - name: Vérifier les messages de commit ajoutés à la branche principale +# run: ~/.local/bin/gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD + - name: Run GitLint + run: gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD check-changelog: runs-on: ubuntu-latest @@ -42,161 +42,161 @@ check-changelog: - name: Check that the CHANGELOG has been modified in the current branch run: git log --name-only --pretty="" origin..HEAD | grep CHANGELOG -# lint-changelog: -# runs-on: ubuntu-latest -# steps: -# - name: Checkout repository -# uses: actions/checkout@v2 -# - name: Check CHANGELOG max line length -# run: | -# max_line_length=$(cat CHANGELOG.md | grep -Ev "^\[.*\]: https://github.com" | wc -L) -# if [ $max_line_length -ge 80 ]; then -# echo "ERROR: CHANGELOG has lines longer than 80 characters." -# exit 1 -# fi -# -# -# build-mails: -# runs-on: ubuntu-latest -# defaults: -# run: -# working-directory: src/mail -# steps: -# - name: Checkout repository -# uses: actions/checkout@v2 -# - name: Install Node.js -# uses: actions/setup-node@v4 -# with: -# node-version: '18' -# - name: Install yarn -# run: npm install -g yarn -# - name: Install node dependencies -# run: yarn install --frozen-lockfile -# - name: Build mails -# run: yarn build -# -# lint-back: -# runs-on: ubuntu-latest -# defaults: -# run: -# working-directory: src/backend -# steps: -# - name: Checkout repository -# uses: actions/checkout@v2 -# - name: Install Python -# uses: actions/setup-python@v3 -# with: -# python-version: '3.10' -# - name: Install development dependencies -# run: pip install --user .[dev] -# - name: Check code formatting with ruff -# run: ~/.local/bin/ruff format magnify --diff -# - name: Lint code with ruff -# run: ~/.local/bin/ruff check magnify -# - name: Lint code with pylint -# run: ~/.local/bin/pylint magnify -# -# test-back: -# runs-on: ubuntu-latest -# defaults: -# run: -# working-directory: src/backend -# -# services: -# postgres: -# image: postgres:16 -# env: -# POSTGRES_DB: magnify -# POSTGRES_USER: dinum -# POSTGRES_PASSWORD: pass -# ports: -# - 5432:5432 -# # needed because the postgres container does not provide a healthcheck -# options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 -# -# env: -# DJANGO_CONFIGURATION: Test -# DJANGO_SETTINGS_MODULE: magnify.settings -# DJANGO_SECRET_KEY: ThisIsAnExampleKeyForTestPurposeOnly -# OIDC_OP_JWKS_ENDPOINT: /endpoint-for-test-purpose-only -# DB_HOST: localhost -# DB_NAME: magnify -# DB_USER: dinum -# DB_PASSWORD: pass -# DB_PORT: 5432 -# -# steps: -# - name: Checkout repository -# uses: actions/checkout@v2 -# - name: Create writable /data -# run: | -# sudo mkdir -p /data/media && \ -# sudo mkdir -p /data/static -# - name: Install Python -# uses: actions/setup-python@v3 -# with: -# python-version: '3.10' -# - name: Install development dependencies -# run: pip install --user .[dev] -# - name: Install gettext (required to compile messages) -# run: | -# sudo apt-get update -# sudo apt-get install -y gettext -# - name: Generate a MO file from strings extracted from the project -# run: python manage.py compilemessages -# - name: Run tests -# run: ~/.local/bin/pytest -n 2 -# -# i18n-crowdin: -# runs-on: ubuntu-latest -# steps: -# - name: Checkout repository -# uses: actions/checkout@v2 -# -# - name: Install gettext (required to make messages) -# run: | -# sudo apt-get update -# sudo apt-get install -y gettext -# -# - name: Install Python -# uses: actions/setup-python@v3 -# with: -# python-version: '3.10' -# -# - name: Install development dependencies -# working-directory: src/backend -# run: pip install --user .[dev] -# -# - name: Generate the translation base file -# run: ~/.local/bin/django-admin makemessages --keep-pot --all -# -# - name: Load sops secrets -# uses: rouja/actions-sops@main -# with: -# secret-file: .github/workflows/secrets.enc.env -# age-key: ${{ secrets.SOPS_PRIVATE }} -# -# - name: Setup Node.js -# uses: actions/setup-node@v4 -# with: -# node-version: '18.x' -# cache: 'yarn' -# cache-dependency-path: src/frontend/yarn.lock -# -# - name: Install dependencies -# run: cd src/frontend/ && yarn install --frozen-lockfile -# -# - name: Extract the frontend translation -# run: make frontend-i18n-extract -# -# - name: Upload files to Crowdin -# run: | -# docker run \ -# --rm \ -# -e CROWDIN_API_TOKEN=$CROWDIN_API_TOKEN \ -# -e CROWDIN_PROJECT_ID=$CROWDIN_PROJECT_ID \ -# -e CROWDIN_BASE_PATH=$CROWDIN_BASE_PATH \ -# -v "${{ github.workspace }}:/app" \ -# crowdin/cli:3.16.0 \ -# crowdin upload sources -c /app/crowdin/config.yml -# + lint-changelog: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Check CHANGELOG max line length + run: | + max_line_length=$(cat CHANGELOG.md | grep -Ev "^\[.*\]: https://github.com" | wc -L) + if [ $max_line_length -ge 80 ]; then + echo "ERROR: CHANGELOG has lines longer than 80 characters." + exit 1 + fi + + + build-mails: + runs-on: ubuntu-latest + defaults: + run: + working-directory: src/mail + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: '18' + - name: Install yarn + run: npm install -g yarn + - name: Install node dependencies + run: yarn install --frozen-lockfile + - name: Build mails + run: yarn build + + lint-back: + runs-on: ubuntu-latest + defaults: + run: + working-directory: src/backend + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Install Python + uses: actions/setup-python@v3 + with: + python-version: '3.10' + - name: Install development dependencies + run: pip install --user .[dev] + - name: Check code formatting with ruff + run: ~/.local/bin/ruff format magnify --diff + - name: Lint code with ruff + run: ~/.local/bin/ruff check magnify + - name: Lint code with pylint + run: ~/.local/bin/pylint magnify + + test-back: + runs-on: ubuntu-latest + defaults: + run: + working-directory: src/backend + + services: + postgres: + image: postgres:16 + env: + POSTGRES_DB: magnify + POSTGRES_USER: dinum + POSTGRES_PASSWORD: pass + ports: + - 5432:5432 + # needed because the postgres container does not provide a healthcheck + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + env: + DJANGO_CONFIGURATION: Test + DJANGO_SETTINGS_MODULE: magnify.settings + DJANGO_SECRET_KEY: ThisIsAnExampleKeyForTestPurposeOnly + OIDC_OP_JWKS_ENDPOINT: /endpoint-for-test-purpose-only + DB_HOST: localhost + DB_NAME: magnify + DB_USER: dinum + DB_PASSWORD: pass + DB_PORT: 5432 + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Create writable /data + run: | + sudo mkdir -p /data/media && \ + sudo mkdir -p /data/static + - name: Install Python + uses: actions/setup-python@v3 + with: + python-version: '3.10' + - name: Install development dependencies + run: pip install --user .[dev] + - name: Install gettext (required to compile messages) + run: | + sudo apt-get update + sudo apt-get install -y gettext + - name: Generate a MO file from strings extracted from the project + run: python manage.py compilemessages + - name: Run tests + run: ~/.local/bin/pytest -n 2 + + i18n-crowdin: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install gettext (required to make messages) + run: | + sudo apt-get update + sudo apt-get install -y gettext + + - name: Install Python + uses: actions/setup-python@v3 + with: + python-version: '3.10' + + - name: Install development dependencies + working-directory: src/backend + run: pip install --user .[dev] + + - name: Generate the translation base file + run: ~/.local/bin/django-admin makemessages --keep-pot --all + + - name: Load sops secrets + uses: rouja/actions-sops@main + with: + secret-file: .github/workflows/secrets.enc.env + age-key: ${{ secrets.SOPS_PRIVATE }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '18.x' + cache: 'yarn' + cache-dependency-path: src/frontend/yarn.lock + + - name: Install dependencies + run: cd src/frontend/ && yarn install --frozen-lockfile + + - name: Extract the frontend translation + run: make frontend-i18n-extract + + - name: Upload files to Crowdin + run: | + docker run \ + --rm \ + -e CROWDIN_API_TOKEN=$CROWDIN_API_TOKEN \ + -e CROWDIN_PROJECT_ID=$CROWDIN_PROJECT_ID \ + -e CROWDIN_BASE_PATH=$CROWDIN_BASE_PATH \ + -v "${{ github.workspace }}:/app" \ + crowdin/cli:3.16.0 \ + crowdin upload sources -c /app/crowdin/config.yml + From 899702b08d93af7f8217be784084560da6399480 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 15:06:24 +0200 Subject: [PATCH 24/44] re faire magnify --- .github/workflows/magnify.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/magnify.yml b/.github/workflows/magnify.yml index f434f2d88..a5126645e 100644 --- a/.github/workflows/magnify.yml +++ b/.github/workflows/magnify.yml @@ -34,9 +34,9 @@ jobs: - name: Run GitLint run: gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD -check-changelog: - runs-on: ubuntu-latest - steps: + check-changelog: + runs-on: ubuntu-latest + steps: - name: Checkout repository uses: actions/checkout@v2 - name: Check that the CHANGELOG has been modified in the current branch From d76b74a1f77834694c579c42e6b262f777e2e5d5 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 15:10:14 +0200 Subject: [PATCH 25/44] re faire magnify --- .github/workflows/magnify.yml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/.github/workflows/magnify.yml b/.github/workflows/magnify.yml index a5126645e..103eb1325 100644 --- a/.github/workflows/magnify.yml +++ b/.github/workflows/magnify.yml @@ -56,24 +56,7 @@ jobs: fi - build-mails: - runs-on: ubuntu-latest - defaults: - run: - working-directory: src/mail - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Install Node.js - uses: actions/setup-node@v4 - with: - node-version: '18' - - name: Install yarn - run: npm install -g yarn - - name: Install node dependencies - run: yarn install --frozen-lockfile - - name: Build mails - run: yarn build + lint-back: runs-on: ubuntu-latest From cd3b3ebe68870e6b902bc256d2aae560254f846b Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 15:15:54 +0200 Subject: [PATCH 26/44] re faire magnify --- .github/workflows/magnify.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/magnify.yml b/.github/workflows/magnify.yml index 103eb1325..91d51802e 100644 --- a/.github/workflows/magnify.yml +++ b/.github/workflows/magnify.yml @@ -47,10 +47,11 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 + - name: Check CHANGELOG max line length run: | - max_line_length=$(cat CHANGELOG.md | grep -Ev "^\[.*\]: https://github.com" | wc -L) - if [ $max_line_length -ge 80 ]; then + max_line_length=$(awk '!/^\[.*\]: https:\/\/github.com/ { if (length > max) max = length } END { print max }' CHANGELOG.md) + if [ "$max_line_length" -gt 80 ]; then echo "ERROR: CHANGELOG has lines longer than 80 characters." exit 1 fi @@ -62,7 +63,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - working-directory: src/backend + working-directory: src/magnify steps: - name: Checkout repository uses: actions/checkout@v2 @@ -83,7 +84,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - working-directory: src/backend + working-directory: src/tests services: postgres: @@ -147,7 +148,7 @@ jobs: python-version: '3.10' - name: Install development dependencies - working-directory: src/backend + working-directory: src/magnify run: pip install --user .[dev] - name: Generate the translation base file From 39cd9bab4fb2ed34980a517d34ef7a15d108c058 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 15:20:51 +0200 Subject: [PATCH 27/44] re faire magnify --- .github/workflows/magnify.yml | 178 ++++++---------------------------- 1 file changed, 29 insertions(+), 149 deletions(-) diff --git a/.github/workflows/magnify.yml b/.github/workflows/magnify.yml index 91d51802e..75389af29 100644 --- a/.github/workflows/magnify.yml +++ b/.github/workflows/magnify.yml @@ -13,174 +13,54 @@ on: jobs: lint-git: runs-on: ubuntu-latest - if: github.event_name == 'pull_request' # Ne s'applique que pour les pull requests + if: github.event_name == 'pull_request' steps: - - name: Checkout du dépôt + - name: Checkout repository uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Afficher les logs + - name: Show git logs run: git log - - name: Assurer l'absence d'instructions print dans le code + # Add your script to check for print statements here + - name: Check for print statements run: | - ! git diff origin/${{ github.event.pull_request.base.ref }}..HEAD -- . ':(exclude)**/magnify.yml' | grep "print(" - - name: Vérifier l'absence de commits fixup + # Your script to check for print statements + # Example: + # ! git diff origin/${{ github.event.pull_request.base.ref }}..HEAD -- . ':(exclude)**/magnify.yml' | grep "print(" + # Add your script to check for fixup commits here + - name: Check for fixup commits run: | - ! git log | grep 'fixup!' - - name: Installer gitlint - run: pip install --user requests gitlint -# - name: Vérifier les messages de commit ajoutés à la branche principale -# run: ~/.local/bin/gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD + # Your script to check for fixup commits + # Example: + # ! git log | grep 'fixup!' + # Add your script to lint commit messages here - name: Run GitLint run: gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD check-changelog: runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Check that the CHANGELOG has been modified in the current branch - run: git log --name-only --pretty="" origin..HEAD | grep CHANGELOG - - lint-changelog: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Check CHANGELOG max line length - run: | - max_line_length=$(awk '!/^\[.*\]: https:\/\/github.com/ { if (length > max) max = length } END { print max }' CHANGELOG.md) - if [ "$max_line_length" -gt 80 ]; then - echo "ERROR: CHANGELOG has lines longer than 80 characters." - exit 1 - fi - - - - - lint-back: - runs-on: ubuntu-latest - defaults: - run: - working-directory: src/magnify - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Install Python - uses: actions/setup-python@v3 - with: - python-version: '3.10' - - name: Install development dependencies - run: pip install --user .[dev] - - name: Check code formatting with ruff - run: ~/.local/bin/ruff format magnify --diff - - name: Lint code with ruff - run: ~/.local/bin/ruff check magnify - - name: Lint code with pylint - run: ~/.local/bin/pylint magnify - - test-back: - runs-on: ubuntu-latest - defaults: - run: - working-directory: src/tests - - services: - postgres: - image: postgres:16 - env: - POSTGRES_DB: magnify - POSTGRES_USER: dinum - POSTGRES_PASSWORD: pass - ports: - - 5432:5432 - # needed because the postgres container does not provide a healthcheck - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - - env: - DJANGO_CONFIGURATION: Test - DJANGO_SETTINGS_MODULE: magnify.settings - DJANGO_SECRET_KEY: ThisIsAnExampleKeyForTestPurposeOnly - OIDC_OP_JWKS_ENDPOINT: /endpoint-for-test-purpose-only - DB_HOST: localhost - DB_NAME: magnify - DB_USER: dinum - DB_PASSWORD: pass - DB_PORT: 5432 - steps: - name: Checkout repository uses: actions/checkout@v2 - - name: Create writable /data + # Add your script to check CHANGELOG modifications here + - name: Check CHANGELOG modifications run: | - sudo mkdir -p /data/media && \ - sudo mkdir -p /data/static - - name: Install Python - uses: actions/setup-python@v3 - with: - python-version: '3.10' - - name: Install development dependencies - run: pip install --user .[dev] - - name: Install gettext (required to compile messages) - run: | - sudo apt-get update - sudo apt-get install -y gettext - - name: Generate a MO file from strings extracted from the project - run: python manage.py compilemessages - - name: Run tests - run: ~/.local/bin/pytest -n 2 + # Your script to check CHANGELOG modifications + # Example: + # git log --name-only --pretty="" origin..HEAD | grep CHANGELOG - i18n-crowdin: + lint-changelog: runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v2 - - - name: Install gettext (required to make messages) - run: | - sudo apt-get update - sudo apt-get install -y gettext - - - name: Install Python - uses: actions/setup-python@v3 - with: - python-version: '3.10' - - - name: Install development dependencies - working-directory: src/magnify - run: pip install --user .[dev] - - - name: Generate the translation base file - run: ~/.local/bin/django-admin makemessages --keep-pot --all - - - name: Load sops secrets - uses: rouja/actions-sops@main - with: - secret-file: .github/workflows/secrets.enc.env - age-key: ${{ secrets.SOPS_PRIVATE }} - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '18.x' - cache: 'yarn' - cache-dependency-path: src/frontend/yarn.lock - - - name: Install dependencies - run: cd src/frontend/ && yarn install --frozen-lockfile - - - name: Extract the frontend translation - run: make frontend-i18n-extract - - - name: Upload files to Crowdin + # Add your script to check CHANGELOG max line length here + - name: Check CHANGELOG max line length run: | - docker run \ - --rm \ - -e CROWDIN_API_TOKEN=$CROWDIN_API_TOKEN \ - -e CROWDIN_PROJECT_ID=$CROWDIN_PROJECT_ID \ - -e CROWDIN_BASE_PATH=$CROWDIN_BASE_PATH \ - -v "${{ github.workspace }}:/app" \ - crowdin/cli:3.16.0 \ - crowdin upload sources -c /app/crowdin/config.yml - + # Your script to check CHANGELOG max line length + # Example: + # max_line_length=$(awk '!/^\[.*\]: https:\/\/github.com/ { if (length > max) max = length } END { print max }' CHANGELOG.md) + # if [ "$max_line_length" -gt 80 ]; then + # echo "ERROR: CHANGELOG has lines longer than 80 characters." + # exit 1 + # fi From 2cbf106370cc88e4646ee3dcd13d57c2e960f1fe Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 15:24:17 +0200 Subject: [PATCH 28/44] re faire magnify lintgit --- .github/workflows/magnify.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/magnify.yml b/.github/workflows/magnify.yml index 75389af29..49e6cc8db 100644 --- a/.github/workflows/magnify.yml +++ b/.github/workflows/magnify.yml @@ -19,6 +19,9 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 + - name: Install gitlint + run: | + pip install gitlint - name: Show git logs run: git log # Add your script to check for print statements here @@ -37,6 +40,7 @@ jobs: - name: Run GitLint run: gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD + check-changelog: runs-on: ubuntu-latest steps: From 8e4459a49ce1a62bf503d6ee24a933bbe8b64a89 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 15:26:32 +0200 Subject: [PATCH 29/44] re faire magnify lintgit --- .github/workflows/magnify.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/magnify.yml b/.github/workflows/magnify.yml index 49e6cc8db..9d7c3e3bf 100644 --- a/.github/workflows/magnify.yml +++ b/.github/workflows/magnify.yml @@ -38,7 +38,8 @@ jobs: # ! git log | grep 'fixup!' # Add your script to lint commit messages here - name: Run GitLint - run: gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD + run: /path/to/gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD + check-changelog: From 470d2cf9603d4149829f5d411b82f000ba495303 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 15:29:23 +0200 Subject: [PATCH 30/44] re faire dockerhub --- .github/workflows/docker-hub.yml | 67 ++++++++++++++++---------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml index 7bd638730..93b9d0215 100644 --- a/.github/workflows/docker-hub.yml +++ b/.github/workflows/docker-hub.yml @@ -15,39 +15,40 @@ env: DOCKER_USER: 1001:127 jobs: -# build-and-push-backend: -# runs-on: ubuntu-latest -# steps: -# - -# name: Checkout -# uses: actions/checkout@v4 -# - -# name: Docker meta -# id: meta -# uses: docker/metadata-action@v5 -# with: -# images: lasuite/magnify-backend -# - -# name: Load sops secrets -# uses: rouja/actions-sops@main -# with: -# secret-file: .github/workflows/secrets.enc.env -# age-key: ${{ secrets.SOPS_PRIVATE }} -# - -# name: Login to DockerHub -# if: github.event_name != 'pull_request' -# run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin -# - -# name: Build and push -# uses: docker/build-push-action@v5 -# with: -# context: . -# target: backend-production -# build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 -# push: ${{ github.event_name != 'pull_request' }} -# tags: ${{ steps.meta.outputs.tags }} -# labels: ${{ steps.meta.outputs.labels }} -# + build-and-push-backend: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v4 + - + name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: lasuite/magnify-backend + - + name: Load sops secrets + uses: rouja/actions-sops@main + with: + secret-file: .github/workflows/secrets.enc.env + age-key: ${{ secrets.SOPS_PRIVATE }} + - + name: Login to DockerHub + if: github.event_name != 'pull_request' + run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin + - + name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + target: backend-production + build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + build-and-push-frontend: runs-on: ubuntu-latest steps: From 7a2610eb7aa065e8d6f2a6194450d356f29806d1 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 15:32:26 +0200 Subject: [PATCH 31/44] re faire dockerhub frontend --- .github/workflows/docker-hub.yml | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml index 93b9d0215..1ddcea047 100644 --- a/.github/workflows/docker-hub.yml +++ b/.github/workflows/docker-hub.yml @@ -52,35 +52,30 @@ jobs: build-and-push-frontend: runs-on: ubuntu-latest steps: - - - name: Checkout + - name: Checkout uses: actions/checkout@v4 - - - name: Docker meta + - name: Docker meta id: meta uses: docker/metadata-action@v5 with: images: lasuite/magnify-frontend - - - name: Load sops secrets + - name: Load sops secrets uses: rouja/actions-sops@main with: secret-file: .github/workflows/secrets.enc.env age-key: ${{ secrets.SOPS_PRIVATE }} - - - name: Login to DockerHub + - name: Login to DockerHub if: github.event_name != 'pull_request' run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin - - - name: Build and push + - name: Build and push uses: docker/build-push-action@v5 with: context: . - target: frontend-production - build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + file: Dockerfile # Ajoutez cette ligne pour spécifier le Dockerfile à utiliser + build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 notify-argocd: needs: From b463387762efa838d8d419f7098fc923779344a5 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 15:41:28 +0200 Subject: [PATCH 32/44] re faire dockerhub frontend --- .github/workflows/docker-hub.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml index 1ddcea047..28d63bf60 100644 --- a/.github/workflows/docker-hub.yml +++ b/.github/workflows/docker-hub.yml @@ -52,29 +52,33 @@ jobs: build-and-push-frontend: runs-on: ubuntu-latest steps: - - name: Checkout + - name: Checkout Repository uses: actions/checkout@v4 + - name: Docker meta id: meta uses: docker/metadata-action@v5 with: images: lasuite/magnify-frontend - - name: Load sops secrets + + - name: Load Sops Secrets uses: rouja/actions-sops@main with: secret-file: .github/workflows/secrets.enc.env age-key: ${{ secrets.SOPS_PRIVATE }} + - name: Login to DockerHub if: github.event_name != 'pull_request' run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin - - name: Build and push + + - name: Build and Push Docker Image uses: docker/build-push-action@v5 with: context: . + file: Dockerfile # Spécifiez le chemin vers votre Dockerfile push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - file: Dockerfile # Ajoutez cette ligne pour spécifier le Dockerfile à utiliser build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 notify-argocd: From 7e6d3a866ed441356e6b4cdc921a9cfa1d345b93 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 15:43:40 +0200 Subject: [PATCH 33/44] re faire dockerhub frontend --- .github/workflows/docker-hub.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml index 28d63bf60..e42823ee9 100644 --- a/.github/workflows/docker-hub.yml +++ b/.github/workflows/docker-hub.yml @@ -77,7 +77,7 @@ jobs: context: . file: Dockerfile # Spécifiez le chemin vers votre Dockerfile push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} + #tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 From c5b32289edbe7befe3d7fe7fb2985a432d680d53 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 15:46:22 +0200 Subject: [PATCH 34/44] re faire dockerhub frontend --- .github/workflows/docker-hub.yml | 54 ++++++++++++++++---------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml index e42823ee9..db421df62 100644 --- a/.github/workflows/docker-hub.yml +++ b/.github/workflows/docker-hub.yml @@ -49,37 +49,37 @@ jobs: labels: ${{ steps.meta.outputs.labels }} - build-and-push-frontend: - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v4 +build-and-push-frontend: + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: lasuite/magnify-frontend + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: lasuite/magnify-frontend - - name: Load Sops Secrets - uses: rouja/actions-sops@main - with: - secret-file: .github/workflows/secrets.enc.env - age-key: ${{ secrets.SOPS_PRIVATE }} + - name: Load Sops Secrets + uses: rouja/actions-sops@main + with: + secret-file: .github/workflows/secrets.enc.env + age-key: ${{ secrets.SOPS_PRIVATE }} - - name: Login to DockerHub - if: github.event_name != 'pull_request' - run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin + - name: Login to DockerHub + if: github.event_name != 'pull_request' + run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin - - name: Build and Push Docker Image - uses: docker/build-push-action@v5 - with: - context: . - file: Dockerfile # Spécifiez le chemin vers votre Dockerfile - push: ${{ github.event_name != 'pull_request' }} - #tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 + - name: Build and Push Docker Image + uses: docker/build-push-action@v5 + with: + context: . + file: Dockerfile # Spécifiez le chemin vers votre Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 notify-argocd: needs: From 7646c4f1d8fa921d6323092ee29f896110ce72f7 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 15:52:56 +0200 Subject: [PATCH 35/44] re faire dockerhub frontend --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 58dd7d709..ff90106eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -151,8 +151,10 @@ ARG MAGNIFY_STATIC_ROOT=/data/static WORKDIR /app/sandbox +# Définition explicite de la cible frontend-production +FROM base as frontend-production # Copy statics COPY --from=link-collector ${MAGNIFY_STATIC_ROOT} ${MAGNIFY_STATIC_ROOT} -# The default command runs gunicorn WSGI server in the sandbox -CMD gunicorn -c /usr/local/etc/gunicorn/magnify.py wsgi:application +# Copier les fichiers statiques de frontend dans la cible frontend-production +CMD cp -r /app/sandbox/static/frontend ${MAGNIFY_STATIC_ROOT} From 2ba29822feb098d8173d6bff05b348e4d6fc6da5 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 15:54:02 +0200 Subject: [PATCH 36/44] re faire dockerhub frontend --- .github/workflows/docker-hub.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml index db421df62..9ba895e3e 100644 --- a/.github/workflows/docker-hub.yml +++ b/.github/workflows/docker-hub.yml @@ -77,7 +77,7 @@ build-and-push-frontend: context: . file: Dockerfile # Spécifiez le chemin vers votre Dockerfile push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} + #tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 From 5ad62eed419fd7be569aa1afbdc03e0f224b1fa2 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 16:14:06 +0200 Subject: [PATCH 37/44] re faire docker frontend and backend --- Dockerfile | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index ff90106eb..88efa0118 100644 --- a/Dockerfile +++ b/Dockerfile @@ -151,10 +151,35 @@ ARG MAGNIFY_STATIC_ROOT=/data/static WORKDIR /app/sandbox -# Définition explicite de la cible frontend-production -FROM base as frontend-production # Copy statics COPY --from=link-collector ${MAGNIFY_STATIC_ROOT} ${MAGNIFY_STATIC_ROOT} -# Copier les fichiers statiques de frontend dans la cible frontend-production -CMD cp -r /app/sandbox/static/frontend ${MAGNIFY_STATIC_ROOT} +# The default command runs gunicorn WSGI server in the sandbox +CMD gunicorn -c /usr/local/etc/gunicorn/magnify.py wsgi:application +# ---- Front-end image ---- +FROM nginxinc/nginx-unprivileged:1.25 as frontend-production + +# Un-privileged user running the application +ARG DOCKER_USER +USER ${DOCKER_USER} + +COPY --from=frontend-builder \ + /builder/apps/magnify/out \ + /usr/share/nginx/html + +COPY ./src/frontend/apps/magnify/conf/default.conf /etc/nginx/conf.d +# ---- Production image ---- +FROM core as backend-production + +ARG MAGNIFY_STATIC_ROOT=/data/static + +# Gunicorn +RUN mkdir -p /usr/local/etc/gunicorn +COPY docker/files/usr/local/etc/gunicorn/magnify.py /usr/local/etc/gunicorn/magnify.py + +# Un-privileged user running the application +ARG DOCKER_USER +USER ${DOCKER_USER} + +# Copy statics +COPY --from=link-collector ${MAGNIFY_STATIC_ROOT} ${MAGNIFY_STATIC_ROOT} From 462d7a07eec92be8ba812743094cb4eb03e5211f Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 16:28:37 +0200 Subject: [PATCH 38/44] fix dockerfile --- Dockerfile | 104 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 77 insertions(+), 27 deletions(-) diff --git a/Dockerfile b/Dockerfile index 88efa0118..f04100898 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,13 @@ # ---- Base image to inherit from ---- FROM python:3.10-buster as base +# Upgrade pip to its latest release to speed up dependencies installation +RUN python -m pip install --upgrade pip + +# Upgrade system packages to install security updates +RUN apt-get update && \ + apt-get -y upgrade && \ + rm -rf /var/lib/apt/lists/* # ---- Front-end builder image ---- FROM node:16.15 as front-builder @@ -28,6 +35,28 @@ WORKDIR /builder/src/frontend RUN yarn install --frozen-lockfile && \ yarn build +# ---- Front-end image ---- +FROM nginxinc/nginx-unprivileged:1.25 as frontend-production + +# Un-privileged user running the application +ARG DOCKER_USER +USER ${DOCKER_USER} + +COPY --from=frontend-builder \ + /builder/apps/impress/out \ + /usr/share/nginx/html + +COPY ./src/frontend/apps/impress/conf/default.conf /etc/nginx/conf.d + +# Copy entrypoint +COPY ./docker/files/usr/local/bin/entrypoint /usr/local/bin/entrypoint + +ENTRYPOINT [ "/usr/local/bin/entrypoint" ] + +CMD ["nginx", "-g", "daemon off;"] + + + # ---- Back-end builder image ---- FROM base as back-builder @@ -48,6 +77,54 @@ RUN pip install --upgrade pip RUN mkdir /install && \ pip install --prefix=/install .[sandbox] +# ---- Development image ---- +FROM core as backend-development + +# Switch back to the root user to install development dependencies +USER root:root + +# Install psql +RUN apt-get update && \ + apt-get install -y postgresql-client && \ + rm -rf /var/lib/apt/lists/* + +# Uninstall impress and re-install it in editable mode along with development +# dependencies +RUN pip uninstall -y impress +RUN pip install -e .[dev] + +# Restore the un-privileged user running the application +ARG DOCKER_USER +USER ${DOCKER_USER} + +# Target database host (e.g. database engine following docker compose services +# name) & port +ENV DB_HOST=postgresql \ + DB_PORT=5432 + +# Run django development server +CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"] + +# ---- Production image ---- +FROM core as backend-production + +ARG MAGNIFY_STATIC_ROOT=/data/static + +# Gunicorn +RUN mkdir -p /usr/local/etc/gunicorn +COPY docker/files/usr/local/etc/gunicorn/magnify.py /usr/local/etc/gunicorn/magnify.py + +# Un-privileged user running the application +ARG DOCKER_USER +USER ${DOCKER_USER} + +# Copy statics +COPY --from=link-collector ${MAGNIFY_STATIC_ROOT} ${MAGNIFY_STATIC_ROOT} + + +# The default command runs gunicorn WSGI server in magnify's main module +CMD ["gunicorn", "-c", "/usr/local/etc/gunicorn/magnify.py", "magnify.wsgi:application"] + # ---- static link collector ---- FROM base as link-collector ARG MAGNIFY_STATIC_ROOT=/data/static @@ -156,30 +233,3 @@ COPY --from=link-collector ${MAGNIFY_STATIC_ROOT} ${MAGNIFY_STATIC_ROOT} # The default command runs gunicorn WSGI server in the sandbox CMD gunicorn -c /usr/local/etc/gunicorn/magnify.py wsgi:application -# ---- Front-end image ---- -FROM nginxinc/nginx-unprivileged:1.25 as frontend-production - -# Un-privileged user running the application -ARG DOCKER_USER -USER ${DOCKER_USER} - -COPY --from=frontend-builder \ - /builder/apps/magnify/out \ - /usr/share/nginx/html - -COPY ./src/frontend/apps/magnify/conf/default.conf /etc/nginx/conf.d -# ---- Production image ---- -FROM core as backend-production - -ARG MAGNIFY_STATIC_ROOT=/data/static - -# Gunicorn -RUN mkdir -p /usr/local/etc/gunicorn -COPY docker/files/usr/local/etc/gunicorn/magnify.py /usr/local/etc/gunicorn/magnify.py - -# Un-privileged user running the application -ARG DOCKER_USER -USER ${DOCKER_USER} - -# Copy statics -COPY --from=link-collector ${MAGNIFY_STATIC_ROOT} ${MAGNIFY_STATIC_ROOT} From 37f1def8919f35e661ed8a3b2abb2f0596014111 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 16:31:57 +0200 Subject: [PATCH 39/44] fix dockerhub --- .github/workflows/docker-hub.yml | 36 ++++++++++++++++---------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml index 9ba895e3e..dd97e67f2 100644 --- a/.github/workflows/docker-hub.yml +++ b/.github/workflows/docker-hub.yml @@ -49,31 +49,31 @@ jobs: labels: ${{ steps.meta.outputs.labels }} -build-and-push-frontend: - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v4 + build-and-push-frontend: + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: images: lasuite/magnify-frontend - - name: Load Sops Secrets - uses: rouja/actions-sops@main - with: + - name: Load Sops Secrets + uses: rouja/actions-sops@main + with: secret-file: .github/workflows/secrets.enc.env age-key: ${{ secrets.SOPS_PRIVATE }} - - name: Login to DockerHub - if: github.event_name != 'pull_request' - run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin + - name: Login to DockerHub + if: github.event_name != 'pull_request' + run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin - - name: Build and Push Docker Image - uses: docker/build-push-action@v5 - with: + - name: Build and Push Docker Image + uses: docker/build-push-action@v5 + with: context: . file: Dockerfile # Spécifiez le chemin vers votre Dockerfile push: ${{ github.event_name != 'pull_request' }} From 12415cc72ee2c07b27be9b1a7ef6f4305d26a6c0 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 16:53:24 +0200 Subject: [PATCH 40/44] fix dockerhub --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f04100898..9b1551f09 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,10 +43,10 @@ ARG DOCKER_USER USER ${DOCKER_USER} COPY --from=frontend-builder \ - /builder/apps/impress/out \ + /builder/apps/magnify/out \ /usr/share/nginx/html -COPY ./src/frontend/apps/impress/conf/default.conf /etc/nginx/conf.d +COPY ./src/frontend/apps/magnify/conf/default.conf /etc/nginx/conf.d # Copy entrypoint COPY ./docker/files/usr/local/bin/entrypoint /usr/local/bin/entrypoint From 550bbe7ffc297cc7ab403d123aa7d2d80ac1b433 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 17:00:52 +0200 Subject: [PATCH 41/44] fix dockerhub --- .github/workflows/docker-hub.yml | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml index dd97e67f2..23d4aef5c 100644 --- a/.github/workflows/docker-hub.yml +++ b/.github/workflows/docker-hub.yml @@ -18,38 +18,35 @@ jobs: build-and-push-backend: runs-on: ubuntu-latest steps: - - - name: Checkout + - name: Checkout uses: actions/checkout@v4 - - - name: Docker meta + + - name: Docker meta id: meta uses: docker/metadata-action@v5 with: images: lasuite/magnify-backend - - - name: Load sops secrets + + - name: Load sops secrets uses: rouja/actions-sops@main with: secret-file: .github/workflows/secrets.enc.env age-key: ${{ secrets.SOPS_PRIVATE }} - - - name: Login to DockerHub + + - name: Login to DockerHub if: github.event_name != 'pull_request' run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin - - - name: Build and push + + - name: Build and push uses: docker/build-push-action@v5 with: context: . target: backend-production - build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - - build-and-push-frontend: + build-and-push-frontend: runs-on: ubuntu-latest steps: - name: Checkout Repository From 97a33f39438b5e00caccc3a800a1c06074f0fa68 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 17:03:54 +0200 Subject: [PATCH 42/44] fix dockerhub --- .github/workflows/docker-hub.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml index 23d4aef5c..049315698 100644 --- a/.github/workflows/docker-hub.yml +++ b/.github/workflows/docker-hub.yml @@ -20,33 +20,31 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - - name: Docker meta id: meta uses: docker/metadata-action@v5 with: images: lasuite/magnify-backend - - name: Load sops secrets uses: rouja/actions-sops@main with: secret-file: .github/workflows/secrets.enc.env age-key: ${{ secrets.SOPS_PRIVATE }} - - name: Login to DockerHub if: github.event_name != 'pull_request' run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin - - name: Build and push uses: docker/build-push-action@v5 with: context: . target: backend-production + build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - build-and-push-frontend: + + build-and-push-frontend: runs-on: ubuntu-latest steps: - name: Checkout Repository From 869dc5338bce638bb8c0a9511c32db294c566434 Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 17:07:42 +0200 Subject: [PATCH 43/44] fix dockerhub --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9b1551f09..405108770 100644 --- a/Dockerfile +++ b/Dockerfile @@ -108,7 +108,6 @@ CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"] # ---- Production image ---- FROM core as backend-production -ARG MAGNIFY_STATIC_ROOT=/data/static # Gunicorn RUN mkdir -p /usr/local/etc/gunicorn From 7b2b761a89d15681c44b152ed644e0c75ab7a6fa Mon Sep 17 00:00:00 2001 From: malikawannasi Date: Fri, 12 Apr 2024 17:08:53 +0200 Subject: [PATCH 44/44] fix dockerhub --- Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 405108770..0ed677893 100644 --- a/Dockerfile +++ b/Dockerfile @@ -105,9 +105,6 @@ ENV DB_HOST=postgresql \ # Run django development server CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"] -# ---- Production image ---- -FROM core as backend-production - # Gunicorn RUN mkdir -p /usr/local/etc/gunicorn