iD editor upgrade to v2.30.2

.circleci/config.yml

@@ -32,7 +32,7 @@ jobs:
           name: Run yarn test
           command: |
             cd ${CIRCLE_WORKING_DIRECTORY}/frontend/
-            CI=true yarn test -w 1
+            CI=true yarn test -w 3 --silent
             CI=true GENERATE_SOURCEMAP=false yarn build
 
   backend-code-check-PEP8:
@@ -42,7 +42,7 @@ jobs:
       - checkout
       - run:
           name: flake8 tests
-          command: | 
+          command: |
             pip install flake8
             flake8 manage.py backend tests migrations
 
@@ -53,7 +53,7 @@ jobs:
       - checkout
       - run:
           name: black tests
-          command: | 
+          command: |
             pip install 'black==23.12.1' ## TODO: Update to version 24
             black --check manage.py backend tests migrations
 
@@ -81,7 +81,7 @@ jobs:
       - run: sudo apt-get update
       - run: sudo apt-get -y install libgeos-dev # Required for shapely
       - run: sudo apt-get -y install proj-bin libproj-dev
-      - run: pip install --upgrade pip pdm==2.7.4
+      - run: pip install --upgrade pip pdm==2.18.1
       - run: pdm export --dev --without-hashes > requirements.txt
       - run: pip install -r requirements.txt
       - run: mkdir --mode 766 -p /tmp/logs
@@ -294,7 +294,7 @@ workflows:
           name: Deploy backend production
           gitsha: $CIRCLE_SHA1
           stack_name: "tm4-production"
-          host_ami: "ami-01e5ff16fd6e8c542"
+          host_ami: "/aws/service/canonical/ubuntu/server/20.04/stable/current/amd64/hvm/ebs-gp2/ami-id"
           backend_instance_type: c6a.large
           pg_version: "13.10"
           pg_param_group: "default.postgres13"
@@ -357,7 +357,7 @@ workflows:
           name: Deploy backend production
           gitsha: $CIRCLE_SHA1
           stack_name: "tm4-production"
-          host_ami: "ami-01e5ff16fd6e8c542"
+          host_ami: "/aws/service/canonical/ubuntu/server/20.04/stable/current/amd64/hvm/ebs-gp2/ami-id"
           backend_instance_type: c6a.large
           pg_version: "13.10"
           pg_param_group: "default.postgres13"
@@ -384,7 +384,7 @@ workflows:
           name: Deploy TeachOSM Backend
           gitsha: $CIRCLECI_SHA1
           stack_name: "teachosm"
-          host_ami: "ami-01e5ff16fd6e8c542"
+          host_ami: "/aws/service/canonical/ubuntu/server/20.04/stable/current/amd64/hvm/ebs-gp2/ami-id"
           requires:
             - backend-functional-tests
           context: tasking-manager-teachosm
@@ -422,7 +422,7 @@ workflows:
           name: Deploy staging backend
           gitsha: $CIRCLE_SHA1
           stack_name: "staging"
-          host_ami: "ami-01e5ff16fd6e8c542"
+          host_ami: "/aws/service/canonical/ubuntu/server/20.04/stable/current/amd64/hvm/ebs-gp2/ami-id"
           pg_version: "14.8"
           pg_param_group: "default.postgres14"
           db_instance_type: "db.t4g.small"
@@ -452,8 +452,8 @@ workflows:
               pattern: "^deployment/.*"
               value: << pipeline.git.branch >>
         - or:
-          - equal: [ develop, << pipeline.git.branch >> ]
-          - equal: [ fix/febdeployment, << pipeline.git.branch >> ] # change this to the branch you wish to test
+          ## - equal: [ develop, << pipeline.git.branch >> ] # Disabled while we use dev setup for e2e testing
+          - equal: [ dev-switch-to-sandbox, << pipeline.git.branch >> ]
     jobs:
       - database-backup:
           name: Backup development database
@@ -473,8 +473,8 @@ workflows:
           name: Deploy development backend
           gitsha: $CIRCLE_SHA1
           stack_name: "dev"
-          host_ami: "ami-01e5ff16fd6e8c542"
-          pg_version: "14.8"
+          host_ami: "/aws/service/canonical/ubuntu/server/20.04/stable/current/amd64/hvm/ebs-gp2/ami-id"
+          pg_version: "14.10"
           pg_param_group: "default.postgres14"
           db_instance_type: "db.t4g.small"
           backend_instance_type: "t3.medium"
@@ -509,5 +509,3 @@ workflows:
       - backend-code-check-PEP8
       - backend-code-check-Black
       - backend-functional-tests
-
-

.circleci/rdsid.sh

@@ -6,4 +6,4 @@ for line in $ARNS; do
     if [[ ! -z $MATCHES ]]; then
         echo $MATCHES
     fi
-done
+done

.coveragerc

@@ -21,7 +21,7 @@ exclude_lines =
 
     # Don't complain if tests don't hit defensive assertion code:
     except Exception as e:
-        
+
 
 ignore_errors = True
 

.dockerignore

@@ -20,4 +20,3 @@ frontend/package-lock.json
 frontend/.env
 frontend/.eslintcache
 frontend/coverage/
-

.github/pull_request_template.md

@@ -36,6 +36,6 @@ Notes for the reviewer. How to test this change?
 - ✅ Provide tests for your changes.
 - 📝 Use descriptive commit messages.
 - 📗 Update any related documentation and include any relevant screenshots.
-- 🔠 Does this PR introduce or change any environment variables? If so, make sure to specify this change in the description. 
+- 🔠 Does this PR introduce or change any environment variables? If so, make sure to specify this change in the description.
 
 ## [optional] What gif best describes this PR or how it makes you feel?

.github/workflows/docker-image-push.yml

@@ -1,4 +1,3 @@
-#
 name: Build & publish TM backend container image
 
 on:
@@ -18,50 +17,15 @@ on:
       - deployment/container-tasking-manager
 
 
-env:
-  REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
-
-
 jobs:
-  build-and-push-image:
-    runs-on: ubuntu-latest
-    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
-    permissions:
-      contents: read
-      packages: write
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Log in to the Container registry
-        uses: docker/[email protected]
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/[email protected]
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-
-          tags: |
-            type=ref,event=pr
-            type=ref,event=tag
-            type=ref,event=branch
-            type=semver,pattern=raw
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}}
-            type=sha
-
-      - name: Build and push Docker image
-        uses: docker/[email protected]
-        with:
-          context: .
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+  backend-build:
+    uses: hotosm/gh-workflows/.github/workflows/[email protected]
+    with:
+      image_name: ghcr.io/${{ github.repository }}/backend
+      build_target: prod
+
+  # frontend-build:
+  #   uses: hotosm/gh-workflows/.github/workflows/[email protected]
+  #   with:
+  #     image_name: ghcr.io/${{ github.repository }}/backend
+  #     dockerfile: scripts/docker/Dockerfile.frontend

.github/workflows/ecs-deploy.yml

@@ -97,6 +97,3 @@ jobs:
           service: ${{ env.ECS_SERVICE }}
           cluster: ${{ env.ECS_CLUSTER }}
           wait-for-service-stability: true
-
-
-

.pre-commit-config.yaml

@@ -0,0 +1,106 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+    - id: trailing-whitespace
+      name: trim trailing whitespace
+      description: trims trailing whitespace.
+      entry: trailing-whitespace-fixer
+      language: python
+      types: [text]
+      stages: [commit, push, manual]
+
+    - id: end-of-file-fixer
+      name: fix end of files
+      description: ensures that a file is either empty, or ends with one newline.
+      entry: end-of-file-fixer
+      language: python
+      types: [text]
+      stages: [commit, push, manual]
+
+    - id: detect-aws-credentials
+      name: detect aws credentials
+      description: detects *your* aws credentials from the aws cli credentials file.
+      entry: detect-aws-credentials
+      language: python
+      types: [text]
+      args:
+        - "--allow-missing-credentials"
+
+    - id: detect-private-key
+      name: detect private key
+      description: detects the presence of private keys.
+      entry: detect-private-key
+      language: python
+      types: [text]
+
+    # - id: pretty-format-json
+    #   name: pretty format json
+    #   description: sets a standard for formatting json files.
+    #   entry: pretty-format-json
+    #   language: python
+    #   types: [text]
+    #   files: \.(json)$
+    #   args:
+    #     - "--autofix"
+
+    - id: check-merge-conflict
+      name: check for merge conflicts
+      description: checks for files that contain merge conflict strings.
+      entry: check-merge-conflict
+      language: python
+      types: [text]
+
+    - id: check-yaml
+      name: YAML File check
+      description: Checks if YAML File is valid and there are not any indention errors.
+
+    - id: check-added-large-files
+      name: check for added large files
+      description: prevents giant files from being committed.
+      entry: check-added-large-files
+      language: python
+      stages: [commit, push, manual]
+      args: ['--maxkb=10240']
+
+  # Versioning: Commit messages & changelog
+  - repo: https://github.com/commitizen-tools/commitizen
+    rev: v3.29.0
+    hooks:
+      - id: commitizen
+        stages: [commit-msg]
+
+  # Lint / autoformat: Python code
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: "v0.6.4"
+    hooks:
+      # Run the linter
+      - id: ruff
+        files: ^backend/(?:.*/)*.*$
+        args: [--fix, --exit-non-zero-on-fix]
+      # Run the formatter
+      - id: ruff-format
+        files: ^backend/(?:.*/)*.*$
+
+  # INFO: Searches for code that is used or lingering around. (Disabled since there were a lot of work from dev end to remove stuff)
+  # - repo: https://github.com/asottile/dead
+  #   rev: v1.5.2
+  #   hooks:
+  #   - id: dead
+
+  # Autoformat: YAML, JSON, Markdown, etc. (Disabled since pre-commit wasn't able to fix and needs dev input)
+  # - repo: https://github.com/pre-commit/mirrors-prettier
+  #   rev: v3.1.0
+  #   hooks:
+  #     - id: prettier
+  #       args:
+  #         [
+  #           --ignore-unknown,
+  #           --no-error-on-unmatched-pattern,
+  #           "!chart/**",
+  #           "!CHANGELOG.md",
+  #           "!CONTRIBUTING.md",
+  #           "!frontend/pnpm-lock.yaml",
+  #           "!backend/tests/test_data/**",
+  #         ]

.tx/config

@@ -30,4 +30,3 @@ trans.el    = frontend/src/locales/el.json
 trans.it    = frontend/src/locales/it.json
 trans.ja    = frontend/src/locales/ja.json
 trans.zh_TW = frontend/src/locales/zh_TW.json
-

Dockerfile

@@ -1,10 +1,13 @@
 ARG DEBIAN_IMG_TAG=slim-bookworm
 ARG PYTHON_IMG_TAG=3.10
 
+
+
 FROM docker.io/python:${PYTHON_IMG_TAG}-${DEBIAN_IMG_TAG} as base
 ARG APP_VERSION=0.1.0
 ARG DOCKERFILE_VERSION=0.5.0
 ARG ALPINE_IMG_TAG
+ARG DEBIAN_IMG_TAG
 ARG PYTHON_IMG_TAG
 ARG [email protected]
 LABEL org.hotosm.tasks.app-version="${APP_VERSION}" \
@@ -25,7 +28,7 @@ FROM base as extract-deps
 RUN pip install --no-cache-dir --upgrade pip
 WORKDIR /opt/python
 COPY pyproject.toml pdm.lock README.md /opt/python/
-RUN pip install --no-cache-dir pdm==2.7.4
+RUN pip install --no-cache-dir pdm==2.18.1
 RUN pdm export --prod --without-hashes > requirements.txt
 
 
@@ -34,13 +37,13 @@ FROM base as build
 RUN pip install --no-cache-dir --upgrade pip
 WORKDIR /opt/python
 # Setup backend build-time dependencies
-RUN apt-get update
-RUN apt-get install --no-install-recommends -y build-essential
-RUN apt-get install --no-install-recommends -y \
+RUN apt-get update && apt-get install --no-install-recommends -y \
+        build-essential \
+        libffi-dev \
+        libgeos-dev \
         postgresql-server-dev-15 \
         python3-dev \
-        libffi-dev \
-        libgeos-dev
+    && rm -rf /var/lib/apt/lists/*
 # Setup backend Python dependencies
 COPY --from=extract-deps \
     /opt/python/requirements.txt /opt/python/
@@ -63,8 +66,8 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
 # Setup backend runtime dependencies
 RUN apt-get update && \
     apt-get install --no-install-recommends -y \
-        postgresql-client libgeos3.11.1 proj-bin && \
-    apt-get clean && rm -rf /var/lib/apt/lists/*
+        libgeos3.11.1 postgresql-client proj-bin && \
+    rm -rf /var/lib/apt/lists/*
 COPY --from=build \
     /home/appuser/.local \
     /home/appuser/.local
@@ -79,7 +82,7 @@ COPY manage.py .
 
 FROM runtime as debug
 RUN pip install --user --no-warn-script-location \
-    --no-cache-dir debugpy==1.6.7
+    --no-cache-dir debugpy==1.8.5
 EXPOSE 5678/tcp
 CMD ["python", "-m", "debugpy", "--wait-for-client", "--listen", "0.0.0.0:5678", \
     "-m", "gunicorn", "-c", "python:backend.gunicorn", "manage:application", \
@@ -91,12 +94,13 @@ FROM runtime as prod
 USER root
 RUN apt-get update && \
 	apt-get install -y curl && \
-	apt-get clean && \
 	rm -rf /var/lib/apt/lists/*
 # Pre-compile packages to .pyc (init speed gains)
 RUN python -c "import compileall; compileall.compile_path(maxlevels=10, quiet=1)"
 RUN python -m compileall .
 EXPOSE 5000/tcp
 USER appuser:appuser
+# Default gunicorn worker count is 1
+# For prod the WEB_CONCURRENCY env var can be used to set this
 CMD ["gunicorn", "-c", "python:backend.gunicorn", "manage:application", \
-    "--workers", "1", "--log-level", "error"]
+    "--log-level", "error"]