Use uv for dependency management and versioning (#553)

Author: pederhan

.dockerignore

@@ -0,0 +1,5 @@
+build
+ci
+logs/*
+.tox
+.venv

.github/workflows/container-image.yml

@@ -72,28 +72,30 @@ jobs:
       # There's a docker-compose.yml file in the mreg-cli repo that wants the image from ghcr.io,
       # but we want to use the newly built custom image
       run: docker tag mreg ghcr.io/unioslo/mreg:latest
-    - name: Setup Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: 3.11
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
+    - name: Set up Python
+      run: uv python install 3.11
+    - name: Install git
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y git
     - name: Install mreg-cli
       run: |
-        curl -L -o mreg-cli https://github.com/unioslo/mreg-cli/releases/download/1.0.1/mreg-cli-ubuntu-latest-3.12
-        chmod 0755 mreg-cli
-        sudo mv mreg-cli /usr/bin/  # put it somewhere in $PATH
+        git clone https://github.com/unioslo/mreg-cli.git
+        cd mreg-cli
+        uv venv
+        uv pip install -e ".[dev]"
     - name: Run the tests
       run: |
-        wget -nd https://github.com/unioslo/mreg-cli/archive/refs/heads/master.zip
-        unzip master.zip
-        cd mreg-cli-master
-        cp data/mreg-cli.conf ~/.config/mreg-cli.conf
-        pip install -r requirements-dev.txt
-        ci/run_testsuite_and_record.sh
+        cd mreg-cli
+        . .venv/bin/activate
+        uv run ci/run_testsuite_and_record.sh
     - name: Upload the log as an artifact
       uses: actions/upload-artifact@v4
       with:
         name: new_testsuite_log.json
-        path: mreg-cli-master/ci/new_testsuite_log.json
+        path: mreg-cli/ci/new_testsuite_log.json
 
   test-with-curl:
     name: Test with curl
@@ -129,7 +131,7 @@ jobs:
     - name: Wait for mreg to create the database schema and start up
       run: sleep 10s
     - name: Create a user
-      run: docker exec -t mreg /app/manage.py create_mreg_superuser --username test --password test123
+      run: docker exec -t mreg uv run /app/manage.py create_mreg_superuser --username test --password test123
     - name: Authenticate using curl
       shell: bash
       run: |

.github/workflows/docker-image.yml

@@ -43,7 +43,7 @@ jobs:
           --entry-point=bin/mreg-wrapper
           -m ./ci/manifest.scm
       - name: Upload artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: mreg-docker.tar.gz
           path: mreg-docker.tar.gz
@@ -57,7 +57,7 @@ jobs:
       contents: read
     steps:
       - name: Download artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: mreg-docker.tar.gz
       - name: Load image

.github/workflows/test.yml

@@ -8,6 +8,9 @@ on:
     types: [opened, reopened, synchronize]
   workflow_dispatch:
 
+env:
+  UV_FROZEN: 1
+
 name: CI
 jobs:
   test:
@@ -32,56 +35,44 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         python-version:
-          - "3.8"
-          - "3.9"
           - "3.10"
           - "3.11"
           - "3.12"
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Cache pip
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pip
-          key: v1-pip-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements-*.txt') }}
-          restore-keys: |
-            v1-pip-${{ runner.os }}-${{ matrix.python-version }}
-            v1-pip-${{ runner.os }}
-            v1-pip-
-      - name: Install Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
           # Needed to build the native python-ldap extension.
           sudo apt-get update
           sudo apt-get -y install libsasl2-dev libldap2-dev
-          python -m pip install --upgrade pip
-          python -m pip install tox tox-gh-actions
-          python -m pip install -r requirements-test.txt
+          uv sync --group ci
       - name: Test with tox
-        run: tox -r
+        run: uv run tox -r
         env:
           MREG_DB_PASSWORD: postgres
       - name: Check migrations
         run: |
           export MREG_DB_NAME=mreg MREG_DB_USER=mreg MREG_DB_PASSWORD=postgres
-          python manage.py makemigrations --check
-#      - name: Export OpenAPI schema
-#        run: python manage.py generateschema > openapi.yml
-#      - name: Upload OpenAPI schema
-#        if: matrix.python-version == '3.10'
-#        uses: actions/upload-artifact@v3
-#        with:
-#          name: openapi.yml
-#          path: openapi.yml
+          uv run manage.py makemigrations --check
+      # - name: Export OpenAPI schema
+      #   run: uv run manage.py generateschema > openapi.yml
+      # - name: Upload OpenAPI schema
+      #   if: matrix.python-version == '3.10'
+      #   uses: actions/upload-artifact@v4
+      #   with:
+      #     name: openapi.yml
+      #     path: openapi.yml
       - name: Upload coverage
         uses: actions/upload-artifact@v4
         with:
           name: coverage-${{ matrix.python-version }}
           path: .coverage
+          if-no-files-found: error
           include-hidden-files: true
 
   coveralls:
@@ -93,34 +84,26 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         python-version:
-          - "3.8"
-          - "3.9"
           - "3.10"
           - "3.11"
           - "3.12"
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Cache pip
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pip
-          key: v1-pip-${{ runner.os }}-${{ matrix.python-version }}
-          restore-keys: |
-            v1-pip-${{ runner.os }}
-            v1-pip-
-      - name: Install Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
       - name: Download coverage
         uses: actions/download-artifact@v4
         with:
           name: coverage-${{ matrix.python-version }}
       - name: Install Coveralls
-        run: pip install coveralls
+        run: |
+          uv venv --no-project
+          uv pip install coveralls
       - name: Run Coveralls
-        run: coveralls
+        run: uv run --no-project coveralls
         env:
           # Note: Set service name to work around
           # https://github.com/TheKevJames/coveralls-python/issues/252

.gitignore

@@ -30,6 +30,7 @@ wheels/
 .installed.cfg
 *.egg
 MANIFEST
+mreg/_version.py
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

Dockerfile

@@ -5,25 +5,39 @@ ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
 
 RUN apk update
-RUN apk add --virtual build-deps gcc python3-dev openldap-dev musl-dev
-RUN pip install --upgrade pip
-COPY requirements*.txt ./
-RUN pip wheel --no-cache-dir --wheel-dir /usr/src/mreg/wheels -r requirements.txt
+RUN apk add --virtual build-deps gcc python3-dev openldap-dev musl-dev git
+# Copy entire build context to the image.
+# In order to build the project, we need both the .git directory and project files.
+# However, we cannot mix files and directories in the COPY command, because
+# COPY will unpack the contents of source directories into the target directory,
+# and we need to keep the .git directory intact.
+# The workaround is to copy everything, but limit it with .dockerignore.
+COPY . .
+COPY --from=ghcr.io/astral-sh/uv:0.4.30 /uv /uvx /bin/
+RUN  uv venv \
+    && uv sync --frozen --no-dev  \
+    && uv export --no-hashes -o requirements.txt \
+    && uv run python -m ensurepip --upgrade \
+    && uv run python -m pip wheel --no-cache-dir --wheel-dir /usr/src/mreg/wheels -r requirements.txt \
+    && uv build --wheel --out-dir /usr/src/mreg/wheels
+
+ENTRYPOINT [ "/bin/sh" ]
 
 # final stage
 FROM alpine:3.18
 EXPOSE 8000
 
-COPY requirements*.txt entrypoint* manage.py /app/
+COPY entrypoint* manage.py /app/
 COPY mreg /app/mreg/
 COPY mregsite /app/mregsite/
 RUN  mkdir /app/logs
 COPY hostpolicy /app/hostpolicy/
 COPY --from=builder /usr/src/mreg/wheels /wheels
+COPY --from=ghcr.io/astral-sh/uv:0.4.30 /uv /uvx /bin/
 RUN apk update && apk upgrade \
-    && apk add python3 py3-pip libldap vim findutils \
-    && pip install --upgrade pip \
-    && pip install --no-cache /wheels/*
+    && apk add python3 libldap vim findutils \
+    && uv venv \
+    && uv pip install --no-cache /wheels/*
 RUN chmod a+x /app/entrypoint*
 
 CMD /app/entrypoint.sh

README.md

@@ -48,41 +48,50 @@ For a full example, see `docker-compose.yml`.
 
 #### Manually
 
-[!TIP] Depending on your operating system, you may need to install additional packages to get the necessary dependencies for the project. At the very least you will probably require development packages for Python 3.
+> [!TIP] 
+> Depending on your operating system, you may need to install additional packages to get the necessary dependencies for the project. At the very least you will probably require development packages for Python 3.
 
 ##### A step by step
 
-Start by cloning the project from github. You need a terminal, `python3`, and access to a package manager that can install the necessary requirements from `requirements.txt`. We use pip.
+Start by cloning the project from github. You need a terminal and the [uv](https://docs.astral.sh/uv/) package manager.
 
-When you've got your copy of the mreg directory, setup you virtual environment:
+> [!IMPORTANT]  
+> mreg relies on PEP 735 dependency groups for development, which is [not supported by pip](https://github.com/pypa/pip/issues/12963) as of version 24.3.1.
+
+When you've got your copy of the mreg directory, set up the venv and install the dependencies:
 
 ```bash
-> python3 -m venv venv
-> source venv/bin/activate
+> uv sync --frozen
 ```
 
-Then install the required packages:
+<details>
+  <summary>Activating the venv (optional)</summary>
+
+Optionally, you can also activate the created virtual environment. However, we will use `uv run` to run the commands in the virtual environment in this guide, which foregoes the need to activate the environment.
 
 ```bash
-> pip install -r requirements.txt
+. .venv/bin/activate
 ```
 
+Activating the venv allows you to run the commands with `python` instead of `uv run`.
+</details>
+
 Perform database migrations:
 
 ```bash
-> python manage.py migrate
+> uv run manage.py migrate
 ```
 
 Load sample data from fixtures into the now migrated database:
 
 ```bash
-> python manage.py loaddata mreg/fixtures/fixtures.json
+> uv run manage.py loaddata mreg/fixtures/fixtures.json
 ```
 
 And finally, run the server:
 
 ```bash
-> python manage.py runserver
+> uv run manage.py runserver
 ```
 
 You should now be able to open up a browser and go to http://localhost:8000/hosts/ and see
@@ -99,7 +108,7 @@ the returned data.
 To run the tests for the system, simply run
 
 ```bash
-> python manage.py test
+> uv run manage.py test
 ```
 
 ## Local Settings

entrypoint-test.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
 set -e
 cd /app
-./manage.py create_citext_extension --database template1
-./manage.py test --noinput --failfast
+uv run ./manage.py create_citext_extension --database template1
+uv run ./manage.py test --noinput --failfast

entrypoint.sh

@@ -1,9 +1,9 @@
 #!/bin/sh
 set -e
 cd /app
-./manage.py create_citext_extension
-./manage.py migrate
-#./manage.py runserver 0.0.0.0:8000
+uv run ./manage.py create_citext_extension
+uv run ./manage.py migrate
+#uv run ./manage.py runserver 0.0.0.0:8000
 
 # pass signals on to the gunicorn process
 function sigterm()
@@ -14,5 +14,5 @@ function sigterm()
 trap sigterm SIGTERM
 
 # doing it this way to be able to forward signals
-/usr/bin/gunicorn --workers=3 --bind=0.0.0.0 mregsite.wsgi --pid /var/run/gunicorn.pid &
+uv run gunicorn --workers=3 --bind=0.0.0.0 mregsite.wsgi --pid /var/run/gunicorn.pid &
 wait $!

mreg/__about__.py

@@ -0,0 +1,7 @@
+"""Metadata for the mreg package."""
+
+from __future__ import annotations
+
+from importlib.metadata import version
+
+__version__ = version("mreg")