Merge pull request #306 from SciCatProject/better-task-runner

Manage project with Just and uv

Author: jl-wynen

.github/workflows/ci.yml

@@ -13,10 +13,16 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-      - run: python -m pip install --upgrade pip
-      - run: python -m pip install -r requirements/ci.txt
-      - run: tox -e static
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+      - name: Install Just
+        uses: extractions/setup-just@v3
+      - name: Install pre-commit
+        run: uv tool install pre-commit --with pre-commit-uv
+      # The first run of pre-commit may reformat files. If this happens, it returns 1 but this
+      # should not fail the job. So just run again if it fails. A second failure means that
+      # either the different formatters can't agree on a format or that static analysis failed.
+      - run: pre-commit run -a || pre-commit run -a
       - uses: stefanzweifel/git-auto-commit-action@v6
         with:
           commit_message: Apply automatic formatting
@@ -27,30 +33,33 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
         with:
-          python-version: 3.11
-      - run: python -m pip install --upgrade pip
-      - run: python -m pip install -r requirements/ci.txt
-      - run: tox -e mypy
+          entrypoint: "3.11" # TODO remove once we drop Python 3.10
+      # TODO remove once we drop Python 3.10
+      - run: uv python pin 3.11
+      - name: Install Just
+        uses: extractions/setup-just@v3
+      - run: just mypy
 
   tests:
     name: Tests
     needs: formatting
     strategy:
       matrix:
         include:
-          - {python: '3.13', os: ubuntu-24.04, tox: py313-full}
-          - {python: '3.12', os: ubuntu-24.04, tox: py312-full}
-          - {python: '3.11', os: ubuntu-24.04, tox: py311-full}
-          - {python: '3.10', os: ubuntu-24.04, tox: py310-full}
-          - {python: '3.10', os: macos-14, tox: py310}
-          - {python: '3.10', os: windows-2022, tox: py310}
+          - {python: '3.13', os: ubuntu-24.04, command: test-all}
+          - {python: '3.12', os: ubuntu-24.04, command: test-all}
+          - {python: '3.11', os: ubuntu-24.04, command: test-all}
+          - {python: '3.10', os: ubuntu-24.04, command: test-all}
+          - {python: '3.10', os: macos-14, command: test}
+          - {python: '3.10', os: windows-2022, command: test}
     uses: ./.github/workflows/test.yml
     with:
       os-variant: ${{ matrix.os }}
       python-version: ${{ matrix.python }}
-      tox-env: ${{ matrix.tox }}
+      command: ${{ matrix.command }}
     secrets: inherit
 
   docs:

.github/workflows/docs.yml

@@ -22,13 +22,14 @@ jobs:
     steps:
       - run: sudo apt install --yes pandoc
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
         with:
-          python-version: 3.11
-      - run: python -m pip install --upgrade pip
-      - run: python -m pip install -r requirements/ci.txt
-      - run: tox -e docs
-      - run: touch html/.nojekyll
+          entrypoint: "3.11" # TODO remove once we drop Python 3.10
+      - name: Install Just
+        uses: extractions/setup-just@v3
+      - name: Build and test docs
+        run: just docs
       - uses: actions/upload-pages-artifact@v3
         name: Upload documentation as artifact
         with:

.github/workflows/release.yml

@@ -15,12 +15,13 @@ jobs:
         with:
           fetch-depth: 0  # history required so setuptools_scm can determine version
 
-      - uses: actions/setup-python@v5
-      - run: python -m pip install --upgrade pip
-      - run: python -m pip install -r requirements/wheels.txt
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+      - name: Install Just
+        uses: extractions/setup-just@v3
 
       - name: Build wheels
-        run: python -m build
+        run: just build
 
       - name: Upload wheels
         uses: actions/upload-artifact@v4

.github/workflows/test.yml

@@ -10,9 +10,9 @@ on:
       python-version:
         type: string
         description: 'Python interpreter version'
-      tox-env:
+      command:
         type: string
-        description: 'Tox environment to run'
+        description: 'Test command to run'
       checkout-ref:
         default: ''
         type: string
@@ -28,7 +28,7 @@ on:
         type: string
       python-version:
         type: string
-      tox-env:
+      command:
         default: 'test'
         type: string
       checkout-ref:
@@ -40,20 +40,21 @@ on:
 
 jobs:
   tests:
-    name: Tests ${{ inputs.os-variant }} ${{ inputs.tox-env }}
+    name: Tests ${{ inputs.os-variant }} ${{ inputs.command }}
     runs-on: ${{ inputs.os-variant }}
     steps:
       - run: sudo apt install --yes docker-compose
         if: ${{ contains(inputs.os-variant, 'ubuntu') }}
       - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.checkout-ref }}
-      - uses: actions/setup-python@v5
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
         with:
-          python-version: ${{ inputs.python-version }}
-      - run: python -m pip install --upgrade pip
-      - run: python -m pip install -r requirements/ci.txt
-      - run: tox -e ${{ inputs.tox-env }}
+          entrypoint: "${{ inputs.python-version }}"
+      - name: Install Just
+        uses: extractions/setup-just@v3
+      - run: just ${{ inputs.command }}
         if: ${{ !inputs.backend-version }}
-      - run: tox -e ${{ inputs.tox-env }} -- --scitacean-backend-version=${{ inputs.backend-version }}
+      - run: just ${{ inputs.command }} --scitacean-backend-version=${{ inputs.backend-version }}
         if: ${{ inputs.backend-version }}

.gitignore

@@ -26,6 +26,3 @@ __pycache__/
 Pipfile
 venv
 .venv
-
-# We don't use uv for project management
-uv.lock

.pre-commit-config.yaml

@@ -1,48 +1,70 @@
 repos:
-- repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v5.0.0
-  hooks:
-    - id: check-added-large-files
-    - id: check-case-conflict
-    - id: check-illegal-windows-names
-    - id: check-json
-    - id: check-merge-conflict
-    - id: check-toml
-    - id: check-yaml
-    - id: detect-private-key
-    - id: end-of-file-fixer
-    - id: trailing-whitespace
-      args: [--markdown-linebreak-ext=md]
-- repo: https://github.com/kynan/nbstripout
-  rev: 0.8.1
-  hooks:
-    - id: nbstripout
-      types: ["jupyter"]
-      args: ["--drop-empty-cells",
-             "--extra-keys 'metadata.language_info.version cell.metadata.jp-MarkdownHeadingCollapsed cell.metadata.pycharm'"]
-- repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.12.0
-  hooks:
-    - id: ruff-format
-      types_or: [ python, pyi ]
-    - id: ruff
-      args: [--fix, --exit-non-zero-on-fix]
-      types_or: [ python, pyi, jupyter ]
-- repo: https://github.com/codespell-project/codespell
-  rev: v2.4.1
-  hooks:
-    - id: codespell
-      additional_dependencies:
-        - tomli
-- repo: https://github.com/pre-commit/pygrep-hooks
-  rev: v1.10.0
-  hooks:
-    - id: python-use-type-annotations
-    - id: rst-backticks
-    - id: rst-directive-colons
-    - id: rst-inline-touching-normal
-    - id: text-unicode-replacement-char
-- repo: https://github.com/rhysd/actionlint
-  rev: v1.7.7
-  hooks:
-    - id: actionlint
+  - repo: local
+    hooks:
+      - id: uv-lock
+        name: lock dependencies
+        entry: just lock
+        language: system
+        files: ^(uv\.lock|pyproject\.toml|uv\.toml)$
+        pass_filenames: false
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: check-added-large-files
+      - id: check-case-conflict
+      - id: check-illegal-windows-names
+      - id: check-json
+      - id: check-merge-conflict
+      - id: check-toml
+      - id: check-yaml
+      - id: detect-private-key
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+        args: [--markdown-linebreak-ext=md]
+
+  - repo: local
+    hooks:
+      - id: nbstripout
+        name: strip notebook outputs
+        entry: just strip-output
+        language: system
+        types_or: ["jupyter"]
+        require_serial: true
+      - id: format-python
+        name: format python code
+        entry: just format-python
+        language: system
+        types_or: ["python", "pyi"]
+        require_serial: true
+      - id: format-md
+        name: format markdown
+        entry: just format-md
+        language: system
+        types: [file, markdown]
+
+      - id: lint-python
+        name: lint python code
+        entry: just lint-python
+        language: system
+        types_or: ["python", "pyi", "jupyter"]
+        require_serial: true
+      - id: spelling
+        name: check spelling
+        entry: just spell
+        language: system
+        types: [text]
+
+  - repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.10.0
+    hooks:
+      - id: python-use-type-annotations
+      - id: rst-backticks
+      - id: rst-directive-colons
+      - id: rst-inline-touching-normal
+      - id: text-unicode-replacement-char
+
+  - repo: https://github.com/rhysd/actionlint
+    rev: v1.7.7
+    hooks:
+      - id: actionlint

CODE_OF_CONDUCT.md

@@ -17,23 +17,23 @@ diverse, inclusive, and healthy community.
 Examples of behavior that contributes to a positive environment for our
 community include:
 
-* Demonstrating empathy and kindness toward other people
-* Being respectful of differing opinions, viewpoints, and experiences
-* Giving and gracefully accepting constructive feedback
-* Accepting responsibility and apologizing to those affected by our mistakes,
+- Demonstrating empathy and kindness toward other people
+- Being respectful of differing opinions, viewpoints, and experiences
+- Giving and gracefully accepting constructive feedback
+- Accepting responsibility and apologizing to those affected by our mistakes,
   and learning from the experience
-* Focusing on what is best not just for us as individuals, but for the
+- Focusing on what is best not just for us as individuals, but for the
   overall community
 
 Examples of unacceptable behavior include:
 
-* The use of sexualized language or imagery, and sexual attention or
+- The use of sexualized language or imagery, and sexual attention or
   advances of any kind
-* Trolling, insulting or derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or email
+- Trolling, insulting or derogatory comments, and personal or political attacks
+- Public or private harassment
+- Publishing others' private information, such as a physical or email
   address, without their explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
+- Other conduct which could reasonably be considered inappropriate in a
   professional setting
 
 ## Enforcement Responsibilities
@@ -106,7 +106,7 @@ Violating these terms may lead to a permanent ban.
 ### 4. Permanent Ban
 
 **Community Impact**: Demonstrating a pattern of violation of community
-standards, including sustained inappropriate behavior,  harassment of an
+standards, including sustained inappropriate behavior, harassment of an
 individual, or aggression toward or disparagement of classes of individuals.
 
 **Consequence**: A permanent ban from any sort of public interaction within
@@ -121,8 +121,8 @@ https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
 Community Impact Guidelines were inspired by [Mozilla's code of conduct
 enforcement ladder](https://github.com/mozilla/diversity).
 
-[homepage]: https://www.contributor-covenant.org
-
 For answers to common questions about this code of conduct, see the FAQ at
 https://www.contributor-covenant.org/faq. Translations are available at
 https://www.contributor-covenant.org/translations.
+
+[homepage]: https://www.contributor-covenant.org

SECURITY.md

@@ -7,6 +7,7 @@ Only the latest version is supported.
 ## Reporting a Vulnerability
 
 If you believe you have found a security vulnerability in Scitacean, please
+
 - ✅ report it to us by creating a [security advisory](https://github.com/SciCatProject/scitacean/security/advisories/new).
 - ❌ do not report security vulnerabilities through public GitHub issues, discussions, or pull requests, etc.
 

docs/conf.py

@@ -192,6 +192,8 @@
 linkcheck_ignore = [
     # Specific lines in GitHub blobs cannot be found by linkcheck.
     r"https?://github\.com/.*?/blob/[a-f0-9]+/.+?#",
+    # These hit a rate limit and are very slow.
+    r"https?://github\.com/.*?/blob/",
 ]
 
 # -- Options for autodoc_pydantic -----------------------------------------

docs/developer/dependency-management.rst

@@ -2,15 +2,16 @@ Dependency management
 =====================
 
 Scitacean is a library, so the package dependencies are never pinned.
-But lower bounds are fine and individual versions can be excluded.
+But (correct) lower bounds are encouraged and individual versions can be excluded.
 See, e.g., `Should You Use Upper Bound Version Constraints <https://iscinumpy.dev/post/bound-version-constraints/>`_ for an explanation.
 
 Development dependencies [#0]_ are pinned to an exact version in order to ensure reproducibility.
-This is done by specifying packages and version constraints in ``requirements/*.in`` files and locking those dependencies using `pip-compile-multi <https://pip-compile-multi.readthedocs.io/en/latest/index.html>`_ to produce ``requirements/*.txt`` files.
-Those files are then used by `tox <https://tox.wiki/en/latest/>`_ to create isolated environments and run tests, build docs, etc.
+This is done automatically by `uv <https://docs.astral.sh/uv/>`_ whenever ``pyproject.toml`` is updated.
+Pins can be updated explicitly using
 
-tox can be cumbersome to use for local development.
-So ``requirements/dev.txt`` can be used to create a virtual environment with all dependencies.
+.. code-block:: bash
+
+    just lock
 
 .. rubric:: Footnotes