Formalize extra CI tests: new tests/extra/ pytest suite + URL-backend regression

.github/workflows/build-macos-arm64.yaml

@@ -340,7 +340,11 @@ jobs:
 
             See <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}> for more information.
 
-  test-annex-more:
+  test-extra:
+    # Runs the pytest extra-tests suite in tests/extra/pytest/.  Each
+    # test declares its own skip conditions, so the default is: run on
+    # every platform, skip only where an individual test's dependencies
+    # aren't available (e.g. dynlibs needs strace).
     runs-on: macos-15
     needs: build-package
     steps:
@@ -353,7 +357,7 @@ jobs:
           .github/workflows/tools/set-pr-status \
             "${{ github.event.inputs.pr }}" \
             macOS ARM64 \
-            test-annex-more \
+            test-extra \
             pending
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -370,29 +374,35 @@ jobs:
           hdiutil detach /Volumes/git-annex/
           echo /Applications/git-annex.app/Contents/MacOS >> "$GITHUB_PATH"
 
-      - name: Seek of dynlibs
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.12'
+
+      - name: Install pytest
+        run: python -m pip install --upgrade pip pytest
+
+      - name: Configure git identity
         run: |
-          mkdir /tmp/testrepo; cd /tmp/testrepo; git init
-          function nfailed() {
-            strace -f git-annex "$1" 2>&1 | awk "/$2.*ENOENT/{print}" | tee /dev/fd/2 | wc -l
-          }
-          # We should get some reasonable number (not 40) of directories look up for dynamic libraries
-          liblookups=
-          PS4='> '; set -x
-          test $(nfailed version "libpcre.*so") -lt 7
-          test $(nfailed init "libpcre.*so") -lt 260
+          git config --global user.email "test@github.land"
+          git config --global user.name "GitHub Almighty"
+
+      - name: Run pytest suite
+        run: python -m pytest -v tests/extra/pytest/
 
       - name: Set final PR status
         if: always() && github.event.inputs.pr != ''
         run: |
           .github/workflows/tools/set-pr-status \
             "${{ github.event.inputs.pr }}" \
             macOS ARM64 \
-            test-annex-more \
+            test-extra \
             "${{ job.status }}"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+
   test-datalad:
     runs-on: macos-15
     needs: build-package

.github/workflows/build-macos.yaml

@@ -339,7 +339,11 @@ jobs:
 
             See <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}> for more information.
 
-  test-annex-more:
+  test-extra:
+    # Runs the pytest extra-tests suite in tests/extra/pytest/.  Each
+    # test declares its own skip conditions, so the default is: run on
+    # every platform, skip only where an individual test's dependencies
+    # aren't available (e.g. dynlibs needs strace).
     runs-on: macos-15-intel
     needs: build-package
     steps:
@@ -352,7 +356,7 @@ jobs:
           .github/workflows/tools/set-pr-status \
             "${{ github.event.inputs.pr }}" \
             macOS \
-            test-annex-more \
+            test-extra \
             pending
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -369,29 +373,35 @@ jobs:
           hdiutil detach /Volumes/git-annex/
           echo /Applications/git-annex.app/Contents/MacOS >> "$GITHUB_PATH"
 
-      - name: Seek of dynlibs
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.12'
+
+      - name: Install pytest
+        run: python -m pip install --upgrade pip pytest
+
+      - name: Configure git identity
         run: |
-          mkdir /tmp/testrepo; cd /tmp/testrepo; git init
-          function nfailed() {
-            strace -f git-annex "$1" 2>&1 | awk "/$2.*ENOENT/{print}" | tee /dev/fd/2 | wc -l
-          }
-          # We should get some reasonable number (not 40) of directories look up for dynamic libraries
-          liblookups=
-          PS4='> '; set -x
-          test $(nfailed version "libpcre.*so") -lt 7
-          test $(nfailed init "libpcre.*so") -lt 260
+          git config --global user.email "test@github.land"
+          git config --global user.name "GitHub Almighty"
+
+      - name: Run pytest suite
+        run: python -m pytest -v tests/extra/pytest/
 
       - name: Set final PR status
         if: always() && github.event.inputs.pr != ''
         run: |
           .github/workflows/tools/set-pr-status \
             "${{ github.event.inputs.pr }}" \
             macOS \
-            test-annex-more \
+            test-extra \
             "${{ job.status }}"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+
   test-datalad:
     runs-on: macos-15-intel
     needs: build-package

.github/workflows/build-ubuntu.yaml

@@ -377,7 +377,11 @@ jobs:
 
             See <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}> for more information.
 
-  test-annex-more:
+  test-extra:
+    # Runs the pytest extra-tests suite in tests/extra/pytest/.  Each
+    # test declares its own skip conditions, so the default is: run on
+    # every platform, skip only where an individual test's dependencies
+    # aren't available (e.g. dynlibs needs strace).
     runs-on: ubuntu-24.04
     needs: build-package
     steps:
@@ -390,7 +394,7 @@ jobs:
           .github/workflows/tools/set-pr-status \
             "${{ github.event.inputs.pr }}" \
             Ubuntu \
-            test-annex-more \
+            test-extra \
             pending
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -404,29 +408,37 @@ jobs:
         run: |
           sudo dpkg -i git-annex*.deb
 
-      - name: Seek of dynlibs
+      - name: Install strace
+        run: sudo apt-get update -qq && sudo apt-get install -y strace
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.12'
+
+      - name: Install pytest
+        run: python -m pip install --upgrade pip pytest
+
+      - name: Configure git identity
         run: |
-          mkdir /tmp/testrepo; cd /tmp/testrepo; git init
-          function nfailed() {
-            strace -f git-annex "$1" 2>&1 | awk "/$2.*ENOENT/{print}" | tee /dev/fd/2 | wc -l
-          }
-          # We should get some reasonable number (not 40) of directories look up for dynamic libraries
-          liblookups=
-          PS4='> '; set -x
-          test $(nfailed version "libpcre.*so") -lt 7
-          test $(nfailed init "libpcre.*so") -lt 260
+          git config --global user.email "test@github.land"
+          git config --global user.name "GitHub Almighty"
+
+      - name: Run pytest suite
+        run: python -m pytest -v tests/extra/pytest/
 
       - name: Set final PR status
         if: always() && github.event.inputs.pr != ''
         run: |
           .github/workflows/tools/set-pr-status \
             "${{ github.event.inputs.pr }}" \
             Ubuntu \
-            test-annex-more \
+            test-extra \
             "${{ job.status }}"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+
   test-datalad:
     runs-on: ubuntu-24.04
     needs: build-package

.github/workflows/build-windows.yaml

@@ -325,6 +325,67 @@ jobs:
 
             See <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}> for more information.
 
+  test-extra:
+    # Runs the pytest extra-tests suite in tests/extra/pytest/.  Each
+    # test declares its own skip conditions, so the default is: run on
+    # every platform, skip only where an individual test's dependencies
+    # aren't available (e.g. dynlibs needs strace).
+    runs-on: windows-2025
+    needs: build-package
+    steps:
+      - name: Checkout this repository
+        uses: actions/checkout@v6
+
+      - name: Create pending PR status
+        if: github.event.inputs.pr != ''
+        run: |
+          .github/workflows/tools/set-pr-status \
+            "${{ github.event.inputs.pr }}" \
+            Windows \
+            test-extra \
+            pending
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Download git-annex package
+        uses: actions/download-artifact@v8
+        with:
+          name: git-annex-windows-installer_${{ needs.build-package.outputs.build-version }}
+
+      - name: Install git-annex package
+        shell: powershell
+        run: |
+          ./git-annex-installer_*.exe /S
+
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.12'
+
+      - name: Install pytest
+        run: python -m pip install --upgrade pip pytest
+
+      - name: Configure git identity
+        run: |
+          git config --global user.email "test@github.land"
+          git config --global user.name "GitHub Almighty"
+
+      - name: Run pytest suite
+        run: python -m pytest -v tests/extra/pytest/
+
+      - name: Set final PR status
+        if: always() && github.event.inputs.pr != ''
+        run: |
+          .github/workflows/tools/set-pr-status \
+            "${{ github.event.inputs.pr }}" \
+            Windows \
+            test-extra \
+            "${{ job.status }}"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+
   test-datalad:
     runs-on: windows-2025
     needs: build-package

.github/workflows/template/build-{{ostype}}.yaml.j2

@@ -570,8 +570,11 @@ jobs:
 
             See <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}> for more information.
 
-{% if ostype == "ubuntu" or ostype.startswith("macos") %}
-  test-annex-more:
+  test-extra:
+    # Runs the pytest extra-tests suite in tests/extra/pytest/.  Each
+    # test declares its own skip conditions, so the default is: run on
+    # every platform, skip only where an individual test's dependencies
+    # aren't available (e.g. dynlibs needs strace).
     runs-on: {{runs_on}}
     needs: build-package
     steps:
@@ -584,7 +587,7 @@ jobs:
           .github/workflows/tools/set-pr-status \
             "${{ github.event.inputs.pr }}" \
             {{osname}} \
-            test-annex-more \
+            test-extra \
             pending
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -603,30 +606,39 @@ jobs:
           {{step}}
           {% endfor %}
 
-      - name: Seek of dynlibs
+    {% if ostype == "ubuntu" %}
+      - name: Install strace
+        run: sudo apt-get update -qq && sudo apt-get install -y strace
+    {% endif %}
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.12'
+
+      - name: Install pytest
+        run: python -m pip install --upgrade pip pytest
+
+      - name: Configure git identity
         run: |
-          mkdir /tmp/testrepo; cd /tmp/testrepo; git init
-          function nfailed() {
-            strace -f git-annex "$1" 2>&1 | awk "/$2.*ENOENT/{print}" | tee /dev/fd/2 | wc -l
-          }
-          # We should get some reasonable number (not 40) of directories look up for dynamic libraries
-          liblookups=
-          PS4='> '; set -x
-          test $(nfailed version "libpcre.*so") -lt 7
-          test $(nfailed init "libpcre.*so") -lt 260
+          git config --global user.email "test@github.land"
+          git config --global user.name "GitHub Almighty"
+
+      - name: Run pytest suite
+        run: python -m pytest -v tests/extra/pytest/
 
       - name: Set final PR status
         if: always() && github.event.inputs.pr != ''
         run: |
           .github/workflows/tools/set-pr-status \
             "${{ github.event.inputs.pr }}" \
             {{osname}} \
-            test-annex-more \
+            test-extra \
             "${{ job.status }}"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-{% endif %}
+
 {% if test_datalad %}
   test-datalad:
     runs-on: {{runs_on}}

REUSE.toml

@@ -13,6 +13,7 @@ path = [
   ".github/**",
   "clients/**",
   "docs/**",
+  "tests/**",
 ]
 precedence = "aggregate"
 SPDX-FileCopyrightText = "2020-2026 DataLad Team <team@datalad.org>"

setup.cfg

@@ -1,3 +1,9 @@
+[tool:pytest]
+# Put tests/ on sys.path so both the conftest and individual test
+# modules can import shared helpers via `from _helpers import ...`
+# under any --import-mode (prepend, importlib, ...).
+pythonpath = tests
+
 [flake8]
 doctests = True
 #max-doc-length = 100

tests/.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+.pytest_cache/