diff --git a/.github/workflows/build-macos-arm64.yaml b/.github/workflows/build-macos-arm64.yaml index dab3b92c98..db6f0c6d1d 100644 --- a/.github/workflows/build-macos-arm64.yaml +++ b/.github/workflows/build-macos-arm64.yaml @@ -340,7 +340,11 @@ jobs: See for more information. - test-annex-more: + test-extra: + # Runs the pytest extra-tests suite in tests/extra/pytest/. Each + # test declares its own skip conditions, so the default is: run on + # every platform, skip only where an individual test's dependencies + # aren't available (e.g. dynlibs needs strace). runs-on: macos-15 needs: build-package steps: @@ -353,7 +357,7 @@ jobs: .github/workflows/tools/set-pr-status \ "${{ github.event.inputs.pr }}" \ macOS ARM64 \ - test-annex-more \ + test-extra \ pending env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -370,17 +374,22 @@ jobs: hdiutil detach /Volumes/git-annex/ echo /Applications/git-annex.app/Contents/MacOS >> "$GITHUB_PATH" - - name: Seek of dynlibs + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Install pytest + run: python -m pip install --upgrade pip pytest + + - name: Configure git identity run: | - mkdir /tmp/testrepo; cd /tmp/testrepo; git init - function nfailed() { - strace -f git-annex "$1" 2>&1 | awk "/$2.*ENOENT/{print}" | tee /dev/fd/2 | wc -l - } - # We should get some reasonable number (not 40) of directories look up for dynamic libraries - liblookups= - PS4='> '; set -x - test $(nfailed version "libpcre.*so") -lt 7 - test $(nfailed init "libpcre.*so") -lt 260 + git config --global user.email "test@github.land" + git config --global user.name "GitHub Almighty" + + - name: Run pytest suite + run: python -m pytest -v tests/extra/pytest/ - name: Set final PR status if: always() && github.event.inputs.pr != '' @@ -388,11 +397,12 @@ jobs: .github/workflows/tools/set-pr-status \ "${{ github.event.inputs.pr }}" \ macOS ARM64 \ - test-annex-more \ + test-extra \ "${{ job.status }}" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + test-datalad: runs-on: macos-15 needs: build-package diff --git a/.github/workflows/build-macos.yaml b/.github/workflows/build-macos.yaml index f0c9659ade..47017c5ef2 100644 --- a/.github/workflows/build-macos.yaml +++ b/.github/workflows/build-macos.yaml @@ -339,7 +339,11 @@ jobs: See for more information. - test-annex-more: + test-extra: + # Runs the pytest extra-tests suite in tests/extra/pytest/. Each + # test declares its own skip conditions, so the default is: run on + # every platform, skip only where an individual test's dependencies + # aren't available (e.g. dynlibs needs strace). runs-on: macos-15-intel needs: build-package steps: @@ -352,7 +356,7 @@ jobs: .github/workflows/tools/set-pr-status \ "${{ github.event.inputs.pr }}" \ macOS \ - test-annex-more \ + test-extra \ pending env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -369,17 +373,22 @@ jobs: hdiutil detach /Volumes/git-annex/ echo /Applications/git-annex.app/Contents/MacOS >> "$GITHUB_PATH" - - name: Seek of dynlibs + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Install pytest + run: python -m pip install --upgrade pip pytest + + - name: Configure git identity run: | - mkdir /tmp/testrepo; cd /tmp/testrepo; git init - function nfailed() { - strace -f git-annex "$1" 2>&1 | awk "/$2.*ENOENT/{print}" | tee /dev/fd/2 | wc -l - } - # We should get some reasonable number (not 40) of directories look up for dynamic libraries - liblookups= - PS4='> '; set -x - test $(nfailed version "libpcre.*so") -lt 7 - test $(nfailed init "libpcre.*so") -lt 260 + git config --global user.email "test@github.land" + git config --global user.name "GitHub Almighty" + + - name: Run pytest suite + run: python -m pytest -v tests/extra/pytest/ - name: Set final PR status if: always() && github.event.inputs.pr != '' @@ -387,11 +396,12 @@ jobs: .github/workflows/tools/set-pr-status \ "${{ github.event.inputs.pr }}" \ macOS \ - test-annex-more \ + test-extra \ "${{ job.status }}" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + test-datalad: runs-on: macos-15-intel needs: build-package diff --git a/.github/workflows/build-ubuntu.yaml b/.github/workflows/build-ubuntu.yaml index b8ee4daf27..c7b3a54fe1 100644 --- a/.github/workflows/build-ubuntu.yaml +++ b/.github/workflows/build-ubuntu.yaml @@ -377,7 +377,11 @@ jobs: See for more information. - test-annex-more: + test-extra: + # Runs the pytest extra-tests suite in tests/extra/pytest/. Each + # test declares its own skip conditions, so the default is: run on + # every platform, skip only where an individual test's dependencies + # aren't available (e.g. dynlibs needs strace). runs-on: ubuntu-24.04 needs: build-package steps: @@ -390,7 +394,7 @@ jobs: .github/workflows/tools/set-pr-status \ "${{ github.event.inputs.pr }}" \ Ubuntu \ - test-annex-more \ + test-extra \ pending env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -404,17 +408,24 @@ jobs: run: | sudo dpkg -i git-annex*.deb - - name: Seek of dynlibs + - name: Install strace + run: sudo apt-get update -qq && sudo apt-get install -y strace + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Install pytest + run: python -m pip install --upgrade pip pytest + + - name: Configure git identity run: | - mkdir /tmp/testrepo; cd /tmp/testrepo; git init - function nfailed() { - strace -f git-annex "$1" 2>&1 | awk "/$2.*ENOENT/{print}" | tee /dev/fd/2 | wc -l - } - # We should get some reasonable number (not 40) of directories look up for dynamic libraries - liblookups= - PS4='> '; set -x - test $(nfailed version "libpcre.*so") -lt 7 - test $(nfailed init "libpcre.*so") -lt 260 + git config --global user.email "test@github.land" + git config --global user.name "GitHub Almighty" + + - name: Run pytest suite + run: python -m pytest -v tests/extra/pytest/ - name: Set final PR status if: always() && github.event.inputs.pr != '' @@ -422,11 +433,12 @@ jobs: .github/workflows/tools/set-pr-status \ "${{ github.event.inputs.pr }}" \ Ubuntu \ - test-annex-more \ + test-extra \ "${{ job.status }}" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + test-datalad: runs-on: ubuntu-24.04 needs: build-package diff --git a/.github/workflows/build-windows.yaml b/.github/workflows/build-windows.yaml index e7e259a44f..dc586cc803 100644 --- a/.github/workflows/build-windows.yaml +++ b/.github/workflows/build-windows.yaml @@ -325,6 +325,67 @@ jobs: See for more information. + test-extra: + # Runs the pytest extra-tests suite in tests/extra/pytest/. Each + # test declares its own skip conditions, so the default is: run on + # every platform, skip only where an individual test's dependencies + # aren't available (e.g. dynlibs needs strace). + runs-on: windows-2025 + needs: build-package + steps: + - name: Checkout this repository + uses: actions/checkout@v6 + + - name: Create pending PR status + if: github.event.inputs.pr != '' + run: | + .github/workflows/tools/set-pr-status \ + "${{ github.event.inputs.pr }}" \ + Windows \ + test-extra \ + pending + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Download git-annex package + uses: actions/download-artifact@v8 + with: + name: git-annex-windows-installer_${{ needs.build-package.outputs.build-version }} + + - name: Install git-annex package + shell: powershell + run: | + ./git-annex-installer_*.exe /S + + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Install pytest + run: python -m pip install --upgrade pip pytest + + - name: Configure git identity + run: | + git config --global user.email "test@github.land" + git config --global user.name "GitHub Almighty" + + - name: Run pytest suite + run: python -m pytest -v tests/extra/pytest/ + + - name: Set final PR status + if: always() && github.event.inputs.pr != '' + run: | + .github/workflows/tools/set-pr-status \ + "${{ github.event.inputs.pr }}" \ + Windows \ + test-extra \ + "${{ job.status }}" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + test-datalad: runs-on: windows-2025 needs: build-package diff --git a/.github/workflows/template/build-{{ostype}}.yaml.j2 b/.github/workflows/template/build-{{ostype}}.yaml.j2 index 8c885f7c62..161a1b7c7f 100644 --- a/.github/workflows/template/build-{{ostype}}.yaml.j2 +++ b/.github/workflows/template/build-{{ostype}}.yaml.j2 @@ -570,8 +570,11 @@ jobs: See for more information. -{% if ostype == "ubuntu" or ostype.startswith("macos") %} - test-annex-more: + test-extra: + # Runs the pytest extra-tests suite in tests/extra/pytest/. Each + # test declares its own skip conditions, so the default is: run on + # every platform, skip only where an individual test's dependencies + # aren't available (e.g. dynlibs needs strace). runs-on: {{runs_on}} needs: build-package steps: @@ -584,7 +587,7 @@ jobs: .github/workflows/tools/set-pr-status \ "${{ github.event.inputs.pr }}" \ {{osname}} \ - test-annex-more \ + test-extra \ pending env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -603,17 +606,26 @@ jobs: {{step}} {% endfor %} - - name: Seek of dynlibs + {% if ostype == "ubuntu" %} + - name: Install strace + run: sudo apt-get update -qq && sudo apt-get install -y strace + {% endif %} + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Install pytest + run: python -m pip install --upgrade pip pytest + + - name: Configure git identity run: | - mkdir /tmp/testrepo; cd /tmp/testrepo; git init - function nfailed() { - strace -f git-annex "$1" 2>&1 | awk "/$2.*ENOENT/{print}" | tee /dev/fd/2 | wc -l - } - # We should get some reasonable number (not 40) of directories look up for dynamic libraries - liblookups= - PS4='> '; set -x - test $(nfailed version "libpcre.*so") -lt 7 - test $(nfailed init "libpcre.*so") -lt 260 + git config --global user.email "test@github.land" + git config --global user.name "GitHub Almighty" + + - name: Run pytest suite + run: python -m pytest -v tests/extra/pytest/ - name: Set final PR status if: always() && github.event.inputs.pr != '' @@ -621,12 +633,12 @@ jobs: .github/workflows/tools/set-pr-status \ "${{ github.event.inputs.pr }}" \ {{osname}} \ - test-annex-more \ + test-extra \ "${{ job.status }}" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -{% endif %} + {% if test_datalad %} test-datalad: runs-on: {{runs_on}} diff --git a/REUSE.toml b/REUSE.toml index 54558d7451..2072c50676 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -13,6 +13,7 @@ path = [ ".github/**", "clients/**", "docs/**", + "tests/**", ] precedence = "aggregate" SPDX-FileCopyrightText = "2020-2026 DataLad Team " diff --git a/setup.cfg b/setup.cfg index 686fdc64d8..4ad063e078 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,9 @@ +[tool:pytest] +# Put tests/ on sys.path so both the conftest and individual test +# modules can import shared helpers via `from _helpers import ...` +# under any --import-mode (prepend, importlib, ...). +pythonpath = tests + [flake8] doctests = True #max-doc-length = 100 diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000000..6c56ff1bd9 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +.pytest_cache/ diff --git a/tests/_helpers.py b/tests/_helpers.py new file mode 100644 index 0000000000..0a7e30990c --- /dev/null +++ b/tests/_helpers.py @@ -0,0 +1,93 @@ +""" +Shared helpers for the tests/ tree. + +Kept as a plain module (not a conftest.py) so both the pytest report +hook in tests/conftest.py and individual test modules can import via +`from _helpers import ...` without relying on pytest's conftest +import-name magic (which broke under `--import-mode=importlib` and +similar). + +pytest puts tests/ on sys.path via the conftest.py at that level, so +`from _helpers import ...` resolves for both callers. +""" + +from __future__ import annotations + +import shutil +import subprocess +from functools import lru_cache + + +@lru_cache(maxsize=1) +def git_annex_version_output() -> str | None: + """Raw stdout of `git annex version`, cached for the session.""" + if shutil.which("git-annex") is None: + return None + try: + return subprocess.run( + ["git", "annex", "version"], + capture_output=True, text=True, timeout=15, check=True, + ).stdout + except (subprocess.SubprocessError, OSError): + return None + + +def git_annex_version() -> str | None: + """ + Bare version string reported by git-annex, e.g. "10.20260421" + (`-g` build suffix stripped). None if git-annex is not + installed / not runnable. + """ + out = git_annex_version_output() + if not out: + return None + for line in out.splitlines(): + if line.startswith("git-annex version:"): + return line.split(":", 1)[1].strip().split("-", 1)[0] + return None + + +def _version_key(s: str) -> tuple[int, ...]: + return tuple(int(p) for p in s.split(".") if p.isdigit()) + + +def git_annex_version_below(threshold: str) -> bool: + """ + True if the installed git-annex version is *strictly* below `threshold`. + Compares as tuples of ints on the "." separator ("10.20220615" < + "10.20260421"). False if git-annex is missing. + """ + v = git_annex_version() + if v is None: + return False + return _version_key(v) < _version_key(threshold) + + +@lru_cache(maxsize=32) +def git_annex_releases_since(threshold: str) -> int | None: + """ + Count git-annex release tags in the current repository that are + strictly newer than `threshold`. Returns None if not inside a git + repo or if there are no matching tags (e.g. a shallow checkout). + """ + if shutil.which("git") is None: + return None + try: + out = subprocess.run( + ["git", "tag", "--list", "10.*"], + capture_output=True, text=True, timeout=10, check=True, + ).stdout + except (subprocess.SubprocessError, OSError): + return None + key = _version_key(threshold) + n = 0 + for tag in out.splitlines(): + tag = tag.strip() + if not tag: + continue + try: + if _version_key(tag) > key: + n += 1 + except ValueError: + continue + return n if n or out.strip() else None diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000000..9471434733 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,99 @@ +""" +Shared pytest configuration for the tests/ tree. + +Lives at tests/ (rather than tests/extra/pytest/) so that +`pytest_report_header` fires regardless of whether the caller runs +`pytest tests`, `pytest tests/extra/pytest`, or a single test file. +pytest loads conftest.py files eagerly along the ancestor chain from +each argument path down to rootdir; a conftest below the given path is +loaded lazily during collection, which is too late for the header. + +Version / release helpers live in tests/_helpers.py so they are +importable both from here and from individual test modules without +relying on pytest's conftest-import magic. +""" + +from __future__ import annotations + +import platform +import shutil +import subprocess + +import pytest + +from _helpers import ( + git_annex_releases_since, + git_annex_version, + git_annex_version_output, +) + + +def _git_annex_summary() -> list[str]: + """Critical fields from `git annex version` (no --json upstream).""" + out = git_annex_version_output() + if out is None: + return ["git-annex: NOT INSTALLED"] + wanted = ( + "git-annex version", + "build flags", + "dependency versions", + "operating system", + "supported repository versions", + ) + lines = [] + for line in out.splitlines(): + key = line.split(":", 1)[0].strip().lower() + if key in wanted: + lines.append(f" {line.strip()}") + return lines + + +def _first_line(cmd: list[str]) -> str | None: + if shutil.which(cmd[0]) is None: + return None + try: + out = subprocess.run( + cmd, capture_output=True, text=True, timeout=15, check=True, + ).stdout + except (subprocess.SubprocessError, OSError): + return None + return out.splitlines()[0].strip() if out.strip() else None + + +def pytest_report_header(config: pytest.Config) -> list[str]: + """Version + tool info at the top of pytest's session banner.""" + tools = [ + ("git", ["git", "--version"]), + ("yt-dlp", ["yt-dlp", "--version"]), + ("youtube-dl", ["youtube-dl", "--version"]), + ("strace", ["strace", "--version"]), + ] + tool_versions = [] + for name, cmd in tools: + first = _first_line(cmd) + tool_versions.append(f"{name}={first if first is not None else '(missing)'}") + + installed = git_annex_version() + if installed is None: + since_line = "git-annex releases newer than installed: (git-annex not installed)" + else: + n_since = git_annex_releases_since(installed) + if n_since is None: + since_line = ( + f"git-annex releases in this repo newer than installed " + f"({installed}): (no tags found)" + ) + else: + since_line = ( + f"git-annex releases in this repo newer than installed " + f"({installed}): {n_since}" + ) + + header = [ + "extra-tests tools: " + ", ".join(tool_versions), + f"platform: {platform.platform()}", + "git-annex:", + *_git_annex_summary(), + since_line, + ] + return header diff --git a/tests/extra/README.md b/tests/extra/README.md new file mode 100644 index 0000000000..b10e2b7e69 --- /dev/null +++ b/tests/extra/README.md @@ -0,0 +1,51 @@ +# Extra tests + +Tests that CI runs on top of `git annex test` and the DataLad test battery. +Each test targets a specific real-world scenario, often a regression that +was seen in the wild and would slip past both upstream test suites. + +Written for **pytest**. Each test declares its own skip conditions +(missing `strace`, unsupported platform, git-annex not on PATH) so the +default is: run everywhere, skip only where the required tool isn't +available. The URL-backend `get` test is `xfail(strict=False)` below +the known-fix version so old git-annex builds do not red the run but a +regression on a fixed build fails loudly. + +## Why pytest and not bats? + +A parallel [Bats](https://bats-core.readthedocs.io/) prototype was +evaluated and dropped. The reasons, briefly: + +- **Cross-platform install cost.** Bats needs three distinct install + recipes (Ubuntu `apt`, macOS `brew`, Windows git-clone bootstrap) + and runs under Git Bash on Windows where `timeout`, `chmod -R u+w` + and `sort -V` behave subtly differently. Pytest is one + `pip install pytest` on all four runners, and Python is already + needed by the `test-datalad` job. +- **No real `xfail` primitive.** Bats only has `skip`, which cannot + distinguish "known-broken on this version, expected to fail" from + "unexpectedly passed, tell me". pytest's + `@pytest.mark.xfail(strict=False)` gives the correct + regression-guard semantics on the URL-backend `get` test. +- **Fixtures and shared helpers.** `conftest.py` gives us cached + `git annex version` parsing, a shared version-reporting hook, + `tmp_path_factory` module-scoped clones, and clean parametrization + — all of which the bats port was re-implementing by hand in + progressively-hairier shell. + +## Tests + +| Test | Purpose | Platforms | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | --------- | +| `dynlibs` | Regression guard on the number of failed dynamic-library lookups (`strace -e ENOENT` on `libpcre.*so`) during `git-annex version` / `init`. | Linux | +| `url_backend` | Regression guard for parsing "odd" URL-backend keys (URL-encoded characters like `&c`, `%%`, `,63v` etc.) on a real DataLad dataset. | All | + +## Running locally + +```bash +python -m pytest -v tests/extra/pytest/ +``` + +Assumes `git-annex` is on `PATH`. The `url_backend` test clones a +small (~18 MB) real DataLad dataset from `datasets.datalad.org`, so +needs network access. diff --git a/tests/extra/pytest/test_dynlibs.py b/tests/extra/pytest/test_dynlibs.py new file mode 100644 index 0000000000..8260c9af7a --- /dev/null +++ b/tests/extra/pytest/test_dynlibs.py @@ -0,0 +1,77 @@ +""" +Guard against regressions in git-annex's dynamic-library lookup behaviour. + +Older git-annex builds probed hundreds of directories for libpcre before +finding it, causing measurable startup slowdowns on some filesystems. +The check is a strace over `git-annex version` / `git-annex init`, +counting ENOENT lookups whose path matches `libpcre.*so`, and asserting +the count stays below a known-reasonable ceiling. + +Linux-only: strace has no cross-platform equivalent that is trivial to +substitute here. +""" + +from __future__ import annotations + +import re +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.skipif( + not sys.platform.startswith("linux") or shutil.which("strace") is None, + reason="strace is Linux-only", +) + + +def _count_enoent(subcommand: str, pattern: str, cwd: Path) -> int: + """ + Return the number of ENOENT lines matching `pattern` under strace. + + Raises on strace failures (bad exit, timeout, empty stderr, or + stderr that lacks any syscall lines). Without these guards a + seccomp-restricted or ptrace_scope-restricted runner would produce + an empty stderr, a count of 0, and a vacuously passing test. + """ + result = subprocess.run( + ["strace", "-f", "git-annex", subcommand], + cwd=cwd, + capture_output=True, + text=True, + timeout=120, + ) + stderr = result.stderr + if result.returncode != 0: + raise RuntimeError( + f"strace exited {result.returncode} for `git-annex {subcommand}`:" + f"\n{stderr[-2000:]}" + ) + # Cheap sanity check: strace always emits at least a "+++ exited" + # and one syscall line if it actually ran. A blocked strace under + # seccomp / ptrace_scope produces essentially nothing on stderr. + if "+++ exited" not in stderr and " ENOENT " not in stderr and " = " not in stderr: + raise RuntimeError( + "strace produced no syscall output; is it blocked by seccomp / " + "ptrace_scope? Cannot trust ENOENT count.\n" + f"stderr head: {stderr[:2000]}" + ) + regex = re.compile(rf"{pattern}.*ENOENT") + matches = [line for line in stderr.splitlines() if regex.search(line)] + for m in matches: + print(m, file=sys.stderr) + return len(matches) + + +def test_libpcre_lookups_on_version(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + n = _count_enoent("version", r"libpcre.*so", tmp_path) + assert n < 7, f"too many libpcre ENOENT lookups on `git-annex version`: {n}" + + +def test_libpcre_lookups_on_init(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + n = _count_enoent("init", r"libpcre.*so", tmp_path) + assert n < 260, f"too many libpcre ENOENT lookups on `git-annex init`: {n}" diff --git a/tests/extra/pytest/test_url_backend.py b/tests/extra/pytest/test_url_backend.py new file mode 100644 index 0000000000..1eb8ca952b --- /dev/null +++ b/tests/extra/pytest/test_url_backend.py @@ -0,0 +1,163 @@ +""" +Regression: older git-annex failed on URL-backend keys whose encoded name +contained characters like `&c`, `%%`, `,63v` (from URL-encoded scheme, +`://`, `?v=`, etc.). The reproducer is a real DataLad dataset that +stores YouTube videos with `yt:` keys. + +Two levels of check on a URL-backend key that decodes to a `yt:` URL: + 1. `git annex whereis` — parses the key and lists its recorded URLs. + This code path was NOT affected by the bug, + and passes on all git-annex versions. + 2. `git annex get` — retrieves the ~18 MB video file, falling + back through the recorded remotes. This + was the affected code path; xfail on + git-annex versions below the fix. + +The DataLad dataset's `origin` remote serves the annex content over +HTTPS, so the retrieval does not require yt-dlp or YouTube access. +""" + +from __future__ import annotations + +import os +import shutil +import stat +import subprocess +import sys +from pathlib import Path +from typing import Any, Callable + +import pytest + +from _helpers import git_annex_version, git_annex_version_below + +# URL_BACKEND_FIX_VERSION is the git-annex release that first shipped +# the fix (upstream commit 8fd9b67ed8 "factor out extendUrlWithPath …", +# 2026-02-16). Older versions xfail so we do not block local dev on +# a known regression while still guaranteeing that once a build is on +# a fixed version, the test acts as a permanent regression guard. +URL_BACKEND_FIX_VERSION = "10.20260420" + +# On CI, forbid xfails: CI runs against a specific build of git-annex, +# and we want every failure — including "known-broken old-version" +# failures — to be loud rather than silently swallowed by an xfail +# marker. Setting condition=False disables the xfail entirely (so a +# failure surfaces as a normal FAIL), independent of the installed +# version. Locally, the version check keeps the marker useful for +# interactive dev on older branches. +_ON_CI = bool(os.environ.get("CI")) + +_xfail_broken_url_backend = pytest.mark.xfail( + condition=(not _ON_CI) and git_annex_version_below(URL_BACKEND_FIX_VERSION), + reason=( + f"URL-encoded-key retrieval bug present in git-annex " + f"< {URL_BACKEND_FIX_VERSION} " + f"(installed: {git_annex_version() or 'unknown'})" + ), + strict=False, +) + +REPRO_URL = "https://datasets.datalad.org/repronim/ReproTube/DataLad/.git/" +TARGET = ( + "videos/2021/07/" + "2021-07-11_Demo-Fully-recomputing-a-real-scientific-paper-DIY/" + "video.mkv" +) + + +def _make_tree_writable(root: Path) -> None: + """ + git-annex sets the key file *and* its containing directory to mode + 0500, which makes both `os.unlink(file)` and `os.rmdir(dir)` fail. + Walk the tree bottom-up and add owner-write to every dir and file + so a subsequent rmtree succeeds. Mirrors what `chmod -R u+w` did + in the dropped bats teardown. + """ + for dirpath, dirnames, filenames in os.walk(root): + for name in (*dirnames, *filenames): + p = os.path.join(dirpath, name) + try: + os.chmod(p, os.stat(p).st_mode | stat.S_IWUSR | stat.S_IRUSR | stat.S_IXUSR) + except OSError: + pass + try: + os.chmod(root, os.stat(root).st_mode | stat.S_IWUSR | stat.S_IRUSR | stat.S_IXUSR) + except OSError: + pass + + +def _chmod_and_retry(func: Callable[..., Any], path: str, _exc: BaseException) -> None: + """rmtree onexc fallback: chmod the file *and its parent dir* writable, retry.""" + for target in (path, os.path.dirname(path)): + try: + os.chmod(target, os.stat(target).st_mode | stat.S_IWUSR | stat.S_IRUSR | stat.S_IXUSR) + except OSError: + pass + func(path) + + +@pytest.fixture(scope="module") +def cloned_repo(tmp_path_factory: pytest.TempPathFactory) -> Path: + workdir = tmp_path_factory.mktemp("ReproTube") + repo = workdir / "DataLad" + # --no-single-branch so we also fetch the git-annex branch, which is + # where URL-backend metadata lives. + subprocess.run( + ["git", "clone", "--depth=1", "--no-single-branch", REPRO_URL, str(repo)], + check=True, + ) + subprocess.run( + ["git", "config", "user.email", "test@github.land"], + cwd=repo, check=True, + ) + subprocess.run( + ["git", "config", "user.name", "GitHub Almighty"], + cwd=repo, check=True, + ) + subprocess.run(["git", "annex", "init", "-q"], cwd=repo, check=True) + yield repo + # Explicit teardown so pytest's later `tmp_path_factory` cleanup + # doesn't trip over git-annex's read-only object files (Windows, + # and also POSIX where the containing key-directory is 0500). + _make_tree_writable(workdir) + # Belt-and-braces: even after the walk, if a race added new + # read-only entries, the onexc handler chmods and retries. Python + # < 3.12 spells the kwarg `onerror`; 3.12+ prefers `onexc`. + if sys.version_info >= (3, 12): + shutil.rmtree(workdir, onexc=_chmod_and_retry) + else: + shutil.rmtree( + workdir, + onerror=lambda f, p, e: _chmod_and_retry(f, p, e[1]), + ) + + +def test_whereis_parses_url_backend_key(cloned_repo: Path) -> None: + """`git annex whereis` on a URL-backend key must list the decoded URL.""" + result = subprocess.run( + ["git", "annex", "whereis", TARGET], + cwd=cloned_repo, + capture_output=True, + text=True, + check=True, + ) + out = result.stdout + # The `,63v,61` chars in the key are the URL-encoded `?v=`; + # git-annex must decode them back to the original YouTube URL. + assert "youtube.com/watch?v=" in out, ( + f"expected decoded youtube URL in `whereis` output; got:\n{out}" + ) + + +@_xfail_broken_url_backend +def test_get_url_backend_key(cloned_repo: Path) -> None: + """Full reproducer: retrieve the URL-backend file.""" + subprocess.run( + ["git", "annex", "get", TARGET], + cwd=cloned_repo, + check=True, + timeout=600, + ) + target = cloned_repo / TARGET + assert target.exists(), f"{TARGET} was not retrieved" + assert target.stat().st_size > 0, f"{TARGET} is empty after get"