From 9a11d1b987530e4723b79c71ad94942956f72f7e Mon Sep 17 00:00:00 2001 From: Subhashree Radhakrishnan Date: Fri, 17 Jul 2026 05:31:59 +0000 Subject: [PATCH 1/4] feat(benchmark): pre-bake hermes agent into dataset images for closed-book runs Closed-book Harbor runs cannot curl-install agents at task time. Bake the NousResearch hermes-agent into the dataset image install layer (pinned via HERMES_VERSION in benchmark/agent-versions.env) and add an install-skip probe to the harbor hermes agent patch so the runtime install() short-circuits when hermes is already present. The agent patch also threads a custom OpenAI-compatible base_url (e.g. a Switchyard gateway) into hermes' config.yaml, which hermes requires for a custom chat-model endpoint (OPENAI_BASE_URL alone is ignored for the chat model). Co-Authored-By: Claude Fable 5 Signed-off-by: Subhashree Radhakrishnan --- benchmark/agent-versions.env | 3 + benchmark/patches/harbor-agent-patches.diff | 102 ++++++++++++++++++++ benchmark/prepare_harbor_dataset.py | 24 ++++- 3 files changed, 128 insertions(+), 1 deletion(-) diff --git a/benchmark/agent-versions.env b/benchmark/agent-versions.env index 843959969..f356b7858 100644 --- a/benchmark/agent-versions.env +++ b/benchmark/agent-versions.env @@ -5,3 +5,6 @@ CLAUDE_CODE_VERSION=2.1.211 CODEX_VERSION=0.144.5 OPENCODE_VERSION=1.18.3 NODE_VERSION=20.11.1 +# Hermes (NousResearch hermes-agent), installed from GitHub at dataset-bake time. +# "main" tracks latest; set to a tag/commit to pin for reproducibility. +HERMES_VERSION=main diff --git a/benchmark/patches/harbor-agent-patches.diff b/benchmark/patches/harbor-agent-patches.diff index bc62838d1..54454cd68 100644 --- a/benchmark/patches/harbor-agent-patches.diff +++ b/benchmark/patches/harbor-agent-patches.diff @@ -426,3 +426,105 @@ +++ b/harbor/switchyard_patch_id.txt @@ -0,0 +1 @@ +switchyard-harbor-patches-2026-05-22-v2 + +--- a/harbor/agents/installed/hermes.py ++++ b/harbor/agents/installed/hermes.py +@@ -59,6 +59,16 @@ + return 'export PATH="$HOME/.local/bin:$PATH"; hermes version' + + async def install(self, environment: BaseEnvironment) -> None: ++ # Skip if hermes is already installed (e.g. from the dataset ++ # agent-bake layer). hermes installs to $HOME/.local/bin, so export ++ # that path before probing. Required for closed-book runs where the ++ # task-time curl install cannot reach github/pypi. ++ probe = await self.exec_as_agent( ++ environment, ++ command='export PATH="$HOME/.local/bin:$PATH"; command -v hermes >/dev/null 2>&1 && echo present || true', ++ ) ++ if "present" in (getattr(probe, "stdout", "") or ""): ++ return + await self.exec_as_root( + environment, + command="apt-get update && apt-get install -y curl git ripgrep xz-utils", +@@ -92,11 +92,31 @@ + # ------------------------------------------------------------------ + + @staticmethod +- def _build_config_yaml(model: str) -> str: +- """Generate a hermes config.yaml with full capabilities enabled.""" ++ def _build_config_yaml( ++ model: str, base_url: str | None = None, api_key: str | None = None ++ ) -> str: ++ """Generate a hermes config.yaml with full capabilities enabled. ++ ++ When ``base_url`` is given, route the model through a custom ++ OpenAI-compatible endpoint (e.g. a Switchyard gateway): hermes only ++ honors a custom endpoint via its config.yaml ``model.provider=custom`` + ++ ``model.base_url`` (the ``OPENAI_BASE_URL`` env var is ignored for the ++ chat model), so we nest the provider/base_url/api_key under ``model``. ++ """ ++ model_field: Any ++ if base_url: ++ model_field = { ++ "default": model, ++ "provider": "custom", ++ "base_url": base_url, ++ } ++ if api_key: ++ model_field["api_key"] = api_key ++ else: ++ model_field = model + config: dict[str, Any] = { +- "model": model, +- "provider": "auto", ++ "model": model_field, ++ **({} if base_url else {"provider": "auto"}), + "toolsets": ["hermes-cli"], + "agent": {"max_turns": 90}, + "memory": { +@@ -357,6 +377,9 @@ + # Try native provider key first, fall back to OpenRouter. + hermes_provider_flag: str | None = None + use_native = False ++ # Custom OpenAI-compatible endpoint (e.g. Switchyard) for the chat model. ++ custom_base_url: str | None = None ++ custom_api_key: str | None = None + + if provider in _NATIVE_PROVIDERS: + native_flag, key_names = _NATIVE_PROVIDERS[provider] +@@ -367,11 +390,15 @@ + hermes_provider_flag = native_flag + use_native = True + break +- # Forward OPENAI_BASE_URL when using native OpenAI key ++ # Forward OPENAI_BASE_URL when using native OpenAI key. Hermes ignores ++ # this env var for the chat model, so also thread it into config.yaml ++ # as a custom provider (see _build_config_yaml). + if use_native and provider == "openai": + base_url = os.environ.get("OPENAI_BASE_URL") + if base_url: + env["OPENAI_BASE_URL"] = base_url ++ custom_base_url = base_url ++ custom_api_key = os.environ.get("OPENAI_API_KEY") + + if not use_native: + openrouter_key = os.environ.get("OPENROUTER_API_KEY") +@@ -386,9 +413,15 @@ + env["OPENROUTER_API_KEY"] = openrouter_key + + # Native providers with --provider flag use just the model name; +- # everything else (OpenRouter, openai direct) uses provider/model. +- cli_model = model if hermes_provider_flag else self.model_name +- config_yaml = self._build_config_yaml(cli_model) ++ # a custom OpenAI-compatible endpoint also uses the bare model (the ++ # base_url already targets the gateway); everything else uses provider/model. ++ if custom_base_url: ++ cli_model = model ++ else: ++ cli_model = model if hermes_provider_flag else self.model_name ++ config_yaml = self._build_config_yaml( ++ cli_model, base_url=custom_base_url, api_key=custom_api_key ++ ) + + # Pass instruction via env var (safe from shell escaping issues) + env["HARBOR_INSTRUCTION"] = instruction diff --git a/benchmark/prepare_harbor_dataset.py b/benchmark/prepare_harbor_dataset.py index fb30a8ce3..14719feb6 100644 --- a/benchmark/prepare_harbor_dataset.py +++ b/benchmark/prepare_harbor_dataset.py @@ -187,10 +187,19 @@ def _install_layer(pins: dict[str, str]) -> str: claude_version = pins["CLAUDE_CODE_VERSION"] codex_version = pins["CODEX_VERSION"] opencode_version = pins["OPENCODE_VERSION"] + # Hermes (NousResearch hermes-agent) is a per-user uv app installed from + # GitHub, not an npm package. Default to main; pin a tag/commit via + # HERMES_VERSION for reproducibility. Baking it here (build-time, with host + # network) means the runtime install() skip-guard short-circuits, so tasks + # need no egress for it — enabling closed-book Hermes runs. + hermes_version = pins.get("HERMES_VERSION", "main") + hermes_branch_flag = ( + f" --branch {hermes_version}" if hermes_version and hermes_version != "main" else "" + ) return f""" # Switchyard benchmark prebaked coding agents. -ENV SWITCHYARD_PREBAKED_AGENT_VERSIONS="claude-code={claude_version},codex={codex_version},opencode={opencode_version},node={node_version}" +ENV SWITCHYARD_PREBAKED_AGENT_VERSIONS="claude-code={claude_version},codex={codex_version},opencode={opencode_version},node={node_version},hermes={hermes_version}" RUN set -eux; \\ if command -v apt-get >/dev/null 2>&1; then \\ apt-get update; \\ @@ -231,6 +240,19 @@ def _install_layer(pins: dict[str, str]) -> str: claude --version; \\ codex --version; \\ opencode --version +RUN set -eux; \\ + export HOME=/root; \\ + export PATH="/root/.local/bin:$PATH"; \\ + if command -v apt-get >/dev/null 2>&1; then \\ + apt-get update; \\ + apt-get install -y --no-install-recommends git ripgrep xz-utils; \\ + rm -rf /var/lib/apt/lists/*; \\ + elif command -v apk >/dev/null 2>&1; then \\ + apk add --no-cache git ripgrep xz; \\ + fi; \\ + curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.sh \\ + | bash -s -- --skip-setup{hermes_branch_flag}; \\ + hermes version """ From 9ccc416d392a09f58c73bf0b36517d4b8a508360 Mon Sep 17 00:00:00 2001 From: Giedrius Burachas Date: Thu, 30 Jul 2026 15:33:32 +0000 Subject: [PATCH 2/4] test(benchmark): record hermes pin in dataset manifest expectation Signed-off-by: Giedrius Burachas --- tests/test_prepare_harbor_dataset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_prepare_harbor_dataset.py b/tests/test_prepare_harbor_dataset.py index 1c69052ff..14493fc02 100644 --- a/tests/test_prepare_harbor_dataset.py +++ b/tests/test_prepare_harbor_dataset.py @@ -284,6 +284,7 @@ def test_generated_dataset_manifest_records_pins_tasks_and_digests(tmp_path: Pat assert manifest["agent_versions"] == { "CLAUDE_CODE_VERSION": "2.1.211", "CODEX_VERSION": "0.144.5", + "HERMES_VERSION": "main", "NODE_VERSION": "20.11.1", "OPENCODE_VERSION": "1.18.3", } From 4fcdde205713651afcd4bac435fe3e39af6dd5c0 Mon Sep 17 00:00:00 2001 From: Giedrius Burachas Date: Tue, 4 Aug 2026 18:18:43 +0000 Subject: [PATCH 3/4] fix(benchmark): pin hermes to an immutable ref and fetch its installer there Signed-off-by: Giedrius Burachas --- benchmark/agent-versions.env | 7 +++-- benchmark/prepare_harbor_dataset.py | 28 +++++++++++++------- tests/test_prepare_harbor_dataset.py | 39 +++++++++++++++++++++++++++- 3 files changed, 61 insertions(+), 13 deletions(-) diff --git a/benchmark/agent-versions.env b/benchmark/agent-versions.env index f356b7858..da2a84ea8 100644 --- a/benchmark/agent-versions.env +++ b/benchmark/agent-versions.env @@ -6,5 +6,8 @@ CODEX_VERSION=0.144.5 OPENCODE_VERSION=1.18.3 NODE_VERSION=20.11.1 # Hermes (NousResearch hermes-agent), installed from GitHub at dataset-bake time. -# "main" tracks latest; set to a tag/commit to pin for reproducibility. -HERMES_VERSION=main +# Must be an immutable ref — a tag or commit SHA. A moving ref such as "main" is +# rejected at build time: the run manifest records this value as a pin, and two +# builds recording the same string while installing different code is worse than +# recording nothing. The installer script is fetched from this ref too. +HERMES_VERSION=v2026.8.3 diff --git a/benchmark/prepare_harbor_dataset.py b/benchmark/prepare_harbor_dataset.py index 14719feb6..371d991f2 100644 --- a/benchmark/prepare_harbor_dataset.py +++ b/benchmark/prepare_harbor_dataset.py @@ -188,14 +188,22 @@ def _install_layer(pins: dict[str, str]) -> str: codex_version = pins["CODEX_VERSION"] opencode_version = pins["OPENCODE_VERSION"] # Hermes (NousResearch hermes-agent) is a per-user uv app installed from - # GitHub, not an npm package. Default to main; pin a tag/commit via - # HERMES_VERSION for reproducibility. Baking it here (build-time, with host - # network) means the runtime install() skip-guard short-circuits, so tasks - # need no egress for it — enabling closed-book Hermes runs. - hermes_version = pins.get("HERMES_VERSION", "main") - hermes_branch_flag = ( - f" --branch {hermes_version}" if hermes_version and hermes_version != "main" else "" - ) + # GitHub, not an npm package. Baking it here (build-time, with host network) + # means the runtime install() skip-guard short-circuits, so tasks need no + # egress for it — enabling closed-book Hermes runs. + # + # HERMES_VERSION must name an immutable ref. A moving ref would let two builds + # record the same version string while installing different code, which is worse + # than not recording it: the manifest would assert a reproducibility it does not + # have. The installer script is fetched from the same ref for the same reason — + # pinning the agent but running whatever installer main has today reintroduces + # exactly the drift the pin exists to prevent. + hermes_version = pins["HERMES_VERSION"] + if hermes_version in ("main", "master", "HEAD"): + raise SystemExit( + f"HERMES_VERSION={hermes_version!r} is a moving ref and cannot be recorded " + "as a reproducible pin; use a tag or commit SHA" + ) return f""" # Switchyard benchmark prebaked coding agents. @@ -250,8 +258,8 @@ def _install_layer(pins: dict[str, str]) -> str: elif command -v apk >/dev/null 2>&1; then \\ apk add --no-cache git ripgrep xz; \\ fi; \\ - curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.sh \\ - | bash -s -- --skip-setup{hermes_branch_flag}; \\ + curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/{hermes_version}/scripts/install.sh \\ + | bash -s -- --skip-setup --branch {hermes_version}; \\ hermes version """ diff --git a/tests/test_prepare_harbor_dataset.py b/tests/test_prepare_harbor_dataset.py index 14493fc02..4b1e30e3f 100644 --- a/tests/test_prepare_harbor_dataset.py +++ b/tests/test_prepare_harbor_dataset.py @@ -8,6 +8,7 @@ from pathlib import Path from types import ModuleType +import pytest import yaml REPO = Path(__file__).resolve().parents[1] @@ -284,7 +285,7 @@ def test_generated_dataset_manifest_records_pins_tasks_and_digests(tmp_path: Pat assert manifest["agent_versions"] == { "CLAUDE_CODE_VERSION": "2.1.211", "CODEX_VERSION": "0.144.5", - "HERMES_VERSION": "main", + "HERMES_VERSION": "v2026.8.3", "NODE_VERSION": "20.11.1", "OPENCODE_VERSION": "1.18.3", } @@ -307,3 +308,39 @@ def test_generated_compose_bakes_task_id_into_proxy_env(tmp_path: Path) -> None: proxy_env = "\n".join(compose["services"]["proxy"]["environment"]) assert "SWITCHYARD_TASK_ID=task-id-check" in proxy_env assert "SWITCHYARD_TRIAL_DIR=${HOST_AGENT_LOGS_PATH:-}" in proxy_env + +def test_a_moving_hermes_ref_is_rejected() -> None: + """A moving ref cannot be recorded as a pin. + + The dataset manifest presents HERMES_VERSION as a reproducibility guarantee. Two + builds recording the same string while installing different Hermes code is worse + than recording nothing, so the build fails rather than asserting a pin it does not + have. + """ + base = { + "CLAUDE_CODE_VERSION": "1", + "CODEX_VERSION": "2", + "OPENCODE_VERSION": "3", + "NODE_VERSION": "4", + } + for moving in ("main", "master", "HEAD"): + with pytest.raises(SystemExit, match="moving ref"): + _load_generator_module()._install_layer({**base, "HERMES_VERSION": moving}) + + +def test_the_hermes_installer_is_fetched_at_the_pinned_ref() -> None: + """Pinning the agent but running main's installer reintroduces the same drift.""" + pins = { + "CLAUDE_CODE_VERSION": "1", + "CODEX_VERSION": "2", + "OPENCODE_VERSION": "3", + "NODE_VERSION": "4", + "HERMES_VERSION": "v2026.8.3", + } + + layer = _load_generator_module()._install_layer(pins) + + assert "hermes-agent/v2026.8.3/scripts/install.sh" in layer + assert "hermes-agent/main/" not in layer + assert "--branch v2026.8.3" in layer + From f4052d9aa6f4dc29afe4db49d68188b9ac17bd54 Mon Sep 17 00:00:00 2001 From: Giedrius Burachas Date: Fri, 14 Aug 2026 09:51:28 -0700 Subject: [PATCH 4/4] fix(benchmark): pin hermes by commit sha and fix the alpine and missing-pin gaps Signed-off-by: Giedrius Burachas --- benchmark/agent-versions.env | 12 ++-- benchmark/prepare_harbor_dataset.py | 39 ++++++++---- tests/test_prepare_harbor_dataset.py | 93 ++++++++++++++++++++++++---- 3 files changed, 116 insertions(+), 28 deletions(-) diff --git a/benchmark/agent-versions.env b/benchmark/agent-versions.env index da2a84ea8..a064b2b37 100644 --- a/benchmark/agent-versions.env +++ b/benchmark/agent-versions.env @@ -6,8 +6,10 @@ CODEX_VERSION=0.144.5 OPENCODE_VERSION=1.18.3 NODE_VERSION=20.11.1 # Hermes (NousResearch hermes-agent), installed from GitHub at dataset-bake time. -# Must be an immutable ref — a tag or commit SHA. A moving ref such as "main" is -# rejected at build time: the run manifest records this value as a pin, and two -# builds recording the same string while installing different code is worse than -# recording nothing. The installer script is fetched from this ref too. -HERMES_VERSION=v2026.8.3 +# Must be a full 40-character commit SHA; anything else is rejected at build time. +# A tag would be enough to read but not to trust — it can be deleted or repointed, +# so two builds could record the same string while installing different code, and +# the run manifest records this value as a pin. The installer script is fetched +# from this same commit, so the installer cannot drift either. +# 3c27eb62 is the commit tagged v2026.8.3 (2026-08-03). +HERMES_VERSION=3c27eb6234bf91b8ceee9e9071591b31e9b148cb diff --git a/benchmark/prepare_harbor_dataset.py b/benchmark/prepare_harbor_dataset.py index 371d991f2..44c08f146 100644 --- a/benchmark/prepare_harbor_dataset.py +++ b/benchmark/prepare_harbor_dataset.py @@ -29,6 +29,7 @@ AGENT_VERSIONS_FILE = SCRIPT_DIR / "agent-versions.env" PROXY_ASSET_DIR = SCRIPT_DIR / "closed_book_proxy" / "proxy" AGENT_ENTRYPOINT = "switchyard-agent-entrypoint.sh" +COMMIT_SHA_PATTERN = re.compile(r"[0-9a-f]{40}") TERMINAL_BENCH_2_SOURCE_DATASET = "terminal-bench/terminal-bench-2" TERMINAL_BENCH_2_1_SOURCE_DATASET = "terminal-bench/terminal-bench-2-1" # Shared across the TB2 family (2.0 + the 2.1 verified iteration): 2.1 tweaks @@ -192,17 +193,25 @@ def _install_layer(pins: dict[str, str]) -> str: # means the runtime install() skip-guard short-circuits, so tasks need no # egress for it — enabling closed-book Hermes runs. # - # HERMES_VERSION must name an immutable ref. A moving ref would let two builds - # record the same version string while installing different code, which is worse - # than not recording it: the manifest would assert a reproducibility it does not - # have. The installer script is fetched from the same ref for the same reason — - # pinning the agent but running whatever installer main has today reintroduces - # exactly the drift the pin exists to prevent. + # HERMES_VERSION must be a full commit SHA. A tag is not enough: tags can be + # deleted or repointed, so two builds could record the same version string while + # installing different code — the manifest would assert a reproducibility it does + # not have. Requiring one shape is complete by construction, where a deny-list of + # moving names ("main", "master", ...) can never be, since any branch name passes. + # The installer script is fetched from the same commit for the same reason: pinning + # the agent but running whatever installer main has today reintroduces the drift. + # + # The pin is applied with the installer's --commit, not --branch: --branch reaches + # `git clone --branch`, which accepts only branch and tag names and rejects a SHA. + # --force-commit is required with it. Without it the installer skips the pin when + # the commit is an ancestor of the freshly cloned HEAD, logging a warning and + # silently leaving the image on the tip of main — the drift this pin exists to + # prevent, arriving as a warning rather than a build failure. hermes_version = pins["HERMES_VERSION"] - if hermes_version in ("main", "master", "HEAD"): + if not COMMIT_SHA_PATTERN.fullmatch(hermes_version): raise SystemExit( - f"HERMES_VERSION={hermes_version!r} is a moving ref and cannot be recorded " - "as a reproducible pin; use a tag or commit SHA" + f"HERMES_VERSION={hermes_version!r} is not a full 40-character commit SHA; " + "a tag or branch can be repointed and cannot be recorded as a reproducible pin" ) return f""" @@ -256,10 +265,10 @@ def _install_layer(pins: dict[str, str]) -> str: apt-get install -y --no-install-recommends git ripgrep xz-utils; \\ rm -rf /var/lib/apt/lists/*; \\ elif command -v apk >/dev/null 2>&1; then \\ - apk add --no-cache git ripgrep xz; \\ + apk add --no-cache bash git ripgrep xz; \\ fi; \\ curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/{hermes_version}/scripts/install.sh \\ - | bash -s -- --skip-setup --branch {hermes_version}; \\ + | bash -s -- --skip-setup --commit {hermes_version} --force-commit; \\ hermes version """ @@ -493,7 +502,13 @@ def prepare_dataset( overwrite: bool, ) -> Path: pins = _read_env_file(AGENT_VERSIONS_FILE) - required = {"CLAUDE_CODE_VERSION", "CODEX_VERSION", "OPENCODE_VERSION", "NODE_VERSION"} + required = { + "CLAUDE_CODE_VERSION", + "CODEX_VERSION", + "HERMES_VERSION", + "NODE_VERSION", + "OPENCODE_VERSION", + } missing = sorted(required - pins.keys()) if missing: raise ValueError(f"missing pins in {AGENT_VERSIONS_FILE}: {', '.join(missing)}") diff --git a/tests/test_prepare_harbor_dataset.py b/tests/test_prepare_harbor_dataset.py index 4b1e30e3f..74dd620bf 100644 --- a/tests/test_prepare_harbor_dataset.py +++ b/tests/test_prepare_harbor_dataset.py @@ -285,7 +285,7 @@ def test_generated_dataset_manifest_records_pins_tasks_and_digests(tmp_path: Pat assert manifest["agent_versions"] == { "CLAUDE_CODE_VERSION": "2.1.211", "CODEX_VERSION": "0.144.5", - "HERMES_VERSION": "v2026.8.3", + "HERMES_VERSION": "3c27eb6234bf91b8ceee9e9071591b31e9b148cb", "NODE_VERSION": "20.11.1", "OPENCODE_VERSION": "1.18.3", } @@ -309,13 +309,14 @@ def test_generated_compose_bakes_task_id_into_proxy_env(tmp_path: Path) -> None: assert "SWITCHYARD_TASK_ID=task-id-check" in proxy_env assert "SWITCHYARD_TRIAL_DIR=${HOST_AGENT_LOGS_PATH:-}" in proxy_env -def test_a_moving_hermes_ref_is_rejected() -> None: - """A moving ref cannot be recorded as a pin. +def test_a_hermes_ref_that_is_not_a_commit_sha_is_rejected() -> None: + """Only a full commit SHA can be recorded as a pin. The dataset manifest presents HERMES_VERSION as a reproducibility guarantee. Two builds recording the same string while installing different Hermes code is worse than recording nothing, so the build fails rather than asserting a pin it does not - have. + have. Tags are rejected alongside branches: a tag can be deleted or repointed, so + it reads as immutable without being so. """ base = { "CLAUDE_CODE_VERSION": "1", @@ -323,24 +324,94 @@ def test_a_moving_hermes_ref_is_rejected() -> None: "OPENCODE_VERSION": "3", "NODE_VERSION": "4", } - for moving in ("main", "master", "HEAD"): - with pytest.raises(SystemExit, match="moving ref"): - _load_generator_module()._install_layer({**base, "HERMES_VERSION": moving}) + rejected = ( + "main", + "master", + "HEAD", + "v2026.8.3", + "release/2026.8", + "3c27eb6", + "3C27EB6234BF91B8CEEE9E9071591B31E9B148CB", + "3c27eb6234bf91b8ceee9e9071591b31e9b148cbb", + ) + for ref in rejected: + with pytest.raises(SystemExit, match="commit SHA"): + _load_generator_module()._install_layer({**base, "HERMES_VERSION": ref}) -def test_the_hermes_installer_is_fetched_at_the_pinned_ref() -> None: +def test_the_hermes_installer_is_fetched_at_the_pinned_commit() -> None: """Pinning the agent but running main's installer reintroduces the same drift.""" + sha = "3c27eb6234bf91b8ceee9e9071591b31e9b148cb" pins = { "CLAUDE_CODE_VERSION": "1", "CODEX_VERSION": "2", "OPENCODE_VERSION": "3", "NODE_VERSION": "4", - "HERMES_VERSION": "v2026.8.3", + "HERMES_VERSION": sha, } layer = _load_generator_module()._install_layer(pins) - assert "hermes-agent/v2026.8.3/scripts/install.sh" in layer + assert f"hermes-agent/{sha}/scripts/install.sh" in layer assert "hermes-agent/main/" not in layer - assert "--branch v2026.8.3" in layer + + +def test_the_hermes_pin_is_applied_by_commit_and_forced() -> None: + """`--branch` reaches `git clone --branch`, which rejects a SHA outright. + + `--force-commit` is what makes the pin take effect. Without it the installer skips + the checkout whenever the commit is an ancestor of the freshly cloned HEAD, warns, + and leaves the image on the tip of main — the drift the pin exists to prevent, + arriving as a warning rather than a build failure. + """ + sha = "3c27eb6234bf91b8ceee9e9071591b31e9b148cb" + pins = { + "CLAUDE_CODE_VERSION": "1", + "CODEX_VERSION": "2", + "OPENCODE_VERSION": "3", + "NODE_VERSION": "4", + "HERMES_VERSION": sha, + } + + layer = _load_generator_module()._install_layer(pins) + + assert f"--commit {sha}" in layer + assert "--force-commit" in layer + assert "--branch" not in layer + + +def test_the_alpine_branch_installs_the_shell_the_installer_needs() -> None: + """The installer is piped into bash, which Alpine does not ship by default.""" + pins = { + "CLAUDE_CODE_VERSION": "1", + "CODEX_VERSION": "2", + "OPENCODE_VERSION": "3", + "NODE_VERSION": "4", + "HERMES_VERSION": "3c27eb6234bf91b8ceee9e9071591b31e9b148cb", + } + + layer = _load_generator_module()._install_layer(pins) + + assert "apk add --no-cache bash git ripgrep xz" in layer + + +def test_a_missing_hermes_pin_is_reported_with_the_other_pins(tmp_path: Path) -> None: + """Absent, it must fail the shared pin check rather than crash reading the layer.""" + module = _load_generator_module() + versions = tmp_path / "agent-versions.env" + versions.write_text( + "CLAUDE_CODE_VERSION=1\nCODEX_VERSION=2\nOPENCODE_VERSION=3\nNODE_VERSION=4\n" + ) + module.AGENT_VERSIONS_FILE = versions + source = tmp_path / "source" + _write_task(source, "task-a", "[environment]\n", "FROM ubuntu:22.04\n") + + with pytest.raises(ValueError, match="missing pins.*HERMES_VERSION"): + module.prepare_dataset( + source_dataset="openthoughts-tblite@2.0", + source_dir=source, + output_dir=tmp_path / "prepared", + harbor_command="harbor", + overwrite=False, + )