From 6a4875f223cf16f12d28d82566cfda9cd77ac7d2 Mon Sep 17 00:00:00 2001 From: Siddhant Sinha Date: Wed, 8 Jul 2026 17:14:19 +0530 Subject: [PATCH 01/10] =?UTF-8?q?docs(0045):=20merge=20command=20&=20merge?= =?UTF-8?q?-rules=20decision=20=E2=80=94=20gates=20the=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BMuEZ1CZDLE9jjhmrNMNw6 --- design/contract/03-commands.md | 184 +++++++++++++ .../decisions/0045-merge-command-and-rules.md | 246 ++++++++++++++++++ design/features.yaml | 4 + 3 files changed, 434 insertions(+) create mode 100644 design/decisions/0045-merge-command-and-rules.md diff --git a/design/contract/03-commands.md b/design/contract/03-commands.md index e8a969d..e953ff5 100644 --- a/design/contract/03-commands.md +++ b/design/contract/03-commands.md @@ -137,3 +137,187 @@ evidence-cli is built in TypeScript/Node and exposes `validate`/`finalize` as library functions, so kane-cli mounts it **in-process** as `kane-cli evidence` with shared types and no subprocess boundary. See decision [0019 — Runtime](#/decisions). + +## `evidence merge --run-id -o ` + +Assembles two or more evidence packs (sealed zips or live directories) into +one live pack under a declarative **merge-rules** policy — the policy gates +whole packs and resolves per-test collisions, merge or discard. Merge never +writes a derived artifact: the existing `finalize` remains the sole producer +of totals, hashes, the root failure index, and `ended` (opt in immediately via +`--finalize`, or later, manually, on the assembled output). See decision +[0045 — Merge command & merge rules](#/decisions). + +```bash +evidence merge --run-id -o \ + [--rules merge-rules.yaml] [--title ] [--finalize] +``` + +- `` — two or more packs; **CLI order is meaningful**: it anchors + `must: same` reference values, defines `prefer_first`, and tie-breaks + `prefer_latest`. +- `--run-id` — **mandatory**; a usage error (exit `2`) without it. Merge takes + no clock reading and generates no random id, so identity is the caller's + supplied fact — the same testability precedent as `finalize`'s `endedAt`. +- `-o` — output pack path, always a live `.evidence/` directory. Refuses + to overwrite an existing path. +- `--rules` — the merge-rules YAML (below). Omitted → strict defaults. +- `--title` — the merged `run.yaml` title; default is the first eligible + pack's title. +- `--finalize` — after assembling, runs the real `finalize` on the output with + `endedAt = max(source ended)` (fallback `max(source started)`) — the + truthful end of the logical run. Without the flag, a later manual + `finalize` stamps seal time instead (a documented difference). + +### The merge-rules file + +A flat rule list `{file, key, must, on_violation}` beside two fixed knob +blocks. Every rule's `file` also fixes its **scope**: `run.yaml` rules are +compared **across all eligible packs** and gate whole packs; `result.yaml` +rules are compared **between two colliding tests** and resolve collisions. +`key` is a dot-path into the parsed YAML. + +```yaml +# merge-rules.yaml +packs: + require_status: finalized # finalized | running | any + require_valid: L0 # L0 | L1 | off + on_ineligible: abort # abort | skip +tests: + on_collision: error # error | prefer_first | prefer_latest | discard +rules: + - file: run.yaml # scope: compared ACROSS all eligible packs + key: environment.producer.name + must: same # same | different + on_violation: abort # abort | skip (pack-scoped actions) + - file: run.yaml + key: run_id + must: different + on_violation: skip + - file: result.yaml # scope: compared BETWEEN two colliding tests + key: environment.model + must: same + on_violation: discard # error | prefer_first | prefer_latest | discard +``` + +Pack-scoped rules only allow pack actions (`abort`/`skip`); collision rules +only allow collision actions (`error`/`prefer_first`/`prefer_latest`/ +`discard`) — enforced by the schema itself. `src/schemas/merge-rules.schema.json` +validates the rules file; it lives **beside**, not under, the versioned `0.1/` +tree (tool input, not pack contract), compiled through the same AJV path used +elsewhere. A rules file that fails to parse or conform is a **usage error +(exit `2`)** before any pack is opened. + +Omitting `--rules` uses the strict defaults — nothing dropped silently: + +```yaml +packs: { require_status: finalized, require_valid: L0, on_ineligible: abort } +tests: { on_collision: error } +rules: [] +``` + +### Pack gates — in order, cheap → expensive + +1. **Readable manifest** — `run.yaml` missing or unparseable → ineligible. +2. **Version gate** — `evidence != "0.1"` → ineligible; never merge across + contract versions. +3. **`require_status`** (default `finalized`) — `running`/`aborted` → + ineligible. +4. **`require_valid`** (default `L0`; `L1` or `off` settable) — runs the + existing `validate`; any error diagnostic → ineligible. +5. **Generic `run.yaml` rules** — `{file: run.yaml, key, must, on_violation}`. + +Ineligibility resolves per `packs.on_ineligible`: **`skip`** (drop the pack, +record why in the report, continue) or **`abort`** (the whole merge fails, +exit `1`). A rule's own `on_violation` overrides the global for that rule. +Zero eligible packs is always an error; **one** eligible pack still merges (a +valid single-source pack — CI scripts stay unconditional). + +Packs are processed one at a time, in CLI order: a pack must pass **all** +gates and rules to join the eligible set, and rules compare only against +**previously eligible** packs. A pack that anchors a `same` value but then +fails a later rule never becomes eligible, and its anchor is discarded — the +next fully-surviving pack anchors instead. + +### `must` semantics + +- **`same`** — the *first eligible pack* anchors the reference value; each + later pack that differs violates. Deterministic; no majority voting. +- **`different`** — the first occurrence of a value keeps; a later pack + repeating it violates — e.g. `{file: run.yaml, key: run_id, must: different, + on_violation: skip}` dedupes a double-submitted shard automatically. +- **Absent keys** — absent == absent counts as `same`; absent vs. present + counts as `different`. +- Values compare by canonical deep equality (objects/arrays included). + +### Test-level collisions + +Each eligible pack's `tests/` ids are claimed in CLI order; the first +claimant is the **incumbent**, a later pack with the same id is a +**collision**, resolved pairwise (incumbent vs. challenger) by the first +collision rule that fires (in file order), else the `tests.on_collision` +default. + +| Action | Meaning | +| --- | --- | +| `error` | abort the whole merge (exit `1`) — the strict default | +| `prefer_first` | incumbent wins (CLI order) | +| `prefer_latest` | the copy from the pack with the later `run.yaml` `ended` wins (fallback `started`; tie → CLI order) | +| `discard` | drop the test **entirely** — both copies; the id is **tombstoned** so a third pack's copy cannot resurrect it | + +3+-way collisions resolve pairwise in CLI order: the winner of (1 vs 2) faces +pack 3's copy, and so on; any `discard` verdict tombstones the id for good. +The winner's entire `tests//` directory travels **whole** — definition, +`result.yaml`, `logs/`, `steps/` (screenshots and failure records), video — +the loser's tree is dropped completely, with no artifact-level mixing between +copies. Folding collisions into `attempts[]` (retry semantics) is +deliberately out of scope — see decision +[0033 — attempts is a per-attempt outcome list](#/decisions). + +### `run.yaml` — per-key disposition + +| Key | Across the N inputs | Value in merged `run.yaml` | +| --- | --- | --- | +| `evidence` | must be same — hard version gate, not policy | `"0.1"` | +| `run_id` | no constraint by default | **`--run-id`** (mandatory) | +| `status` | gated by `require_status`, never compared | `running` (live until finalize) | +| `title` | no constraint | first eligible pack's title; `--title` overrides | +| `started` | no constraint — execution fact | `min(started)` across eligible packs | +| `ended` | no constraint — execution fact | **not written by merge**; `--finalize` seals with `max(source ended)` (fallback `max(source started)`) | +| `totals` | ignored — never compared, never summed | absent; `finalize` re-derives from the merged union | +| `metrics` | no constraint — free-form semantics unknowable | **namespaced by flattening** into the metric name: `-/` (1-based index over eligible packs in CLI order; skipped packs consume no ordinal), each original typed object intact | +| `environment` | no built-in constraint; rules pick sub-keys | **common subset** stays run-level; divergent keys **pushed down** per test (per-test value wins where already present) | +| `merged_from` *(new, additive)* | — | list of source `run_id`s, in CLI order | + +`coverage/` from each eligible source nests under `coverage/-/` +(the same label used for metrics). The root `failure.yaml` index is +**deliberately not copied** — the live merged pack has no root index (valid: +presence is only required at `finalized`, decision +[0044 — Failure records](#/decisions)); `finalize` regenerates it from the +merged tree, exactly as it does for any other live pack. + +### Exit codes + +`0` merged (policy-sanctioned skips/discards included) · `1` abort/error +(rule abort, collision `error`, zero eligible packs) · `2` usage (missing +`--run-id`, unreadable/invalid rules file, output path exists). + +### `MergeReport` + +`src/merge/` exposes `merge(inputs, opts): Promise`; the CLI +prints it via the existing reporter conventions. + +```yaml +packs: + eligible: [shard-a, shard-b] # run_ids, CLI order + skipped: [{ run_id: shard-a2, rule: "run.yaml run_id must different", reason: duplicate of shard-a }] +tests: + merged: 214 + collisions: [{ test: checkout, winner: shard-b, rule: tests.on_collision=prefer_latest }] + discarded: [flaky-login] +output: { path: merged.evidence, run_id: nightly-2026-07-08, finalized: true } +``` + +Policy-sanctioned skips/discards exit `0` — the report carries the story. The +pack itself stays clean: `merged_from` is the only in-pack trace of the +merge. diff --git a/design/decisions/0045-merge-command-and-rules.md b/design/decisions/0045-merge-command-and-rules.md new file mode 100644 index 0000000..bae0494 --- /dev/null +++ b/design/decisions/0045-merge-command-and-rules.md @@ -0,0 +1,246 @@ +--- +id: 45 +slug: merge-command-and-rules +title: Merge command — policy-driven pack merging (merge-rules) +status: accepted +date: 2026-07-08 +proposition: > + Sharded runs produce N packs that must become one. What does `evidence merge` + do, what does a passed merge-rule govern (merge or discard at pack and test + level), and how does the result relate to finalize's derived artifacts? +options: + - id: assemble-then-finalize + summary: > + merge ASSEMBLES a live pack under a declarative rules file (pack gates + + collision policy + generic {file, key, must} predicates); the existing + finalize derives totals/hashes/failure-index/ended and seals (--finalize + convenience). Mandatory --run-id; merged_from lineage. + chosen: true + - id: merge-seals-directly + summary: merge produces a sealed zip itself, re-implementing finalize's derivation. + chosen: false + - id: finalize-merge-mode + summary: finalize --merge a b c — no new command, overloads 0035's contract. + chosen: false +decision: > + `evidence merge --run-id -o [--rules f] [--title t] + [--finalize]` ASSEMBLES a new live pack from two or more inputs under a + declarative merge-rules policy; merge never writes a derived artifact — the + existing `finalize` remains the SOLE producer of totals, hashes, the root + failure index, and `ended`, whether invoked immediately via the `--finalize` + convenience flag or later, manually, on the resulting live directory. + `` order is meaningful: it anchors `must: same` reference values, + defines `prefer_first`, and tie-breaks `prefer_latest`. `--run-id` is + MANDATORY — its absence is a usage error (exit 2); merge is fully + deterministic (no clock, no randomness), so identity is the caller's + supplied fact, mirroring the `finalize` `endedAt` testability precedent. + `-o` refuses an existing path. `--rules` names the merge-rules YAML; + omitted, STRICT DEFAULTS apply (below). `--title` overrides the default + merged title (the first eligible pack's). `--finalize` runs the real + `finalize` on the assembled output with `endedAt = max(source ended)` + (fallback `max(source started)` when no source carries `ended`) — the + truthful end of the logical run; without the flag, a later manual + `finalize` stamps its own seal time instead (a documented difference). The + merge-rules file is a flat rule list `{file, key, must, on_violation}` + beside two fixed knob blocks: `packs: {require_status, require_valid, + on_ineligible}` and `tests: {on_collision}`. A rule's `file` fixes its + SCOPE — `run.yaml` rules gate whole packs (evaluated during gating, in file + order); `result.yaml` rules resolve test collisions (evaluated pairwise + between two colliding tests, in file order; the first VIOLATED rule + applies its `on_violation`, else the default `tests.on_collision` applies). + `key` is a dot-path into the parsed YAML. Pack-scoped rules only permit + pack actions (`abort`/`skip`); collision rules only permit collision + actions (`error`/`prefer_first`/`prefer_latest`/`discard`) — enforced by + the schema itself (`if file == run.yaml then …`). + `src/schemas/merge-rules.schema.json` lives BESIDE, not under, the + versioned `0.1/` tree — it is tool input, not pack contract — compiled + through the same AJV path. A + rules file that fails to parse or conform is a USAGE ERROR (exit 2) before + any pack is opened. Pack gates run cheap-to-expensive, in order: readable + `run.yaml`; the hard version gate (`evidence != "0.1"` → ineligible, never + merge across contract versions); `require_status` (default `finalized` — a + live directory is normally `running`/`aborted` so is ineligible unless + relaxed); `require_valid` (default `L0`; `L1`/`off` settable — runs the + existing `validate`, any error diagnostic → ineligible); then the generic + `run.yaml` rules. Ineligibility resolves per `packs.on_ineligible` — + `skip` (drop the pack, record why in the report, continue) or `abort` (the + whole merge fails, exit 1) — and a rule's own `on_violation` overrides the + global for that rule. Zero eligible packs is always an error; exactly ONE + eligible pack still merges (a valid single-source pack, so CI scripts stay + unconditional). `must` semantics are pinned: `same` — the FIRST ELIGIBLE + PACK anchors the reference value, and each later pack that differs + violates (deterministic; no majority vote); `different` — the FIRST + OCCURRENCE of a value KEEPS, and a later pack repeating it violates (so + `{file: run.yaml, key: run_id, must: different, on_violation: skip}` + dedupes a double-submitted shard automatically); absent keys — absent == + absent counts as `same`, absent-vs-present counts as `different`; + comparison is canonical deep equality (objects/arrays included). + SEQUENTIAL ELIGIBILITY pins the chicken-and-egg: packs are processed one + at a time in CLI order, a pack must pass all gates and rules to join the + eligible set, and rules compare only against PREVIOUSLY ELIGIBLE packs — a + pack that anchors a `same` value but then fails a later rule never becomes + eligible, and its anchor is discarded; the next fully-surviving pack + anchors instead. When `--rules` is omitted, STRICT DEFAULTS apply — + nothing dropped silently: `packs: {require_status: finalized, + require_valid: L0, on_ineligible: abort}`, `tests: {on_collision: error}`, + `rules: []`. Test-level collisions resolve by a UNION WALK: in CLI order, + each eligible pack's `tests/` ids are claimed; the first claimant is + the INCUMBENT, a later pack with the same id is a COLLISION, resolved + PAIRWISE (incumbent vs. challenger) by the firing rule or the + `tests.on_collision` default. Four actions: `error` — abort the whole + merge (exit 1), the strict default; `prefer_first` — the incumbent (CLI + order) wins; `prefer_latest` — the copy from the pack with the later + `run.yaml` `ended` wins (fallback `started`; tie → CLI order); `discard` — + drop the test ENTIRELY, both copies, and TOMBSTONE the id so a third + pack's copy cannot resurrect it. 3+-WAY collisions resolve pairwise in CLI + order — the winner of (1 vs 2) faces pack 3's copy, and so on — and any + `discard` verdict tombstones the id for good. WHOLE-TREE ATOMICITY: the + winner's entire `tests//` directory travels intact — definition, + `result.yaml`, `logs/`, `steps/` (screenshots and failure records), video + — and the loser's tree is dropped completely; there is no artifact-level + mixing between copies. The merged `run.yaml` is synthesized per key: + `evidence` stays `"0.1"` (the hard version gate, not policy); `run_id` is + the mandatory `--run-id`; `status` is `running` (live until finalize); + `title` is the first eligible pack's title unless `--title` overrides; + `started` is `min(started)` across eligible packs; `ended` is NOT WRITTEN + by merge — `--finalize` seals it with `max(source ended)` (fallback + `max(source started)`); `totals` is absent — never compared, never summed + (a sum would be falsified by collisions/discards) — finalize re-derives + it from the merged union; `metrics` are NAMESPACED BY FLATTENING into the + metric name (`-/`, each original typed `{value, type}` + object intact, since literal nesting under `metrics:` would violate + 0012's shape) — the `` is a 1-based index over ELIGIBLE packs in CLI + order (skipped packs consume no ordinal), and the same `-` + label nests `coverage/-/` per source; `environment` keeps only + the COMMON SUBSET across every eligible pack at run level (deep equality + per top-level path) and PUSHES DOWN each divergent key into the affected + tests' `result.yaml` `environment` blocks (per 0043's lossless merge + design) — kept as-is where a test already carries its own value (per-test + wins) — via the comment-preserving `parseDoc`/`setIn` path, so the + definition file is never touched and hash checks stay green; `merged_from` + (new, additive) is the list of source `run_id`s in CLI order — the small, + additive slice of 0014's deferred lineage. The root `failure.yaml` index + is DELIBERATELY NOT COPIED by merge — the live merged pack simply has no + root index, which is valid because 0044 only requires its presence at + `finalized`. `finalize` on a merged pack REGENERATES every derived + artifact from the merged tree exactly as it always has: the failure index + is rebuilt from the merged union's `steps/*/failure.yaml` (discarded/loser + tests contribute no rows; winners' rows have correct paths because whole + trees traveled intact, so 0044's completeness/dangling/row-path + guarantees hold by construction); `totals` is re-derived by counting + merged tests (losers/discards naturally excluded); `definition.sha256` is + re-hashed per test (a no-op re-derivation, since definitions travel + byte-identical); `ended` is set by `--finalize`'s `max(source ended)` + (fallback `max(source started)`), or stamped at seal time by a later + manual `finalize`. The assembled pack validates clean at L0/L1 under + `status: running` (no index or totals demanded) and, after finalize, at + `finalized` with the regenerated index — both are testable assertions. + Exit codes: `0` merged (policy-sanctioned skips/discards included) · `1` + abort/error (a rule's `abort`, collision `error`, zero eligible packs) · + `2` usage (missing `--run-id`, unreadable/invalid rules file, an existing + `-o` path). `src/merge/` exposes `merge(inputs, opts): Promise` + — `{ packs: {eligible, skipped: [{run_id, rule, reason}]}, tests: {merged, + collisions: [{test, winner, rule}], discarded}, output: {path, run_id, + finalized} }` — printed by the CLI via the existing reporter conventions; + the pack itself stays clean, `merged_from` its only in-pack trace of the + merge. +governs: + - design/contract/03-commands.md + - src/schemas/merge-rules.schema.json + - src/ +feature: [merge] +depends_on: [43, 44, 14, 35, 42, 33, 12, 18, 36] +supersedes: [] +--- + +## Reasoning + +**Why assemble, not seal or overload finalize.** Every derived artifact — +totals, `definition.sha256`, the root failure index, `ended` — must keep +exactly ONE producer, or two code paths could each derive it differently and +drift. `finalize` already owns derivation and the +[atomic seal](0042-atomic-seal-in-place.md) on a +[live directory](0035-finalize-targets-live-directory.md); a +`merge-seals-directly` design would have to re-implement that whole pipeline +as a second producer, purely to save one command invocation. Overloading +[`finalize`'s existing contract](0017-commands-validate-and-finalize.md) with +a `--merge` mode (`finalize-merge-mode`) is worse: it conflates two different +operations — combining trees under a policy, and deriving/sealing a single +tree — behind one verb, and would force finalize to understand pack-gating +and collision policy it has no other reason to know about. Keeping merge a +pure ASSEMBLER — it writes `run.yaml`, copies winning test trees, pushes +environment down — and letting `--finalize` be a thin, opt-in convenience +call into the unchanged `finalize` keeps the one-producer-per-artifact +invariant intact and lets `finalize`'s existing atomicity, hashing, and +index-generation apply unmodified to a merged tree exactly as it does to any +other live pack. + +**Why 0043 had to land first.** [0043](0043-environment-at-result-level.md) +added an optional per-test `environment` precisely to unblock this merge: +without it, combining N single-environment packs into one `run.yaml` with one +`environment` slot would be lossy — divergent per-source environments would +have to be discarded or synthesized. With result-level `environment` +available, merge can keep the common subset at run level and push every +divergent key down onto the tests that actually ran under it, so "what did +`checkout` run on?" stays answerable on the merged pack with nothing thrown +away. + +**Why 0044's guarantees hold for free.** The +[failure-record index](0044-failure-records.md) is truthful by construction +on any freshly finalized pack because finalize is its only producer, and +merge changes nothing about that: because collision resolution moves WHOLE +test trees (never individual files), a winner's `steps/*/failure.yaml` and +its folder-anchored `-` naming arrive byte-identical, so +finalize's regenerated index on a merged pack satisfies completeness, +dangling-pointer, and row/path agreement exactly as it would on an unmerged +pack — no special-casing needed in the failure-index checks for merged +input. + +**Why caller-supplied identity, not a generated one.** Mandatory `--run-id` +mirrors the precedent set by +[`finalize`'s testable `endedAt`](0042-atomic-seal-in-place.md) parameter: +merge takes no wall-clock reading and generates no random id, so its output +is a pure function of its declared inputs — reproducible in tests and +predictable in CI, where the caller (a pipeline) already has a natural +identity (the pipeline run, a nightly date) that is truer than anything +merge could invent. + +**Why the rules file is a flat list beside two knob blocks, not one big +schema.** The three fixed knobs (`require_status`, `require_valid`, +`on_ineligible`, `on_collision`) cover the common cases compactly and need +no per-rule ceremony; the generic `{file, key, must, on_violation}` shape +handles the long tail (a specific run-level key must agree, a specific +collision key decides the winner) without the format having to anticipate +every field a producer might want to compare. Scoping each rule by its +`file` — `run.yaml` gates packs, `result.yaml` resolves collisions — keeps +the action enum small (pack actions and collision actions never mix) while +keeping the *nuance* (which key, which direction) in the rule rather than +growing the action vocabulary. + +## Consequences + +- New `src/merge/` module exposing `merge(inputs, opts): Promise` + and the CLI `evidence merge` command. +- New `src/schemas/merge-rules.schema.json`, living BESIDE (not under) the + versioned `0.1/` tree — tool input, not pack contract — compiled through + the existing AJV path ([0024](0024-schemas-single-source-of-truth.md)). +- The run schema gains an optional, additive `merged_from` (array of + non-empty strings) — no version bump, no L0 pack invalidated + ([0027](0027-evidence-version-and-profiles.md)). +- `PackContainer` (`src/pack/container.ts`) gains a pack-root-relative + `readFileBytes` primitive alongside the existing test-relative + `readBytes`/pack-root `readText`, so merge can copy whole-tree binary + artifacts (screenshots, video) verbatim without new per-artifact + knowledge. +- `design/contract/03-commands.md` gains the `## merge` section (command + surface, rules file, gate order, collision table, run.yaml disposition + table, exit codes, `MergeReport` shape). +- `design/features.yaml` gains the `merge` feature. +- Deferred, additively: the `fold_attempts` collision action (folding into + [0033](0033-attempts-and-flaky.md)'s `attempts[]` — retry semantics, not + shard semantics); cross-environment matrix identity (keeping both copies + of a colliding test disambiguated by environment); per-metric merge + arithmetic (sum/max/avg — merge never guesses + [0012](0012-typed-free-form-metrics.md)'s free-form semantics); `must` + predicates beyond `same`/`different` (`equals`, `exists`, regex); remote + inputs (the container interface permits it later; v1 is local paths only). diff --git a/design/features.yaml b/design/features.yaml index f22e3ef..08c278e 100644 --- a/design/features.yaml +++ b/design/features.yaml @@ -47,6 +47,10 @@ features: title: L1 profile blurb: The additive evidence-artifact layer — per-test logs/ and steps/ screenshots, global coverage/, optional video, and structured failure records (step-authored + finalize-indexed); opaque artifacts validated by filesystem cross-checks. spec: design/contract/04-L1.md + - id: merge + title: Merge + blurb: Policy-driven merging of N packs into one — declarative merge-rules gate packs and resolve test collisions; merge assembles, finalize derives and seals. + spec: design/contract/03-commands.md - id: decision-system title: Decision system & viewer blurb: Governance, the ADR format, schemas-as-source-of-truth, and the viewer. From cf1a663ca7b3a5ed71fcc8a024a20ff61b9304f3 Mon Sep 17 00:00:00 2001 From: Siddhant Sinha Date: Wed, 8 Jul 2026 17:24:16 +0530 Subject: [PATCH 02/10] feat(0045): merged_from run field + readFileBytes container primitive Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BMuEZ1CZDLE9jjhmrNMNw6 --- src/pack/container.test.ts | 39 ++++++++++++++++++++++++++++++ src/pack/container.ts | 15 ++++++++++++ src/pack/remote.ts | 4 +++ src/schemas/0.1/L0/run.schema.json | 6 +++++ src/schemas/compile.test.ts | 8 ++++++ 5 files changed, 72 insertions(+) diff --git a/src/pack/container.test.ts b/src/pack/container.test.ts index e7fa102..d209e86 100644 --- a/src/pack/container.test.ts +++ b/src/pack/container.test.ts @@ -16,6 +16,23 @@ async function makeZip(): Promise { return zipPath; } +/** Stage a pack dir with a nested binary artifact (tests/checkout/steps/1-open/screenshot.png), then zip it flat. */ +async function stagePackWithScreenshot(): Promise<{ dirPath: string; zipPath: string }> { + const stageRoot = await fs.mkdtemp(path.join(os.tmpdir(), "evi-stage-")); + const dirPath = path.join(stageRoot, "staged.evidence"); + await fs.cp(SMOKE, dirPath, { recursive: true }); + const stepsDir = path.join(dirPath, "tests", "checkout", "steps", "1-open"); + await fs.mkdir(stepsDir, { recursive: true }); + await fs.writeFile(path.join(stepsDir, "screenshot.png"), Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x00, 0x01, 0x02, 0x03])); + + const zipPath = path.join(stageRoot, "staged-zip.evidence"); + const zip = new AdmZip(); + zip.addLocalFolder(dirPath); // flat: entries are the directory's contents + zip.writeZip(zipPath); + + return { dirPath, zipPath }; +} + describe("PackContainer parity", () => { let zipPath: string; @@ -87,3 +104,25 @@ describe("PackContainer primitives (dir == zip)", () => { if (zipPath) await fs.rm(path.dirname(zipPath), { recursive: true, force: true }); }); }); + +describe("readFileBytes (pack-root-relative, dir/zip parity)", () => { + let stageRoot: string; + + it("reads bytes at a root-relative path in both forms, null when absent", async () => { + const { dirPath, zipPath } = await stagePackWithScreenshot(); + stageRoot = path.dirname(dirPath); + const dir = await openContainer(dirPath); + const zip = await openContainer(zipPath); + + for (const c of [dir, zip]) { + const b = await c.readFileBytes("tests/checkout/steps/1-open/screenshot.png"); + expect(b).not.toBeNull(); + expect(b!.length).toBeGreaterThan(0); + expect(await c.readFileBytes("nope/missing.bin")).toBeNull(); + } + }); + + afterAll(async () => { + if (stageRoot) await fs.rm(stageRoot, { recursive: true, force: true }); + }); +}); diff --git a/src/pack/container.ts b/src/pack/container.ts index a86b71a..8d5344f 100644 --- a/src/pack/container.ts +++ b/src/pack/container.ts @@ -27,6 +27,8 @@ export interface PackContainer { listDir(rel: string): Promise; /** Read a pack-root-relative text file, or null if absent. */ readText(rel: string): Promise; + /** Read a pack-root-relative file's bytes, or null if absent (decision 0045: tree copy for merge). */ + readFileBytes(rel: string): Promise; } function stripEvidence(base: string): string { @@ -110,6 +112,14 @@ export class DirectoryContainer implements PackContainer { async readText(rel: string): Promise { return readFileOrNull(path.join(this.root, rel)); } + + async readFileBytes(rel: string): Promise { + try { + return await fs.readFile(path.join(this.root, rel)); + } catch { + return null; + } + } } export class ZipContainer implements PackContainer { @@ -191,6 +201,11 @@ export class ZipContainer implements PackContainer { const e = this.zip.getEntry(rel); return e ? e.getData().toString("utf8") : null; } + + async readFileBytes(rel: string): Promise { + const e = this.zip.getEntry(rel); + return e ? e.getData() : null; + } } export async function openContainer(target: string): Promise { diff --git a/src/pack/remote.ts b/src/pack/remote.ts index 541a23a..ea9a456 100644 --- a/src/pack/remote.ts +++ b/src/pack/remote.ts @@ -281,6 +281,10 @@ export class RemoteZipContainer implements PackContainer { async readText(rel: string): Promise { return this.readEntryText(rel); } + + async readFileBytes(rel: string): Promise { + return this.readEntryBytes(rel); + } } /** diff --git a/src/schemas/0.1/L0/run.schema.json b/src/schemas/0.1/L0/run.schema.json index a441541..daea48a 100644 --- a/src/schemas/0.1/L0/run.schema.json +++ b/src/schemas/0.1/L0/run.schema.json @@ -75,6 +75,12 @@ "additionalProperties": true } }, + "merged_from": { + "description": "Present only on a pack produced by `evidence merge`: the source run_ids, in CLI order. The additive slice of deferred lineage (0014). See decision 0045.", + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + }, "environment": { "description": "Optional provenance/context block — an OPEN map of key:value pairs. The keys below are conventional, none is required, and any additional key:value pairs are allowed. A non-browser framework (unit, API) may legitimately omit model/surfaces — or the whole block — entirely. See decision 0013.", "type": "object", diff --git a/src/schemas/compile.test.ts b/src/schemas/compile.test.ts index 0da9468..af77598 100644 --- a/src/schemas/compile.test.ts +++ b/src/schemas/compile.test.ts @@ -32,6 +32,14 @@ describe("loadSchemas", () => { }); expect(ok).toBe(false); }); + + it("run.yaml accepts an optional merged_from list of non-empty strings", () => { + const { run } = loadSchemas("0.1", "L0"); + const base = { evidence: "0.1", run_id: "m", status: "running", title: "t", started: "2026-07-08T09:00:00Z" }; + expect(run({ ...base, merged_from: ["a", "b"] })).toBe(true); + expect(run({ ...base, merged_from: [] })).toBe(false); + expect(run({ ...base, merged_from: [""] })).toBe(false); + }); }); describe("loadL1Schemas", () => { From e2d34629629a935bdbd5b8399663481e235774de Mon Sep 17 00:00:00 2001 From: Siddhant Sinha Date: Wed, 8 Jul 2026 17:28:26 +0530 Subject: [PATCH 03/10] feat(0045): merge-rules schema + rules loader with pinned must-semantics Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BMuEZ1CZDLE9jjhmrNMNw6 --- src/merge/rules.test.ts | 59 ++++++++++++++++ src/merge/rules.ts | 101 ++++++++++++++++++++++++++++ src/schemas/merge-rules.schema.json | 63 +++++++++++++++++ 3 files changed, 223 insertions(+) create mode 100644 src/merge/rules.test.ts create mode 100644 src/merge/rules.ts create mode 100644 src/schemas/merge-rules.schema.json diff --git a/src/merge/rules.test.ts b/src/merge/rules.test.ts new file mode 100644 index 0000000..aff5630 --- /dev/null +++ b/src/merge/rules.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "vitest"; +import { promises as fs } from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { DEFAULT_RULES, loadRules, getKey, deepEqual, violates } from "./rules"; + +describe("loadRules", () => { + it("returns strict defaults when no path given", async () => { + expect(await loadRules()).toEqual(DEFAULT_RULES); + expect(DEFAULT_RULES.packs).toEqual({ require_status: "finalized", require_valid: "L0", on_ineligible: "abort" }); + expect(DEFAULT_RULES.tests).toEqual({ on_collision: "error" }); + expect(DEFAULT_RULES.rules).toEqual([]); + }); + + it("merges a partial file over defaults and validates shape", async () => { + const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "evi-rules-")); + const p = path.join(tmp, "r.yaml"); + await fs.writeFile(p, "tests: { on_collision: prefer_latest }\nrules:\n - { file: run.yaml, key: run_id, must: different, on_violation: skip }\n"); + const r = await loadRules(p); + expect(r.packs.on_ineligible).toBe("abort"); // default preserved + expect(r.tests.on_collision).toBe("prefer_latest"); + expect(r.rules).toHaveLength(1); + }); + + it("rejects bad YAML, bad shape, and scope/action mismatch as USAGE", async () => { + const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "evi-rules-")); + const cases = [ + "packs: [unclosed", // parse + "tests: { on_collision: newest }", // enum + "rules: [{ file: run.yaml, key: k, must: same, on_violation: prefer_latest }]", // pack rule w/ collision action + "rules: [{ file: result.yaml, key: k, must: same, on_violation: abort }]", // collision rule w/ pack action + ]; + for (const [i, c] of cases.entries()) { + const p = path.join(tmp, `bad-${i}.yaml`); + await fs.writeFile(p, c); + await expect(loadRules(p)).rejects.toMatchObject({ code: "USAGE" }); + } + }); +}); + +describe("helpers", () => { + it("getKey walks dot paths; undefined for absent", () => { + expect(getKey({ a: { b: 1 } }, "a.b")).toBe(1); + expect(getKey({ a: {} }, "a.b")).toBeUndefined(); + expect(getKey(null, "a")).toBeUndefined(); + }); + + it("deepEqual is canonical", () => { + expect(deepEqual({ x: 1, y: [1, 2] }, { y: [1, 2], x: 1 })).toBe(true); + expect(deepEqual([1, 2], [2, 1])).toBe(false); + }); + + it("violates pins absent-key semantics", () => { + expect(violates("same", undefined, undefined)).toBe(false); // absent==absent → same + expect(violates("same", undefined, "x")).toBe(true); // absent vs present → different + expect(violates("different", "x", "x")).toBe(true); + expect(violates("different", undefined, "x")).toBe(false); + }); +}); diff --git a/src/merge/rules.ts b/src/merge/rules.ts new file mode 100644 index 0000000..5605dbf --- /dev/null +++ b/src/merge/rules.ts @@ -0,0 +1,101 @@ +import { promises as fs } from "node:fs"; +import Ajv2020 from "ajv/dist/2020"; +import addFormats from "ajv-formats"; +import mergeRulesSchema from "../schemas/merge-rules.schema.json"; +import { parseYaml } from "../yaml"; + +export type PackAction = "abort" | "skip"; +export type CollisionAction = "error" | "prefer_first" | "prefer_latest" | "discard"; + +export interface KeyRule { + file: "run.yaml" | "result.yaml"; + key: string; + must: "same" | "different"; + on_violation: string; +} + +export interface MergeRules { + packs: { require_status: "finalized" | "running" | "any"; require_valid: "L0" | "L1" | "off"; on_ineligible: PackAction }; + tests: { on_collision: CollisionAction }; + rules: KeyRule[]; +} + +// Strict defaults (decision 0045): surprises abort, nothing is dropped silently. +export const DEFAULT_RULES: MergeRules = { + packs: { require_status: "finalized", require_valid: "L0", on_ineligible: "abort" }, + tests: { on_collision: "error" }, + rules: [], +}; + +// The rules file is our own tool input, so its schema is compiled once here — +// separate from the pack-contract caches in schemas/compile.ts. +const ajv = new Ajv2020({ allErrors: true, strict: false }); +addFormats(ajv); +const validateRules = ajv.compile(mergeRulesSchema as object); + +function usageErr(message: string): never { + const e = new Error(message) as Error & { code?: string }; + e.code = "USAGE"; + throw e; +} + +/** + * Load and validate a merge-rules.yaml, merged over DEFAULT_RULES. Any + * failure — unreadable file, bad YAML, schema violation — is a USAGE error + * (exit 2), raised before any pack is opened. + */ +export async function loadRules(rulesPath?: string): Promise { + if (!rulesPath) return DEFAULT_RULES; + let raw: string; + try { + raw = await fs.readFile(rulesPath, "utf8"); + } catch { + usageErr(`cannot read merge-rules file "${rulesPath}"`); + } + let doc: any; + try { + doc = parseYaml(raw); + } catch (e: any) { + usageErr(`merge-rules file is not valid YAML: ${e?.message ?? e}`); + } + if (!validateRules(doc)) { + const first = validateRules.errors?.[0]; + usageErr(`merge-rules.yaml${first?.instancePath ?? ""}: ${first?.message ?? "schema violation"}`); + } + return { + packs: { ...DEFAULT_RULES.packs, ...(doc as any).packs }, + tests: { ...DEFAULT_RULES.tests, ...(doc as any).tests }, + rules: (doc as any).rules ?? [], + }; +} + +/** Walk a dot-path into parsed YAML; undefined means the key is absent. */ +export function getKey(obj: unknown, dotPath: string): unknown { + return dotPath.split(".").reduce((o, k) => { + if (o == null || typeof o !== "object") return undefined; + return (o as Record)[k]; + }, obj); +} + +/** Canonical deep equality: key-order-insensitive objects, ordered arrays. */ +export function deepEqual(a: unknown, b: unknown): boolean { + if (Object.is(a, b)) return true; + if (Array.isArray(a) && Array.isArray(b)) { + return a.length === b.length && a.every((v, i) => deepEqual(v, b[i])); + } + if (a != null && b != null && typeof a === "object" && typeof b === "object" && !Array.isArray(a) && !Array.isArray(b)) { + const ka = Object.keys(a as object); + const kb = Object.keys(b as object); + return ka.length === kb.length && ka.every((k) => deepEqual((a as any)[k], (b as any)[k])); + } + return false; +} + +/** + * Does the pair (a, b) violate the predicate? Pins the absent-key semantics + * (decision 0045): absent == absent counts as same; absent vs present counts + * as different. + */ +export function violates(must: "same" | "different", a: unknown, b: unknown): boolean { + return must === "same" ? !deepEqual(a, b) : deepEqual(a, b); +} diff --git a/src/schemas/merge-rules.schema.json b/src/schemas/merge-rules.schema.json new file mode 100644 index 0000000..c1f6d72 --- /dev/null +++ b/src/schemas/merge-rules.schema.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://evidence-cli.dev/schemas/merge-rules.schema.json", + "title": "merge-rules.yaml", + "description": "The declarative policy passed to `evidence merge` (--rules). TOOL INPUT, not pack contract — it lives beside the versioned 0.1/ tree because it configures a command rather than shaping a pack. Fixed knobs gate packs and set the default collision policy; the flat `rules` list adds generic {file, key, must, on_violation} predicates. Scope is implied by `file`: run.yaml rules gate PACKS (compared across all eligible packs), result.yaml rules resolve COLLISIONS (compared between the two colliding tests). Unlike pack declarations, unknown keys are rejected — a typo'd knob must fail loudly. See decision 0045.", + "type": "object", + "properties": { + "packs": { + "type": "object", + "properties": { + "require_status": { + "description": "Run status every input pack must have. Default finalized — shards arrive sealed.", + "enum": ["finalized", "running", "any"] + }, + "require_valid": { + "description": "Validation profile every input pack must pass (any error diagnostic fails the gate), or off.", + "enum": ["L0", "L1", "off"] + }, + "on_ineligible": { + "description": "What an ineligible pack does to the merge: abort the whole merge, or skip the pack (recorded in the report).", + "enum": ["abort", "skip"] + } + }, + "additionalProperties": false + }, + "tests": { + "type": "object", + "properties": { + "on_collision": { + "description": "Default policy when two packs carry the same test id and no result.yaml rule fires. discard tombstones the id.", + "enum": ["error", "prefer_first", "prefer_latest", "discard"] + } + }, + "additionalProperties": false + }, + "rules": { + "description": "Generic key predicates, evaluated in file order. `must: same` anchors on the first eligible pack; `must: different` keeps the first occurrence. Absent==absent counts as same; absent vs present counts as different.", + "type": "array", + "items": { + "type": "object", + "required": ["file", "key", "must", "on_violation"], + "properties": { + "file": { "enum": ["run.yaml", "result.yaml"] }, + "key": { + "description": "Dot-path into the parsed YAML (e.g. environment.producer.name).", + "type": "string", + "minLength": 1 + }, + "must": { "enum": ["same", "different"] }, + "on_violation": { + "description": "Action on violation. Scope-checked by the conditional below: pack rules (file: run.yaml) take abort|skip; collision rules (file: result.yaml) take error|prefer_first|prefer_latest|discard.", + "type": "string" + } + }, + "if": { "properties": { "file": { "const": "run.yaml" } } }, + "then": { "properties": { "on_violation": { "enum": ["abort", "skip"] } } }, + "else": { "properties": { "on_violation": { "enum": ["error", "prefer_first", "prefer_latest", "discard"] } } }, + "additionalProperties": false + } + } + }, + "additionalProperties": false +} From 7adde853d885c53348cf01b91aa2fb8e7697c145 Mon Sep 17 00:00:00 2001 From: Siddhant Sinha Date: Wed, 8 Jul 2026 17:30:37 +0530 Subject: [PATCH 04/10] =?UTF-8?q?feat(0045):=20pack=20gating=20=E2=80=94?= =?UTF-8?q?=20ordered=20gates,=20sequential=20eligibility,=20abort/skip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BMuEZ1CZDLE9jjhmrNMNw6 --- src/merge/gates.test.ts | 95 ++++++++++++++++++++++++++++++++ src/merge/gates.ts | 116 ++++++++++++++++++++++++++++++++++++++++ src/merge/testkit.ts | 102 +++++++++++++++++++++++++++++++++++ 3 files changed, 313 insertions(+) create mode 100644 src/merge/gates.test.ts create mode 100644 src/merge/gates.ts create mode 100644 src/merge/testkit.ts diff --git a/src/merge/gates.test.ts b/src/merge/gates.test.ts new file mode 100644 index 0000000..fc90ef3 --- /dev/null +++ b/src/merge/gates.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from "vitest"; +import { gatePacks, sanitizeLabel } from "./gates"; +import { DEFAULT_RULES } from "./rules"; +import type { MergeRules } from "./rules"; +import { stagePack, sealCopy } from "./testkit"; + +const rules = (over: any = {}): MergeRules => ({ + ...DEFAULT_RULES, + packs: { ...DEFAULT_RULES.packs, ...over.packs }, + rules: over.rules ?? [], +}); + +describe("gatePacks", () => { + it("gates a running pack per require_status; skip records the rule", async () => { + const good = await sealCopy(await stagePack({ runId: "a" })); + const running = await stagePack({ runId: "b" }); + const r = await gatePacks([good, running], rules({ packs: { on_ineligible: "skip" } })); + expect(r.eligible.map((p) => p.run.run_id)).toEqual(["a"]); + expect(r.skipped[0]).toMatchObject({ runId: "b", rule: "packs.require_status=finalized" }); + }); + + it("abort throws with code ABORT", async () => { + const running = await stagePack({ runId: "b" }); + await expect(gatePacks([running], rules())).rejects.toMatchObject({ code: "ABORT" }); + }); + + it("same anchors on first ELIGIBLE pack; different dedupes; sequential eligibility", async () => { + const a = await sealCopy(await stagePack({ runId: "a", environment: { producer: { name: "kane" } } })); + const b = await sealCopy(await stagePack({ runId: "b", environment: { producer: { name: "other" } } })); + const a2 = await sealCopy(await stagePack({ runId: "a" })); // duplicate run_id + const r = await gatePacks( + [a, b, a2], + rules({ + packs: { on_ineligible: "skip" }, + rules: [ + { file: "run.yaml", key: "environment.producer.name", must: "same", on_violation: "skip" }, + { file: "run.yaml", key: "run_id", must: "different", on_violation: "skip" }, + ], + }), + ); + expect(r.eligible.map((p) => p.run.run_id)).toEqual(["a"]); // b: producer differs; a2: run_id repeats + expect(r.skipped.map((s) => s.runId).sort()).toEqual(["a", "b"]); + expect(r.eligible[0].label).toBe("1-a"); + }); + + it("zero eligible always errors; one eligible passes", async () => { + const running = await stagePack({ runId: "x" }); + await expect(gatePacks([running], rules({ packs: { on_ineligible: "skip" } }))).rejects.toMatchObject({ code: "ABORT" }); + const one = await sealCopy(await stagePack({ runId: "solo" })); + expect((await gatePacks([one], rules())).eligible).toHaveLength(1); + }); + + it("per-rule on_violation overrides global on_ineligible", async () => { + const a = await sealCopy(await stagePack({ runId: "a" })); + const a2 = await sealCopy(await stagePack({ runId: "a" })); + // global abort, but the run_id rule says skip → merge continues + const r = await gatePacks( + [a, a2], + rules({ rules: [{ file: "run.yaml", key: "run_id", must: "different", on_violation: "skip" }] }), + ); + expect(r.eligible).toHaveLength(1); + }); + + it("wrong contract version and unreadable manifest are gated", async () => { + const good = await sealCopy(await stagePack({ runId: "a" })); + const bad = await stagePack({ runId: "v2" }); + const fsMod = await import("node:fs/promises"); + const pathMod = await import("node:path"); + const raw = await fsMod.readFile(pathMod.join(bad, "run.yaml"), "utf8"); + await fsMod.writeFile(pathMod.join(bad, "run.yaml"), raw.replace('evidence: "0.1"', 'evidence: "0.2"')); + const r = await gatePacks([good, bad], rules({ packs: { on_ineligible: "skip" } })); + expect(r.eligible.map((p) => p.run.run_id)).toEqual(["a"]); + expect(r.skipped[0].rule).toBe("packs.version"); + }); + + it("require_valid gates an invalid pack", async () => { + const good = await sealCopy(await stagePack({ runId: "a" })); + const broken = await sealCopy(await stagePack({ runId: "b" })); + // corrupt the sealed pack? easier: stage a live pack with a bad result and relax status + const badLive = await stagePack({ runId: "c" }); + const fsMod = await import("node:fs/promises"); + const pathMod = await import("node:path"); + await fsMod.writeFile(pathMod.join(badLive, "tests", "checkout", "result.yaml"), 'evidence: "0.1"\ntest: MISMATCH\nstatus: passed\nsteps: []\n'); + const r = await gatePacks( + [good, broken, badLive], + rules({ packs: { require_status: "any", on_ineligible: "skip" } }), + ); + expect(r.eligible.map((p) => p.run.run_id)).toEqual(["a", "b"]); + expect(r.skipped[0]).toMatchObject({ runId: "c", rule: "packs.require_valid=L0" }); + }); +}); + +it("sanitizeLabel", () => { + expect(sanitizeLabel("run 2026/07")).toBe("run-2026-07"); +}); diff --git a/src/merge/gates.ts b/src/merge/gates.ts new file mode 100644 index 0000000..2e3c1ee --- /dev/null +++ b/src/merge/gates.ts @@ -0,0 +1,116 @@ +import * as path from "node:path"; +import { CONTRACT_VERSION } from "../contract"; +import { openContainer } from "../pack/container"; +import type { PackContainer } from "../pack/container"; +import { validate } from "../validate"; +import { parseYaml } from "../yaml"; +import { getKey, violates } from "./rules"; +import type { MergeRules, PackAction } from "./rules"; + +export interface EligiblePack { + inputPath: string; + container: PackContainer; + run: any; // parsed run.yaml + /** `${i}-${sanitizedRunId}`, 1-based over ELIGIBLE packs — the metrics/coverage namespace. */ + label: string; +} + +export interface SkippedPack { + runId: string; // falls back to the pack's base name when run.yaml is unreadable + rule: string; + reason: string; +} + +export function abortErr(message: string): Error & { code?: string } { + const e = new Error(message) as Error & { code?: string }; + e.code = "ABORT"; + return e; +} + +export function sanitizeLabel(runId: string): string { + return runId.replace(/[^A-Za-z0-9_.-]/g, "-"); +} + +/** + * Gate each input pack, in CLI order (decision 0045). Gates run cheap → + * expensive: manifest → version → require_status → require_valid → key rules. + * SEQUENTIAL ELIGIBILITY: rules compare a candidate only against packs that + * already survived every gate — an anchor from a pack that later fails is + * discarded. Zero eligible packs is always an error. + */ +export async function gatePacks( + inputs: string[], + rules: MergeRules, +): Promise<{ eligible: EligiblePack[]; skipped: SkippedPack[] }> { + const eligible: EligiblePack[] = []; + const skipped: SkippedPack[] = []; + const fail = (runId: string, rule: string, reason: string, action: PackAction): void => { + if (action === "abort") throw abortErr(`pack "${runId}": ${reason} [${rule}]`); + skipped.push({ runId, rule, reason }); + }; + + for (const inputPath of inputs) { + const fallback = path.basename(inputPath).replace(/\.evidence$/, ""); + let container: PackContainer; + try { + container = await openContainer(inputPath); + } catch { + fail(fallback, "packs.manifest", "unreadable pack", rules.packs.on_ineligible); + continue; + } + const raw = await container.readManifest(); + if (raw == null) { + fail(fallback, "packs.manifest", "no run.yaml", rules.packs.on_ineligible); + continue; + } + let run: any; + try { + run = parseYaml(raw); + } catch { + fail(fallback, "packs.manifest", "run.yaml does not parse", rules.packs.on_ineligible); + continue; + } + const runId = typeof run?.run_id === "string" ? run.run_id : fallback; + + if (run?.evidence !== CONTRACT_VERSION) { + fail(runId, "packs.version", `evidence ${JSON.stringify(run?.evidence)} != ${CONTRACT_VERSION}`, rules.packs.on_ineligible); + continue; + } + if (rules.packs.require_status !== "any" && run?.status !== rules.packs.require_status) { + fail(runId, `packs.require_status=${rules.packs.require_status}`, `status is ${run?.status}`, rules.packs.on_ineligible); + continue; + } + if (rules.packs.require_valid !== "off") { + const report = await validate(inputPath, { profile: rules.packs.require_valid }); + if (!report.valid) { + const errors = report.diagnostics.filter((d) => d.severity === "error").length; + fail(runId, `packs.require_valid=${rules.packs.require_valid}`, `${errors} validation error(s)`, rules.packs.on_ineligible); + continue; + } + } + + let violated = false; + for (const rule of rules.rules) { + if (rule.file !== "run.yaml") continue; + const mine = getKey(run, rule.key); + const clash = + rule.must === "same" + ? eligible.length > 0 && violates("same", getKey(eligible[0].run, rule.key), mine) + : eligible.some((p) => violates("different", getKey(p.run, rule.key), mine)); + if (clash) { + fail(runId, `run.yaml ${rule.key} must ${rule.must}`, `value ${JSON.stringify(mine)}`, rule.on_violation as PackAction); + violated = true; + break; + } + } + if (violated) continue; + + eligible.push({ inputPath, container, run, label: "" }); + } + + if (eligible.length === 0) throw abortErr("no eligible packs to merge"); + eligible.forEach((p, i) => { + p.label = `${i + 1}-${sanitizeLabel(p.run.run_id)}`; + }); + return { eligible, skipped }; +} diff --git a/src/merge/testkit.ts b/src/merge/testkit.ts new file mode 100644 index 0000000..0a7f99d --- /dev/null +++ b/src/merge/testkit.ts @@ -0,0 +1,102 @@ +import { promises as fs } from "node:fs"; +import { createHash } from "node:crypto"; +import * as os from "node:os"; +import * as path from "node:path"; +import { finalize } from "../finalize"; +import { parseYaml } from "../yaml"; + +/** + * Test staging helpers for the merge suite. stagePack builds a schema-valid + * LIVE pack (status: running) in a tmp dir; sealCopy runs the real finalize on + * a copy, using the staged run.yaml's `ended` (if any) as the seal timestamp. + */ +export interface StageTestSpec { + status?: string; // test verdict; the 2-pay step carries the same status + environment?: any; // result-level environment block + failure?: string; // content of steps/2-pay/failure.yaml +} + +export interface StagePackSpec { + runId: string; + status?: string; // default "running" (a live pack) + started?: string; + ended?: string; // consumed by sealCopy as finalize's endedAt + title?: string; + environment?: any; + metrics?: any; + l1?: boolean; // add logs/steps/coverage artifacts (L1 shape) + tests?: Record; +} + +const TEST_MD = "# Test\nA staged test definition.\n"; + +export async function stagePack(spec: StagePackSpec): Promise { + const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "evi-mergekit-")); + const dir = path.join(tmp, `${spec.runId}.evidence`); + const tests = spec.tests ?? { checkout: {} }; + + let run = `evidence: "0.1"\nrun_id: ${spec.runId}\nstatus: ${spec.status ?? "running"}\ntitle: ${spec.title ?? `t-${spec.runId}`}\nstarted: ${spec.started ?? "2026-07-08T08:00:00Z"}\n`; + if (spec.ended) run += `ended: ${spec.ended}\n`; + if (spec.environment) run += `environment: ${JSON.stringify(spec.environment)}\n`; + if (spec.metrics) run += `metrics: ${JSON.stringify(spec.metrics)}\n`; + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile(path.join(dir, "run.yaml"), run); + + for (const [id, t] of Object.entries(tests)) { + const testDir = path.join(dir, "tests", id); + await fs.mkdir(testDir, { recursive: true }); + await fs.writeFile(path.join(testDir, "test.md"), TEST_MD); + const status = t.status ?? "passed"; + let result = `evidence: "0.1"\ntest: ${id}\nstatus: ${status}\ndefinition:\n path: test.md\nsteps:\n - { id: open, ordinal: 1, status: passed }\n - { id: pay, ordinal: 2, status: ${status} }\n`; + if (t.environment) result += `environment: ${JSON.stringify(t.environment)}\n`; + await fs.writeFile(path.join(testDir, "result.yaml"), result); + + if (spec.l1) { + const logs = path.join(testDir, "logs"); + await fs.mkdir(logs, { recursive: true }); + await fs.writeFile(path.join(logs, "meta.yaml"), "logs:\n - { name: console, file: console.ndjson, format: ndjson }\n"); + await fs.writeFile(path.join(logs, "console.ndjson"), '{"level":"info"}\n'); + for (const step of ["1-open", "2-pay"]) { + const stepDir = path.join(testDir, "steps", step); + await fs.mkdir(stepDir, { recursive: true }); + await fs.writeFile(path.join(stepDir, "screenshot.png"), Buffer.from([0x89, 0x50, 0x4e, 0x47])); + } + } + if (t.failure) { + const stepDir = path.join(testDir, "steps", "2-pay"); + await fs.mkdir(stepDir, { recursive: true }); + await fs.writeFile(path.join(stepDir, "failure.yaml"), t.failure); + } + } + + if (spec.l1) { + const cov = path.join(dir, "coverage"); + await fs.mkdir(cov, { recursive: true }); + await fs.writeFile(path.join(cov, "lcov.info"), "TN:\nend_of_record\n"); + } + return dir; +} + +/** Seal a COPY of a staged live pack with the real finalize; returns the sealed zip path. */ +export async function sealCopy(dir: string): Promise { + const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "evi-mergekit-seal-")); + const copy = path.join(tmp, path.basename(dir)); + await fs.cp(dir, copy, { recursive: true }); + const run = parseYaml(await fs.readFile(path.join(copy, "run.yaml"), "utf8")) as any; + const endedAt = typeof run?.ended === "string" ? run.ended : "2026-07-08T09:01:00Z"; + await finalize(copy, { endedAt }); + return copy; // finalize sealed in place: the path is now a zip file +} + +/** Write a merge-rules.yaml into a tmp dir; returns its path. */ +export async function writeRules(yaml: string): Promise { + const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "evi-mergekit-rules-")); + const p = path.join(tmp, "merge-rules.yaml"); + await fs.writeFile(p, yaml); + return p; +} + +/** SHA-256 of the staged definition, for assertions on copied trees. */ +export function definitionSha(): string { + return `sha256:${createHash("sha256").update(TEST_MD).digest("hex")}`; +} From 538658a254f4445ffd9af0370cbdad632a264840 Mon Sep 17 00:00:00 2001 From: Siddhant Sinha Date: Wed, 8 Jul 2026 17:31:46 +0530 Subject: [PATCH 05/10] =?UTF-8?q?feat(0045):=20collision=20resolution=20?= =?UTF-8?q?=E2=80=94=20key=20rules,=20prefer/discard,=20tombstoning?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BMuEZ1CZDLE9jjhmrNMNw6 --- src/merge/collide.test.ts | 87 +++++++++++++++++++++++++++++++++++ src/merge/collide.ts | 97 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+) create mode 100644 src/merge/collide.test.ts create mode 100644 src/merge/collide.ts diff --git a/src/merge/collide.test.ts b/src/merge/collide.test.ts new file mode 100644 index 0000000..57c7e51 --- /dev/null +++ b/src/merge/collide.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; +import { resolveCollisions } from "./collide"; +import { DEFAULT_RULES } from "./rules"; +import type { MergeRules } from "./rules"; +import { stagePack, sealCopy } from "./testkit"; +import { gatePacks } from "./gates"; +import type { EligiblePack } from "./gates"; + +async function eligibleOf(...specs: any[]): Promise { + const paths: string[] = []; + for (const s of specs) paths.push(await sealCopy(await stagePack(s))); + return (await gatePacks(paths, DEFAULT_RULES)).eligible; +} + +const rulesWith = (tests: any, keyRules: any[] = []): MergeRules => ({ ...DEFAULT_RULES, tests, rules: keyRules }); + +describe("resolveCollisions", () => { + it("disjoint sets union; default collision errors", async () => { + const e = await eligibleOf({ runId: "a", tests: { login: {} } }, { runId: "b", tests: { checkout: {} } }); + const r = await resolveCollisions(e, DEFAULT_RULES); + expect(r.union.map((u) => u.testId).sort()).toEqual(["checkout", "login"]); + expect(r.collisions).toEqual([]); + expect(r.discarded).toEqual([]); + + const clash = await eligibleOf({ runId: "a", tests: { checkout: {} } }, { runId: "b", tests: { checkout: {} } }); + await expect(resolveCollisions(clash, DEFAULT_RULES)).rejects.toMatchObject({ code: "ABORT" }); + }); + + it("prefer_first / prefer_latest pick deterministically", async () => { + const e = await eligibleOf( + { runId: "a", ended: "2026-07-08T09:00:00Z", tests: { checkout: {} } }, + { runId: "b", ended: "2026-07-08T10:00:00Z", tests: { checkout: {} } }, + ); + expect((await resolveCollisions(e, rulesWith({ on_collision: "prefer_first" }))).union[0].source.run.run_id).toBe("a"); + const latest = await resolveCollisions(e, rulesWith({ on_collision: "prefer_latest" })); + expect(latest.union[0].source.run.run_id).toBe("b"); + expect(latest.collisions[0]).toMatchObject({ test: "checkout", winner: "b", rule: "tests.on_collision=prefer_latest" }); + }); + + it("prefer_latest tie falls back to the incumbent (CLI order)", async () => { + const e = await eligibleOf( + { runId: "a", ended: "2026-07-08T09:00:00Z", tests: { checkout: {} } }, + { runId: "b", ended: "2026-07-08T09:00:00Z", tests: { checkout: {} } }, + ); + const r = await resolveCollisions(e, rulesWith({ on_collision: "prefer_latest" })); + expect(r.union[0].source.run.run_id).toBe("a"); + }); + + it("discard tombstones across a 3rd pack", async () => { + const e = await eligibleOf( + { runId: "a", tests: { checkout: {} } }, + { runId: "b", tests: { checkout: {} } }, + { runId: "c", tests: { checkout: {} } }, + ); + const r = await resolveCollisions(e, rulesWith({ on_collision: "discard" })); + expect(r.union).toHaveLength(0); + expect(r.discarded).toEqual(["checkout"]); + }); + + it("result.yaml key rules fire before the default", async () => { + const e = await eligibleOf( + { runId: "a", ended: "2026-07-08T09:00:00Z", tests: { checkout: { status: "failed" } } }, + { runId: "b", ended: "2026-07-08T10:00:00Z", tests: { checkout: { status: "passed" } } }, + ); + // "if the two verdicts disagree, take the newer run's" + const r = await resolveCollisions( + e, + rulesWith({ on_collision: "error" }, [{ file: "result.yaml", key: "status", must: "same", on_violation: "prefer_latest" }]), + ); + expect(r.union[0].source.run.run_id).toBe("b"); + expect(r.collisions[0].rule).toBe("result.yaml status must same"); + }); + + it("a non-violated key rule falls through to the default", async () => { + const e = await eligibleOf( + { runId: "a", tests: { checkout: { status: "passed" } } }, + { runId: "b", tests: { checkout: { status: "passed" } } }, + ); + // statuses agree → rule does not fire → default prefer_first applies + const r = await resolveCollisions( + e, + rulesWith({ on_collision: "prefer_first" }, [{ file: "result.yaml", key: "status", must: "same", on_violation: "discard" }]), + ); + expect(r.union[0].source.run.run_id).toBe("a"); + expect(r.collisions[0].rule).toBe("tests.on_collision=prefer_first"); + }); +}); diff --git a/src/merge/collide.ts b/src/merge/collide.ts new file mode 100644 index 0000000..fea70ef --- /dev/null +++ b/src/merge/collide.ts @@ -0,0 +1,97 @@ +import { parseYaml } from "../yaml"; +import { abortErr } from "./gates"; +import type { EligiblePack } from "./gates"; +import { getKey, violates } from "./rules"; +import type { CollisionAction, MergeRules } from "./rules"; + +export interface UnionEntry { + testId: string; + source: EligiblePack; +} + +export interface CollisionRecord { + test: string; + winner: string; // run_id of the surviving copy ("" when discarded) + rule: string; // the deciding rule, e.g. "result.yaml status must same" or "tests.on_collision=prefer_latest" +} + +/** + * Union walk over the eligible packs, in CLI order (decision 0045). The first + * claimant of a test id is the INCUMBENT; later copies collide and resolve + * pairwise: result.yaml key rules in file order (first violated rule applies + * its action), else the default tests.on_collision. `discard` TOMBSTONES the + * id — a third pack's copy stays dropped. A test is atomic: the winner's whole + * tree travels later; nothing is mixed between copies. + */ +export async function resolveCollisions( + eligible: EligiblePack[], + rules: MergeRules, +): Promise<{ union: UnionEntry[]; collisions: CollisionRecord[]; discarded: string[] }> { + const claims = new Map(); + const tombstoned = new Set(); + const collisions: CollisionRecord[] = []; + + for (const pack of eligible) { + for (const testId of await pack.container.listTestIds()) { + if (tombstoned.has(testId)) continue; // discarded for good + const incumbent = claims.get(testId); + if (!incumbent) { + claims.set(testId, { testId, source: pack }); + continue; + } + + const [a, b] = await Promise.all([readResult(incumbent.source, testId), readResult(pack, testId)]); + let action: CollisionAction = rules.tests.on_collision; + let ruleLabel = `tests.on_collision=${action}`; + for (const rule of rules.rules) { + if (rule.file !== "result.yaml") continue; + if (violates(rule.must, getKey(a, rule.key), getKey(b, rule.key))) { + action = rule.on_violation as CollisionAction; + ruleLabel = `result.yaml ${rule.key} must ${rule.must}`; + break; + } + } + + switch (action) { + case "error": + throw abortErr(`test "${testId}" collides between packs "${incumbent.source.run.run_id}" and "${pack.run.run_id}" [${ruleLabel}]`); + case "prefer_first": + collisions.push({ test: testId, winner: incumbent.source.run.run_id, rule: ruleLabel }); + break; + case "prefer_latest": { + const winner = laterOf(incumbent.source, pack) === pack ? pack : incumbent.source; + if (winner === pack) claims.set(testId, { testId, source: pack }); + collisions.push({ test: testId, winner: winner.run.run_id, rule: ruleLabel }); + break; + } + case "discard": + claims.delete(testId); + tombstoned.add(testId); + collisions.push({ test: testId, winner: "", rule: ruleLabel }); + break; + } + } + } + + return { union: [...claims.values()], collisions, discarded: [...tombstoned].sort() }; +} + +async function readResult(pack: EligiblePack, testId: string): Promise { + const raw = await pack.container.readResult(testId); + if (raw == null) return undefined; + try { + return parseYaml(raw); + } catch { + return undefined; // an unparseable result compares as absent; require_valid normally catches this first + } +} + +/** The pack whose run ended later (fallback started); ties keep the incumbent. */ +function laterOf(incumbent: EligiblePack, challenger: EligiblePack): EligiblePack { + const stamp = (p: EligiblePack): number => { + const v = typeof p.run?.ended === "string" ? p.run.ended : p.run?.started; + const t = typeof v === "string" ? Date.parse(v) : NaN; + return Number.isNaN(t) ? -Infinity : t; + }; + return stamp(challenger) > stamp(incumbent) ? challenger : incumbent; +} From fb1ea84762a003c73c143008ade5db200309d97f Mon Sep 17 00:00:00 2001 From: Siddhant Sinha Date: Wed, 8 Jul 2026 17:33:21 +0530 Subject: [PATCH 06/10] =?UTF-8?q?feat(0045):=20assembly=20=E2=80=94=20run.?= =?UTF-8?q?yaml=20synthesis,=20env=20push-down,=20tree=20copy,=20coverage/?= =?UTF-8?q?metrics=20namespacing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BMuEZ1CZDLE9jjhmrNMNw6 --- src/merge/assemble.test.ts | 92 ++++++++++++++++++++++++++++++ src/merge/assemble.ts | 114 +++++++++++++++++++++++++++++++++++++ 2 files changed, 206 insertions(+) create mode 100644 src/merge/assemble.test.ts create mode 100644 src/merge/assemble.ts diff --git a/src/merge/assemble.test.ts b/src/merge/assemble.test.ts new file mode 100644 index 0000000..1f62c7c --- /dev/null +++ b/src/merge/assemble.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from "vitest"; +import { promises as fs } from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { assemble } from "./assemble"; +import { resolveCollisions } from "./collide"; +import { gatePacks } from "./gates"; +import { DEFAULT_RULES } from "./rules"; +import { stagePack, sealCopy } from "./testkit"; +import { parseYaml } from "../yaml"; +import { validate } from "../validate"; + +async function stagePair(): Promise<{ out: string; eligible: any[]; union: any[] }> { + const a = await sealCopy( + await stagePack({ + runId: "a", + started: "2026-07-08T08:00:00Z", + ended: "2026-07-08T09:00:00Z", + l1: true, + environment: { producer: { name: "kane" }, ci: { shard: "1" } }, + metrics: { requests: { value: 3, type: "count" } }, + tests: { login: {} }, + }), + ); + const b = await sealCopy( + await stagePack({ + runId: "b", + started: "2026-07-08T08:30:00Z", + ended: "2026-07-08T09:30:00Z", + l1: true, + environment: { producer: { name: "kane" }, ci: { shard: "2" } }, + tests: { checkout: { environment: { ci: { shard: "own" } } } }, + }), + ); + const { eligible } = await gatePacks([a, b], DEFAULT_RULES); + const { union } = await resolveCollisions(eligible, DEFAULT_RULES); + const out = path.join(await fs.mkdtemp(path.join(os.tmpdir(), "evi-asm-")), "m.evidence"); + return { out, eligible, union }; +} + +describe("assemble", () => { + it("synthesizes run.yaml, pushes divergent env down, namespaces coverage/metrics, copies whole trees", async () => { + const { out, eligible, union } = await stagePair(); + await assemble(out, { runId: "nightly" }, eligible, union); + + const run = parseYaml(await fs.readFile(path.join(out, "run.yaml"), "utf8")) as any; + expect(run).toMatchObject({ evidence: "0.1", run_id: "nightly", status: "running", title: "t-a" }); + expect(run.started).toBe("2026-07-08T08:00:00Z"); // min of sources + expect(run.merged_from).toEqual(["a", "b"]); + expect(run.ended).toBeUndefined(); + expect(run.totals).toBeUndefined(); + expect(run.metrics["1-a/requests"]).toEqual({ value: 3, type: "count" }); + expect(Object.keys(run.metrics)).toEqual(["1-a/requests"]); // b has none + expect(run.environment).toEqual({ producer: { name: "kane" } }); // common subset only + + // push-down: divergent ci landed per-test from each source pack + const login = parseYaml(await fs.readFile(path.join(out, "tests/login/result.yaml"), "utf8")) as any; + expect(login.environment.ci).toEqual({ shard: "1" }); // from pack a + // per-test value wins: checkout already carried its own ci + const checkout = parseYaml(await fs.readFile(path.join(out, "tests/checkout/result.yaml"), "utf8")) as any; + expect(checkout.environment.ci).toEqual({ shard: "own" }); + + // whole-tree copy: definition + logs travel byte-identical + expect(await fs.readFile(path.join(out, "tests/login/test.md"), "utf8")).toContain("A staged test definition"); + await fs.access(path.join(out, "tests/login/logs/console.ndjson")); + // coverage nesting; no root failure.yaml on the live merged pack + expect((await fs.stat(path.join(out, "coverage/1-a"))).isDirectory()).toBe(true); + expect((await fs.stat(path.join(out, "coverage/2-b"))).isDirectory()).toBe(true); + await expect(fs.access(path.join(out, "failure.yaml"))).rejects.toThrow(); + }); + + it("the assembled pack validates clean at L0 while running", async () => { + const { out, eligible, union } = await stagePair(); + await assemble(out, { runId: "nightly" }, eligible, union); + const report = await validate(out, { profile: "L0" }); + expect(report.valid).toBe(true); + expect(report.status).toBe("running"); + }); + + it("refuses to overwrite an existing output path (USAGE)", async () => { + const { out, eligible, union } = await stagePair(); + await fs.mkdir(out, { recursive: true }); + await expect(assemble(out, { runId: "nightly" }, eligible, union)).rejects.toMatchObject({ code: "USAGE" }); + }); + + it("--title overrides the first pack's title", async () => { + const { out, eligible, union } = await stagePair(); + await assemble(out, { runId: "nightly", title: "Nightly regression" }, eligible, union); + const run = parseYaml(await fs.readFile(path.join(out, "run.yaml"), "utf8")) as any; + expect(run.title).toBe("Nightly regression"); + }); +}); diff --git a/src/merge/assemble.ts b/src/merge/assemble.ts new file mode 100644 index 0000000..9c5ce1f --- /dev/null +++ b/src/merge/assemble.ts @@ -0,0 +1,114 @@ +import { promises as fs } from "node:fs"; +import * as path from "node:path"; +import { CONTRACT_VERSION } from "../contract"; +import type { PackContainer } from "../pack/container"; +import { parseDoc, parseYaml, stringifyDoc, stringifyYaml } from "../yaml"; +import type { EligiblePack } from "./gates"; +import type { UnionEntry } from "./collide"; +import { deepEqual, getKey } from "./rules"; + +export interface AssembleOptions { + runId: string; + title?: string; +} + +/** + * Write the live merged pack (decision 0045). Merge ASSEMBLES only: a + * synthesized run.yaml, the union winners' whole tests// trees, and each + * source's coverage/ nested under its label. NO derived artifacts — no totals, + * no ended, no root failure index; finalize regenerates them all. + */ +export async function assemble( + outDir: string, + opts: AssembleOptions, + eligible: EligiblePack[], + union: UnionEntry[], +): Promise { + try { + await fs.access(outDir); + const e = new Error(`output path "${outDir}" already exists`) as Error & { code?: string }; + e.code = "USAGE"; + throw e; + } catch (e: any) { + if (e?.code === "USAGE") throw e; // exists → refuse + } + await fs.mkdir(outDir, { recursive: true }); + + // Winners' whole trees + per-source coverage nesting. + for (const entry of union) { + await copyTree(entry.source.container, `tests/${entry.testId}`, path.join(outDir, "tests", entry.testId)); + } + for (const pack of eligible) { + if (await pack.container.isDir("coverage")) { + await copyTree(pack.container, "coverage", path.join(outDir, "coverage", pack.label)); + } + } + + // Environment: common subset stays run-level; divergent keys push down per + // test (0043's lossless merge). Per-test values win over pushed-down ones. + const envs = eligible.map((p) => (p.run?.environment && typeof p.run.environment === "object" ? p.run.environment : {})); + const commonEnv: Record = {}; + const divergent = new Set(); + for (const key of new Set(envs.flatMap((e) => Object.keys(e)))) { + const first = envs[0][key]; + if (envs.every((e) => key in e && deepEqual(e[key], first))) commonEnv[key] = first; + else divergent.add(key); + } + for (const entry of union) { + const sourceEnv = entry.source.run?.environment ?? {}; + const pushable = [...divergent].filter((k) => k in sourceEnv); + if (pushable.length === 0) continue; + const resultPath = path.join(outDir, "tests", entry.testId, "result.yaml"); + const raw = await fs.readFile(resultPath, "utf8"); + const parsed = parseYaml(raw); + const doc = parseDoc(raw); + let dirty = false; + for (const key of pushable) { + if (getKey(parsed, `environment.${key}`) !== undefined) continue; // per-test value wins + doc.setIn(["environment", key], sourceEnv[key]); + dirty = true; + } + if (dirty) await fs.writeFile(resultPath, stringifyDoc(doc), "utf8"); + } + + // Metrics: namespaced by flattening into the name —