Add end-to-end tests for skillz CLI using VHS

.devcontainer/Dockerfile

@@ -1,3 +1,7 @@
+# .NET 9 base. The .NET 10 SDK (required by global.json) and Docker (for the e2e
+# VHS recording) are layered in as devcontainer features — see devcontainer.json.
+# No bookworm-based .NET 10 image is published, so we add the SDK rather than
+# switch the distro.
 FROM mcr.microsoft.com/devcontainers/dotnet:1-9.0-bookworm
 
 RUN rm -f /etc/apt/sources.list.d/yarn.list \

.devcontainer/devcontainer-lock.json

@@ -5,6 +5,16 @@
       "resolved": "ghcr.io/devcontainers/features/common-utils@sha256:c42fdefe6d737a3a6f61cc52b23c7c9a565d08cc4d9c303669a7cf2ee5fd81fc",
       "integrity": "sha256:c42fdefe6d737a3a6f61cc52b23c7c9a565d08cc4d9c303669a7cf2ee5fd81fc"
     },
+    "ghcr.io/devcontainers/features/docker-in-docker:2": {
+      "version": "2.17.0",
+      "resolved": "ghcr.io/devcontainers/features/docker-in-docker@sha256:25b9f05705ffba7dbe503230ac76081419306f8c8bc88e0ce78c4ecd99a0c78c",
+      "integrity": "sha256:25b9f05705ffba7dbe503230ac76081419306f8c8bc88e0ce78c4ecd99a0c78c"
+    },
+    "ghcr.io/devcontainers/features/dotnet:2": {
+      "version": "2.5.0",
+      "resolved": "ghcr.io/devcontainers/features/dotnet@sha256:0fc16547ed4db6d7ff2a9f5981d2b93eb314e568affb9958029ad794f1f9a093",
+      "integrity": "sha256:0fc16547ed4db6d7ff2a9f5981d2b93eb314e568affb9958029ad794f1f9a093"
+    },
     "ghcr.io/devcontainers/features/github-cli:1": {
       "version": "1.1.0",
       "resolved": "ghcr.io/devcontainers/features/github-cli@sha256:d22f50b70ed75339b4eed1ba9ecde3a1791f90e88d37936517e3bace0bbad671",

.devcontainer/devcontainer.json

@@ -11,7 +11,18 @@
     "ghcr.io/devcontainers/features/common-utils:2": {
       "installZsh": true,
       "configureZshAsDefaultShell": true
-    }
+    },
+    // The base image (dotnet:1-9.0-bookworm) ships only .NET 9, but global.json
+    // pins SDK 10.0.100 and test/e2e/run.sh publishes -f net10.0. A bookworm-based
+    // .NET 10 image isn't published, so layer the SDK in here instead of changing
+    // the distro. .NET 9 stays available from the base for the net9.0 test projects.
+    "ghcr.io/devcontainers/features/dotnet:2": {
+      "version": "10.0"
+    },
+    // Docker engine inside the devcontainer so the e2e VHS recording
+    // (test/e2e/run.sh) can run the pinned VHS image locally, byte-for-byte
+    // identical to CI. Requires a host that permits privileged containers.
+    "ghcr.io/devcontainers/features/docker-in-docker:2": {}
   },
   "customizations": {
     "vscode": {
@@ -28,6 +39,6 @@
       }
     }
   },
-  "postCreateCommand": "dotnet --info && node --version && npm --version",
+  "postCreateCommand": "dotnet --info && node --version && npm --version && docker --version",
   "remoteUser": "vscode"
 }

.github/workflows/e2e-demo.yml

@@ -0,0 +1,152 @@
+name: e2e-demo
+
+on:
+  push:
+    paths:
+      - 'src/Skillz/**'
+      - 'test/e2e/**'
+      - 'test/fixtures/sample-skills/**'
+      - 'Directory.Build.props'
+      - 'Directory.Packages.props'
+      - 'global.json'
+      - '.github/workflows/e2e-demo.yml'
+  pull_request:
+    paths:
+      - 'src/Skillz/**'
+      - 'test/e2e/**'
+      - 'test/fixtures/sample-skills/**'
+      - 'Directory.Build.props'
+      - 'Directory.Packages.props'
+      - 'global.json'
+      - '.github/workflows/e2e-demo.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pull-requests: write
+
+concurrency:
+  group: e2e-demo-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  record-and-verify:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-dotnet@v4
+        with:
+          global-json-file: global.json
+
+      - name: Record + verify all flows
+        id: record
+        continue-on-error: true
+        run: ./test/e2e/run.sh
+
+      - name: Upload snapshot artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: skillz-e2e-snapshots
+          path: |
+            test/e2e/out/*.gif
+            test/e2e/out/report/**
+          if-no-files-found: warn
+
+      - name: Comment changed snapshots on PR
+        if: always() && github.event_name == 'pull_request'
+        uses: actions/github-script@v7
+        env:
+          RUN_ID: ${{ github.run_id }}
+        with:
+          script: |
+            const { owner, repo } = context.repo;
+            const pr = context.payload.pull_request;
+            const prNum = pr.number;
+            const baseRef = pr.base.ref;
+            const MARKER = '<!-- skillz-e2e-snapshots -->';
+            const bust = `?run=${process.env.RUN_ID}`;
+
+            const rawBase = `https://raw.githubusercontent.com/${pr.base.repo.full_name}/${pr.base.sha}`;
+            const headFull = pr.head.repo && pr.head.repo.full_name;
+            const rawHead = headFull
+              ? `https://raw.githubusercontent.com/${headFull}/${pr.head.sha}`
+              : null;
+
+            const files = await github.paginate(github.rest.pulls.listFiles,
+              { owner, repo, pull_number: prNum, per_page: 100 });
+            const re = /^test\/e2e\/([^/]+)-flow\.gif$/;
+            const changed = [];
+            for (const f of files) {
+              const m = f.filename.match(re);
+              if (m) changed.push(
+                { flow: m[1], status: f.status, path: f.filename, prev: f.previous_filename });
+            }
+            changed.sort((a, b) => a.flow.localeCompare(b.flow));
+
+            async function upsertComment(body) {
+              const { data: comments } = await github.rest.issues.listComments(
+                { owner, repo, issue_number: prNum, per_page: 100 });
+              const existing = comments.find(c => c.body && c.body.includes(MARKER));
+              if (existing) {
+                await github.rest.issues.updateComment(
+                  { owner, repo, comment_id: existing.id, body });
+              } else {
+                await github.rest.issues.createComment(
+                  { owner, repo, issue_number: prNum, body });
+              }
+            }
+
+            if (changed.length === 0) {
+              try {
+                await upsertComment(
+                  `${MARKER}\n✅ **E2E snapshots** — no flow recordings changed vs \`${baseRef}\`.`);
+              } catch (e) {
+                core.warning(`comment failed (expected on forks): ${e}`);
+              }
+              return;
+            }
+
+            const img = url => `<img src="${url}${bust}" width="480" alt="recording">`;
+            const ICON = { added: '🆕', removed: '🗑️' };
+
+            const blocks = [];
+            for (const { flow, status, path, prev } of changed) {
+              const before = `${rawBase}/${prev || path}`;
+              const after = rawHead ? `${rawHead}/${path}` : null;
+              let inner;
+              if (status === 'added') {
+                inner = after ? `\n\n${img(after)}\n` : '\n\n_after unavailable_\n';
+              } else if (status === 'removed') {
+                inner = `\n\n${img(before)}\n`;
+              } else {
+                const afterCell = after ? img(after) : '<em>unavailable</em>';
+                inner =
+                  `\n\n<table><tr><th>Before — <code>${baseRef}</code></th><th>After — this PR</th></tr>`
+                  + `<tr><td>${img(before)}</td><td>${afterCell}</td></tr></table>\n`;
+              }
+              blocks.push(
+                `<details><summary>${ICON[status] || '🔁'} ${flow} — ${status}</summary>${inner}</details>`);
+            }
+
+            const body = [
+              MARKER,
+              '### 🎬 E2E snapshots changed',
+              '',
+              `These flow recordings differ from \`${baseRef}\` (added, removed, or modified). `
+                + 'Expand each for the before/after.',
+              '',
+              ...blocks,
+              '',
+              `<sub>Run ${process.env.RUN_ID} · full set in the <b>skillz-e2e-snapshots</b> artifact.</sub>`,
+            ].join('\n');
+
+            try { await upsertComment(body); }
+            catch (e) { core.warning(`comment failed (expected on forks): ${e}`); }
+
+      - name: Fail if any flow drifted from its golden
+        if: always() && steps.record.outcome == 'failure'
+        run: |
+          echo "::error::E2E recordings drifted from their committed golden — review the skillz-e2e-snapshots artifact, then run ./test/e2e/run.sh --update <flow> if intended." >&2
+          exit 1

test/e2e/.gitignore

@@ -0,0 +1,4 @@
+# Published binary (74 MB, regenerated by run.sh)
+bin/
+# Raw VHS recordings + extracted frame + screenshots (regenerated by run.sh)
+out/

test/e2e/README.md

@@ -0,0 +1,130 @@
+# End-to-end terminal recordings (VHS)
+
+This directory drives the **real `skillz` binary through its TUI** — one
+[VHS](https://github.com/charmbracelet/vhs) tape per command flow — and produces
+two artifacts per flow from a single recording:
+
+| Artifact | Purpose |
+| --- | --- |
+| `<flow>-flow.gif` | Animated demo for PRs / README |
+| `<flow>-flow.golden.txt` | Final-frame text snapshot, diffed in CI as an integration test |
+
+![skillz add — interactive flow](add-flow.gif)
+
+The same recording is both the demo *and* the assertion: one tape, one run.
+
+## Why a recording (we already have snapshot tests)
+
+The in-process tests (`Skillz.Tests`) drive the prompts through Spectre's
+`TestConsole` — fast and deterministic, but they never exercise the *compiled*
+binary: `Program.cs`, DI wiring, `System.CommandLine` parsing, or real terminal
+rendering. This tier does, end to end, through a real PTY. It is the slow,
+high-confidence layer — a handful of representative flows, not exhaustive.
+
+## The flows
+
+| Flow | Drives | Validates |
+| --- | --- | --- |
+| `add` | interactive: skill multi-select → searchable agent picker → scope → method → confirm | the full interactive install, **symlink** path |
+| `copy` | `add --agent claude-code --copy --skill … -y` | non-interactive install, **copy** path + `Copied:` summary |
+| `global` | `add --agent claude-code --global --skill … -y` | **global scope** (`$HOME`-rooted paths) |
+| `init` | `init my-skill` | skill scaffolding + next-steps output |
+| `list` | `list` (after a hidden install) | the installed-skills table |
+| `remove` | interactive: multi-select → y/n confirm → summary | **interactive removal** + lock/symlink cleanup |
+| `update` | interactive scope picker → Both | the update check (hermetic: local skills are never network-checked) |
+| `error` | `add ./missing` | the **failure UX**: error message + non-zero exit |
+
+Each flow's tape is the source of truth; keep `MARKERS`/`ALL_FLOWS` in
+[`run.sh`](run.sh) in sync with the tape set.
+
+## How it works
+
+```
+<flow>-flow.tape ──▶ VHS container (ttyd + ffmpeg) ──▶ <flow>-flow.gif
+                                                   └──▶ <flow>-flow.txt ──▶ extract-frame.sh ──▶ diff vs golden
+```
+
+1. **`<flow>-flow.tape`** is a VHS script. Interactive flows gate key transitions
+   on `Wait+Screen /.../` sentinels, so the recording syncs on **state** rather
+   than wall-clock timing (short `Sleep`s are still used to let the UI settle).
+   A hidden setup block puts the published binary on `PATH`, works in a throwaway
+   `/tmp/work`, and (for `list`/`remove`/`update`) pre-installs fixtures so the
+   demo has real state.
+2. **`run.sh`** publishes a self-contained `linux-x64` binary once, then for each
+   flow mounts the repo read-only into the pinned VHS container and records.
+3. **`extract-frame.sh`** reduces VHS's multi-frame `.txt` capture to the final
+   completed frame, keyed on a per-flow marker (e.g. `Done!`, `Successfully removed`).
+4. Each frame is diffed against `<flow>-flow.golden.txt`.
+
+## Running it
+
+```bash
+./test/e2e/run.sh                 # record + verify EVERY flow (what CI does)
+./test/e2e/run.sh remove update   # record + verify only the named flows
+./test/e2e/run.sh --update        # accept new output: refresh all goldens + GIFs
+./test/e2e/run.sh --update init   # refresh a single flow
+REBUILD=1 ./test/e2e/run.sh       # force re-publish of the binary first
+```
+
+`run.sh` exits non-zero if any flow's frame differs from its golden (**FAIL**) or
+has no golden yet (**NEW**), and collects the changed/new GIFs, frames, and diffs
+under `out/report/` for CI.
+
+Requirements: `docker` + the .NET SDK. Nothing else — `ttyd`, `ffmpeg`, and the
+fonts are baked into the pinned container.
+
+**In this repo's devcontainer**, both are provided as features (Docker via
+`docker-in-docker`, .NET 10 via the `dotnet` feature) — run **Dev Containers:
+Rebuild Container** once, then `./test/e2e/run.sh` works as above. Docker-in-Docker
+needs a host that permits privileged containers.
+
+## What makes it deterministic
+
+Two independent runs produce a **byte-identical** final frame. The levers:
+
+- **Assert on text, never on the GIF.** GIF bytes go through ffmpeg/gifski and are
+  not stable across versions/platforms. The character grid (`.txt`) is.
+- **Final frame only.** Intermediate frames vary with timing; the end state does not.
+- **Pinned VHS image** (by digest) — a new VHS release can't silently reflow output.
+- **Fixed geometry / theme / `CursorBlink false`** in every tape.
+- **Hermetic inputs**: a local fixture (no network), a fixed `/tmp/work` cwd, and a
+  pinned `$HOME` where it appears in output, so every path is constant. A clean
+  container has no agent env (`AI_AGENT`, `CLAUDECODE`, …) and no agent config, so
+  skillz renders the real interactive prompts with stable defaults.
+- **One skill where order matters.** The install report lists skills in discovery
+  order (filesystem-dependent), so the non-interactive `copy`/`global`/`list` flows
+  pin a single skill with `--skill`. `remove` lists skills sorted, and `update`
+  never enumerates local skills, so those use all three fixtures.
+
+If skillz legitimately changes its output (e.g. a new universal agent), the diff
+fails — that's the test working. Re-run with `--update <flow>` and commit the new
+golden + GIF.
+
+## CI: artifacts + PR comment
+
+The [`e2e-demo`](../../.github/workflows/e2e-demo.yml) workflow records and verifies
+every flow on pushes and PRs that touch the CLI, the tapes, or the fixtures, and:
+
+- **Uploads** all GIFs plus `out/report/**` (changed frames, diffs, per-flow status)
+  as the `skillz-e2e-snapshots` build artifact.
+- On a **PR**, posts a single collapsed comment — one expandable `<details>` per
+  flow that **changed** (🔴) or is **new** (🆕) — with the recording inline. The
+  GIFs are hosted on an `e2e-snapshots` side branch so GitHub renders them in the
+  comment. When everything matches, the comment resets to a ✅ line.
+- **Fails the job** (after uploading + commenting) if any flow changed or is new.
+
+> Inline hosting needs write access, so it is skipped for **fork** PRs (the GIFs
+> are still in the artifact). Same-repo PRs get the inline previews.
+
+## Files
+
+| File | |
+| --- | --- |
+| `<flow>-flow.tape` | The VHS script for a flow (source of truth) |
+| `<flow>-flow.golden.txt` | Committed final-frame snapshot |
+| `<flow>-flow.gif` | Committed demo (regenerate with `run.sh --update <flow>`) |
+| `run.sh` | Publish → record each flow → extract → verify/update + build report |
+| `extract-frame.sh` | VHS `.txt` → final frame (per-flow marker) |
+| `bin/`, `out/` | gitignored: published binary, raw recordings, report |
+
+Fixture skills live in [`../fixtures/sample-skills`](../fixtures/sample-skills).

test/e2e/add-flow.golden.txt

@@ -0,0 +1,32 @@
+$ skillz add ./sample-skills
+
+  /$$$$$$  /$$       /$$ /$$ /$$
+ /$$__  $$| $$      |__/| $$| $$
+| $$  \__/| $$   /$$ /$$| $$| $$ /$$$$$$$$
+|  $$$$$$ | $$  /$$/| $$| $$| $$|____ /$$/
+ \____  $$| $$$$$$/ | $$| $$| $$   /$$$$/
+ /$$  \ $$| $$_  $$ | $$| $$| $$  /$$__/
+|  $$$$$$/| $$ \  $$| $$| $$| $$ /$$$$$$$$
+ \______/ |__/  \__/|__/|__/|__/|________/
+Source: /tmp/work/sample-skills
+Found 3 skill(s)
+Which agents do you want to install to? Selected: Claude Code (claude-code)
+Install 2 skill(s) [commit-helper, pr-describer] to 16 agent(s) [amp, antigravity, claude-code,
+cline, codex, cursor, deepagents, dexto, firebender, gemini-cli, github-copilot, kimi-cli,
+opencode, replit, universal, warp] [y/n] (y): y
+
+┌─Installation Summary────────────────────────────────────────────────────────────────────────┐
+│ Canonical: /tmp/work/.agents/skills                                                         │
+│ Universal:  Amp, Antigravity, Cline, Codex, Cursor, Deep Agents, Dexto, Firebender, Gemini  │
+│ CLI, GitHub Copilot, Kimi Code CLI, OpenCode, Replit, Universal, Warp                       │
+│ Symlinked:  Claude Code                                                                     │
+└─────────────────────────────────────────────────────────────────────────────────────────────┘
+┌─Installed 2 skill(s)────────────────────────────────────────────────────────────────────────┐
+│ ✓ commit-helper                                                                             │
+│   → /tmp/work/.agents/skills/commit-helper                                                  │
+│ ✓ pr-describer                                                                              │
+│   → /tmp/work/.agents/skills/pr-describer                                                   │
+└─────────────────────────────────────────────────────────────────────────────────────────────┘
+
+Done!  Review skills before use; they run with full agent permissions.
+$