diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index f699cfa..9bb8250 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,3 +1,7 @@ +# .NET 9 base. The .NET 10 SDK (required by global.json) and Docker (for the e2e +# VHS recording) are layered in as devcontainer features β€” see devcontainer.json. +# No bookworm-based .NET 10 image is published, so we add the SDK rather than +# switch the distro. FROM mcr.microsoft.com/devcontainers/dotnet:1-9.0-bookworm RUN rm -f /etc/apt/sources.list.d/yarn.list \ diff --git a/.devcontainer/devcontainer-lock.json b/.devcontainer/devcontainer-lock.json index 7c6ffc0..53cf565 100644 --- a/.devcontainer/devcontainer-lock.json +++ b/.devcontainer/devcontainer-lock.json @@ -5,6 +5,16 @@ "resolved": "ghcr.io/devcontainers/features/common-utils@sha256:c42fdefe6d737a3a6f61cc52b23c7c9a565d08cc4d9c303669a7cf2ee5fd81fc", "integrity": "sha256:c42fdefe6d737a3a6f61cc52b23c7c9a565d08cc4d9c303669a7cf2ee5fd81fc" }, + "ghcr.io/devcontainers/features/docker-in-docker:2": { + "version": "2.17.0", + "resolved": "ghcr.io/devcontainers/features/docker-in-docker@sha256:25b9f05705ffba7dbe503230ac76081419306f8c8bc88e0ce78c4ecd99a0c78c", + "integrity": "sha256:25b9f05705ffba7dbe503230ac76081419306f8c8bc88e0ce78c4ecd99a0c78c" + }, + "ghcr.io/devcontainers/features/dotnet:2": { + "version": "2.5.0", + "resolved": "ghcr.io/devcontainers/features/dotnet@sha256:0fc16547ed4db6d7ff2a9f5981d2b93eb314e568affb9958029ad794f1f9a093", + "integrity": "sha256:0fc16547ed4db6d7ff2a9f5981d2b93eb314e568affb9958029ad794f1f9a093" + }, "ghcr.io/devcontainers/features/github-cli:1": { "version": "1.1.0", "resolved": "ghcr.io/devcontainers/features/github-cli@sha256:d22f50b70ed75339b4eed1ba9ecde3a1791f90e88d37936517e3bace0bbad671", diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 9202f73..b77f3ab 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -11,7 +11,18 @@ "ghcr.io/devcontainers/features/common-utils:2": { "installZsh": true, "configureZshAsDefaultShell": true - } + }, + // The base image (dotnet:1-9.0-bookworm) ships only .NET 9, but global.json + // pins SDK 10.0.100 and test/e2e/run.sh publishes -f net10.0. A bookworm-based + // .NET 10 image isn't published, so layer the SDK in here instead of changing + // the distro. .NET 9 stays available from the base for the net9.0 test projects. + "ghcr.io/devcontainers/features/dotnet:2": { + "version": "10.0" + }, + // Docker engine inside the devcontainer so the e2e VHS recording + // (test/e2e/run.sh) can run the pinned VHS image locally, byte-for-byte + // identical to CI. Requires a host that permits privileged containers. + "ghcr.io/devcontainers/features/docker-in-docker:2": {} }, "customizations": { "vscode": { @@ -28,6 +39,6 @@ } } }, - "postCreateCommand": "dotnet --info && node --version && npm --version", + "postCreateCommand": "dotnet --info && node --version && npm --version && docker --version", "remoteUser": "vscode" } diff --git a/.github/workflows/e2e-demo.yml b/.github/workflows/e2e-demo.yml new file mode 100644 index 0000000..6a62da8 --- /dev/null +++ b/.github/workflows/e2e-demo.yml @@ -0,0 +1,152 @@ +name: e2e-demo + +on: + push: + paths: + - 'src/Skillz/**' + - 'test/e2e/**' + - 'test/fixtures/sample-skills/**' + - 'Directory.Build.props' + - 'Directory.Packages.props' + - 'global.json' + - '.github/workflows/e2e-demo.yml' + pull_request: + paths: + - 'src/Skillz/**' + - 'test/e2e/**' + - 'test/fixtures/sample-skills/**' + - 'Directory.Build.props' + - 'Directory.Packages.props' + - 'global.json' + - '.github/workflows/e2e-demo.yml' + workflow_dispatch: + +permissions: + contents: read + pull-requests: write + +concurrency: + group: e2e-demo-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + record-and-verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-dotnet@v4 + with: + global-json-file: global.json + + - name: Record + verify all flows + id: record + continue-on-error: true + run: ./test/e2e/run.sh + + - name: Upload snapshot artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: skillz-e2e-snapshots + path: | + test/e2e/out/*.gif + test/e2e/out/report/** + if-no-files-found: warn + + - name: Comment changed snapshots on PR + if: always() && github.event_name == 'pull_request' + uses: actions/github-script@v7 + env: + RUN_ID: ${{ github.run_id }} + with: + script: | + const { owner, repo } = context.repo; + const pr = context.payload.pull_request; + const prNum = pr.number; + const baseRef = pr.base.ref; + const MARKER = ''; + const bust = `?run=${process.env.RUN_ID}`; + + const rawBase = `https://raw.githubusercontent.com/${pr.base.repo.full_name}/${pr.base.sha}`; + const headFull = pr.head.repo && pr.head.repo.full_name; + const rawHead = headFull + ? `https://raw.githubusercontent.com/${headFull}/${pr.head.sha}` + : null; + + const files = await github.paginate(github.rest.pulls.listFiles, + { owner, repo, pull_number: prNum, per_page: 100 }); + const re = /^test\/e2e\/([^/]+)-flow\.gif$/; + const changed = []; + for (const f of files) { + const m = f.filename.match(re); + if (m) changed.push( + { flow: m[1], status: f.status, path: f.filename, prev: f.previous_filename }); + } + changed.sort((a, b) => a.flow.localeCompare(b.flow)); + + async function upsertComment(body) { + const { data: comments } = await github.rest.issues.listComments( + { owner, repo, issue_number: prNum, per_page: 100 }); + const existing = comments.find(c => c.body && c.body.includes(MARKER)); + if (existing) { + await github.rest.issues.updateComment( + { owner, repo, comment_id: existing.id, body }); + } else { + await github.rest.issues.createComment( + { owner, repo, issue_number: prNum, body }); + } + } + + if (changed.length === 0) { + try { + await upsertComment( + `${MARKER}\nβœ… **E2E snapshots** β€” no flow recordings changed vs \`${baseRef}\`.`); + } catch (e) { + core.warning(`comment failed (expected on forks): ${e}`); + } + return; + } + + const img = url => `recording`; + const ICON = { added: 'πŸ†•', removed: 'πŸ—‘οΈ' }; + + const blocks = []; + for (const { flow, status, path, prev } of changed) { + const before = `${rawBase}/${prev || path}`; + const after = rawHead ? `${rawHead}/${path}` : null; + let inner; + if (status === 'added') { + inner = after ? `\n\n${img(after)}\n` : '\n\n_after unavailable_\n'; + } else if (status === 'removed') { + inner = `\n\n${img(before)}\n`; + } else { + const afterCell = after ? img(after) : 'unavailable'; + inner = + `\n\n` + + `
Before β€” ${baseRef}After β€” this PR
${img(before)}${afterCell}
\n`; + } + blocks.push( + `
${ICON[status] || 'πŸ”'} ${flow} β€” ${status}${inner}
`); + } + + const body = [ + MARKER, + '### 🎬 E2E snapshots changed', + '', + `These flow recordings differ from \`${baseRef}\` (added, removed, or modified). ` + + 'Expand each for the before/after.', + '', + ...blocks, + '', + `Run ${process.env.RUN_ID} Β· full set in the skillz-e2e-snapshots artifact.`, + ].join('\n'); + + try { await upsertComment(body); } + catch (e) { core.warning(`comment failed (expected on forks): ${e}`); } + + - name: Fail if any flow drifted from its golden + if: always() && steps.record.outcome == 'failure' + run: | + echo "::error::E2E recordings drifted from their committed golden β€” review the skillz-e2e-snapshots artifact, then run ./test/e2e/run.sh --update if intended." >&2 + exit 1 diff --git a/test/e2e/.gitignore b/test/e2e/.gitignore new file mode 100644 index 0000000..98718bc --- /dev/null +++ b/test/e2e/.gitignore @@ -0,0 +1,4 @@ +# Published binary (74 MB, regenerated by run.sh) +bin/ +# Raw VHS recordings + extracted frame + screenshots (regenerated by run.sh) +out/ diff --git a/test/e2e/README.md b/test/e2e/README.md new file mode 100644 index 0000000..f9a8501 --- /dev/null +++ b/test/e2e/README.md @@ -0,0 +1,130 @@ +# End-to-end terminal recordings (VHS) + +This directory drives the **real `skillz` binary through its TUI** β€” one +[VHS](https://github.com/charmbracelet/vhs) tape per command flow β€” and produces +two artifacts per flow from a single recording: + +| Artifact | Purpose | +| --- | --- | +| `-flow.gif` | Animated demo for PRs / README | +| `-flow.golden.txt` | Final-frame text snapshot, diffed in CI as an integration test | + +![skillz add β€” interactive flow](add-flow.gif) + +The same recording is both the demo *and* the assertion: one tape, one run. + +## Why a recording (we already have snapshot tests) + +The in-process tests (`Skillz.Tests`) drive the prompts through Spectre's +`TestConsole` β€” fast and deterministic, but they never exercise the *compiled* +binary: `Program.cs`, DI wiring, `System.CommandLine` parsing, or real terminal +rendering. This tier does, end to end, through a real PTY. It is the slow, +high-confidence layer β€” a handful of representative flows, not exhaustive. + +## The flows + +| Flow | Drives | Validates | +| --- | --- | --- | +| `add` | interactive: skill multi-select β†’ searchable agent picker β†’ scope β†’ method β†’ confirm | the full interactive install, **symlink** path | +| `copy` | `add --agent claude-code --copy --skill … -y` | non-interactive install, **copy** path + `Copied:` summary | +| `global` | `add --agent claude-code --global --skill … -y` | **global scope** (`$HOME`-rooted paths) | +| `init` | `init my-skill` | skill scaffolding + next-steps output | +| `list` | `list` (after a hidden install) | the installed-skills table | +| `remove` | interactive: multi-select β†’ y/n confirm β†’ summary | **interactive removal** + lock/symlink cleanup | +| `update` | interactive scope picker β†’ Both | the update check (hermetic: local skills are never network-checked) | +| `error` | `add ./missing` | the **failure UX**: error message + non-zero exit | + +Each flow's tape is the source of truth; keep `MARKERS`/`ALL_FLOWS` in +[`run.sh`](run.sh) in sync with the tape set. + +## How it works + +``` +-flow.tape ──▢ VHS container (ttyd + ffmpeg) ──▢ -flow.gif + └──▢ -flow.txt ──▢ extract-frame.sh ──▢ diff vs golden +``` + +1. **`-flow.tape`** is a VHS script. Interactive flows gate key transitions + on `Wait+Screen /.../` sentinels, so the recording syncs on **state** rather + than wall-clock timing (short `Sleep`s are still used to let the UI settle). + A hidden setup block puts the published binary on `PATH`, works in a throwaway + `/tmp/work`, and (for `list`/`remove`/`update`) pre-installs fixtures so the + demo has real state. +2. **`run.sh`** publishes a self-contained `linux-x64` binary once, then for each + flow mounts the repo read-only into the pinned VHS container and records. +3. **`extract-frame.sh`** reduces VHS's multi-frame `.txt` capture to the final + completed frame, keyed on a per-flow marker (e.g. `Done!`, `Successfully removed`). +4. Each frame is diffed against `-flow.golden.txt`. + +## Running it + +```bash +./test/e2e/run.sh # record + verify EVERY flow (what CI does) +./test/e2e/run.sh remove update # record + verify only the named flows +./test/e2e/run.sh --update # accept new output: refresh all goldens + GIFs +./test/e2e/run.sh --update init # refresh a single flow +REBUILD=1 ./test/e2e/run.sh # force re-publish of the binary first +``` + +`run.sh` exits non-zero if any flow's frame differs from its golden (**FAIL**) or +has no golden yet (**NEW**), and collects the changed/new GIFs, frames, and diffs +under `out/report/` for CI. + +Requirements: `docker` + the .NET SDK. Nothing else β€” `ttyd`, `ffmpeg`, and the +fonts are baked into the pinned container. + +**In this repo's devcontainer**, both are provided as features (Docker via +`docker-in-docker`, .NET 10 via the `dotnet` feature) β€” run **Dev Containers: +Rebuild Container** once, then `./test/e2e/run.sh` works as above. Docker-in-Docker +needs a host that permits privileged containers. + +## What makes it deterministic + +Two independent runs produce a **byte-identical** final frame. The levers: + +- **Assert on text, never on the GIF.** GIF bytes go through ffmpeg/gifski and are + not stable across versions/platforms. The character grid (`.txt`) is. +- **Final frame only.** Intermediate frames vary with timing; the end state does not. +- **Pinned VHS image** (by digest) β€” a new VHS release can't silently reflow output. +- **Fixed geometry / theme / `CursorBlink false`** in every tape. +- **Hermetic inputs**: a local fixture (no network), a fixed `/tmp/work` cwd, and a + pinned `$HOME` where it appears in output, so every path is constant. A clean + container has no agent env (`AI_AGENT`, `CLAUDECODE`, …) and no agent config, so + skillz renders the real interactive prompts with stable defaults. +- **One skill where order matters.** The install report lists skills in discovery + order (filesystem-dependent), so the non-interactive `copy`/`global`/`list` flows + pin a single skill with `--skill`. `remove` lists skills sorted, and `update` + never enumerates local skills, so those use all three fixtures. + +If skillz legitimately changes its output (e.g. a new universal agent), the diff +fails β€” that's the test working. Re-run with `--update ` and commit the new +golden + GIF. + +## CI: artifacts + PR comment + +The [`e2e-demo`](../../.github/workflows/e2e-demo.yml) workflow records and verifies +every flow on pushes and PRs that touch the CLI, the tapes, or the fixtures, and: + +- **Uploads** all GIFs plus `out/report/**` (changed frames, diffs, per-flow status) + as the `skillz-e2e-snapshots` build artifact. +- On a **PR**, posts a single collapsed comment β€” one expandable `
` per + flow that **changed** (πŸ”΄) or is **new** (πŸ†•) β€” with the recording inline. The + GIFs are hosted on an `e2e-snapshots` side branch so GitHub renders them in the + comment. When everything matches, the comment resets to a βœ… line. +- **Fails the job** (after uploading + commenting) if any flow changed or is new. + +> Inline hosting needs write access, so it is skipped for **fork** PRs (the GIFs +> are still in the artifact). Same-repo PRs get the inline previews. + +## Files + +| File | | +| --- | --- | +| `-flow.tape` | The VHS script for a flow (source of truth) | +| `-flow.golden.txt` | Committed final-frame snapshot | +| `-flow.gif` | Committed demo (regenerate with `run.sh --update `) | +| `run.sh` | Publish β†’ record each flow β†’ extract β†’ verify/update + build report | +| `extract-frame.sh` | VHS `.txt` β†’ final frame (per-flow marker) | +| `bin/`, `out/` | gitignored: published binary, raw recordings, report | + +Fixture skills live in [`../fixtures/sample-skills`](../fixtures/sample-skills). diff --git a/test/e2e/add-flow.gif b/test/e2e/add-flow.gif new file mode 100644 index 0000000..3aed34c Binary files /dev/null and b/test/e2e/add-flow.gif differ diff --git a/test/e2e/add-flow.golden.txt b/test/e2e/add-flow.golden.txt new file mode 100644 index 0000000..1739051 --- /dev/null +++ b/test/e2e/add-flow.golden.txt @@ -0,0 +1,32 @@ +$ skillz add ./sample-skills + + /$$$$$$ /$$ /$$ /$$ /$$ + /$$__ $$| $$ |__/| $$| $$ +| $$ \__/| $$ /$$ /$$| $$| $$ /$$$$$$$$ +| $$$$$$ | $$ /$$/| $$| $$| $$|____ /$$/ + \____ $$| $$$$$$/ | $$| $$| $$ /$$$$/ + /$$ \ $$| $$_ $$ | $$| $$| $$ /$$__/ +| $$$$$$/| $$ \ $$| $$| $$| $$ /$$$$$$$$ + \______/ |__/ \__/|__/|__/|__/|________/ +Source: /tmp/work/sample-skills +Found 3 skill(s) +Which agents do you want to install to? Selected: Claude Code (claude-code) +Install 2 skill(s) [commit-helper, pr-describer] to 16 agent(s) [amp, antigravity, claude-code, +cline, codex, cursor, deepagents, dexto, firebender, gemini-cli, github-copilot, kimi-cli, +opencode, replit, universal, warp] [y/n] (y): y + +β”Œβ”€Installation Summary────────────────────────────────────────────────────────────────────────┐ +β”‚ Canonical: /tmp/work/.agents/skills β”‚ +β”‚ Universal: Amp, Antigravity, Cline, Codex, Cursor, Deep Agents, Dexto, Firebender, Gemini β”‚ +β”‚ CLI, GitHub Copilot, Kimi Code CLI, OpenCode, Replit, Universal, Warp β”‚ +β”‚ Symlinked: Claude Code β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +β”Œβ”€Installed 2 skill(s)────────────────────────────────────────────────────────────────────────┐ +β”‚ βœ“ commit-helper β”‚ +β”‚ β†’ /tmp/work/.agents/skills/commit-helper β”‚ +β”‚ βœ“ pr-describer β”‚ +β”‚ β†’ /tmp/work/.agents/skills/pr-describer β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +Done! Review skills before use; they run with full agent permissions. +$ diff --git a/test/e2e/add-flow.tape b/test/e2e/add-flow.tape new file mode 100644 index 0000000..b2c607a --- /dev/null +++ b/test/e2e/add-flow.tape @@ -0,0 +1,79 @@ +# End-to-end recording of `skillz add` driving the real interactive TUI. +# +# Produces two artifacts from ONE run: +# add-flow.gif β€” animated demo for PRs / README (not asserted on) +# add-flow.txt β€” final-frame text capture, scrubbed + diffed against a golden +# +# Run via the pinned VHS container (see run.sh). Inputs are mounted read-only at +# /src; outputs land in VHS's working dir. Every keystroke is gated on a Wait +# sentinel so the recording is synchronised to state, not wall-clock timing. + +Output add-flow.gif +Output add-flow.txt + +Set Shell bash +Set FontSize 18 +Set Width 1100 +Set Height 850 +Set Padding 14 +Set TypingSpeed 50ms +Set CursorBlink false + +# --- setup (hidden from the recording) --- +# Clean prompt, put the published binary on PATH, work in a throwaway dir so the +# install writes under /tmp (never into the mounted source tree). Pin HOME to a +# dedicated dir so the agent picker's persisted last-selection (read from +# $HOME/.local/share/skillz) is hermetic β€” not inherited from the recording host. +Hide +Type `export PS1="$ "` Enter +Type `export PATH=/src/test/e2e/bin:$PATH` Enter +Type `export HOME=/tmp/home && mkdir -p /tmp/home` Enter +Type `cd /tmp && rm -rf work && mkdir work && cd work` Enter +Type `cp -r /src/test/fixtures/sample-skills .` Enter +Type `clear` Enter +Show + +# --- the demo --- +Type `skillz add ./sample-skills` Enter + +# Skill multi-select: pick 2 of the 3 skills (leave changelog-updater off). +Wait+Screen@30s /Select skills to install/ +Sleep 800ms +Down +Sleep 400ms +Space +Sleep 400ms +Down +Sleep 400ms +Space +Sleep 700ms +Enter + +# Searchable agent picker: type to filter, then accept the pre-selected Claude Code. +Wait+Screen@30s /Which agents/ +Sleep 700ms +Type `claude` +Sleep 1200ms +Enter + +# Scope: Project (the default first option). +Wait+Screen@30s /Installation scope/ +Sleep 700ms +Enter + +# Install method: Symlink (the default, recommended). Appears because the selected +# agents resolve to more than one skills directory. +Wait+Screen@30s /Installation method/ +Sleep 700ms +Enter + +# Confirm. +Wait+Screen@30s /Install 2 skill/ +Sleep 800ms +Enter + +# Result. +Wait+Screen@30s /Done/ +Sleep 1800ms +Screenshot add-flow-final.png +Sleep 600ms diff --git a/test/e2e/copy-flow.gif b/test/e2e/copy-flow.gif new file mode 100644 index 0000000..9ece714 Binary files /dev/null and b/test/e2e/copy-flow.gif differ diff --git a/test/e2e/copy-flow.golden.txt b/test/e2e/copy-flow.golden.txt new file mode 100644 index 0000000..1ea659e --- /dev/null +++ b/test/e2e/copy-flow.golden.txt @@ -0,0 +1,23 @@ +$ skillz add ./sample-skills --agent claude-code --copy --skill commit-helper -y + + /$$$$$$ /$$ /$$ /$$ /$$ + /$$__ $$| $$ |__/| $$| $$ +| $$ \__/| $$ /$$ /$$| $$| $$ /$$$$$$$$ +| $$$$$$ | $$ /$$/| $$| $$| $$|____ /$$/ + \____ $$| $$$$$$/ | $$| $$| $$ /$$$$/ + /$$ \ $$| $$_ $$ | $$| $$| $$ /$$__/ +| $$$$$$/| $$ \ $$| $$| $$| $$ /$$$$$$$$ + \______/ |__/ \__/|__/|__/|__/|________/ +Source: /tmp/work/sample-skills +Found 3 skill(s) + +β”Œβ”€Installation Summary────────────────────────────────────────────────────────────────────────┐ +β”‚ Copied: Claude Code β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +β”Œβ”€Installed 1 skill(s)────────────────────────────────────────────────────────────────────────┐ +β”‚ βœ“ commit-helper β”‚ +β”‚ β†’ /tmp/work/.claude/skills/commit-helper β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +Done! Review skills before use; they run with full agent permissions. +$ diff --git a/test/e2e/copy-flow.tape b/test/e2e/copy-flow.tape new file mode 100644 index 0000000..8c65585 --- /dev/null +++ b/test/e2e/copy-flow.tape @@ -0,0 +1,37 @@ +# End-to-end recording of `skillz add --copy` (copy instead of symlink). +# +# Non-interactive (--agent + --skill + -y skip every prompt). Validates the +# copy install path and the "Copied:" summary label β€” the counterpart to +# add-flow.tape, which shows the interactive symlink path. A single skill is +# pinned with --skill so the install report is byte-deterministic (the report +# lists skills in discovery order, which is filesystem-dependent for >1 skill). +# +# Produces copy-flow.gif (demo) + copy-flow.txt (final-frame golden source). + +Output copy-flow.gif +Output copy-flow.txt + +Set Shell bash +Set FontSize 18 +Set Width 1100 +Set Height 850 +Set Padding 14 +Set TypingSpeed 50ms +Set CursorBlink false + +# --- setup (hidden) --- +Hide +Type `export PS1="$ "` Enter +Type `export PATH=/src/test/e2e/bin:$PATH` Enter +Type `cd /tmp && rm -rf work && mkdir work && cd work` Enter +Type `cp -r /src/test/fixtures/sample-skills .` Enter +Type `clear` Enter +Show + +# --- the demo --- +Type `skillz add ./sample-skills --agent claude-code --copy --skill commit-helper -y` Enter + +Wait+Screen@30s /Done/ +Sleep 1800ms +Screenshot copy-flow-final.png +Sleep 500ms diff --git a/test/e2e/error-flow.gif b/test/e2e/error-flow.gif new file mode 100644 index 0000000..a2782d8 Binary files /dev/null and b/test/e2e/error-flow.gif differ diff --git a/test/e2e/error-flow.golden.txt b/test/e2e/error-flow.golden.txt new file mode 100644 index 0000000..1103186 --- /dev/null +++ b/test/e2e/error-flow.golden.txt @@ -0,0 +1,15 @@ +$ skillz add ./missing + + /$$$$$$ /$$ /$$ /$$ /$$ + /$$__ $$| $$ |__/| $$| $$ +| $$ \__/| $$ /$$ /$$| $$| $$ /$$$$$$$$ +| $$$$$$ | $$ /$$/| $$| $$| $$|____ /$$/ + \____ $$| $$$$$$/ | $$| $$| $$ /$$$$/ + /$$ \ $$| $$_ $$ | $$| $$| $$ /$$__/ +| $$$$$$/| $$ \ $$| $$| $$| $$ /$$$$$$$$ + \______/ |__/ \__/|__/|__/|__/|________/ +Source: /tmp/work/missing +Local path does not exist: /tmp/work/missing +$ echo "[skillz exit: $?]" +[skillz exit: 1] +$ diff --git a/test/e2e/error-flow.tape b/test/e2e/error-flow.tape new file mode 100644 index 0000000..f3e0d8a --- /dev/null +++ b/test/e2e/error-flow.tape @@ -0,0 +1,40 @@ +# End-to-end recording of an error path: `skillz add` on a missing local source. +# +# Validates the failure UX end to end β€” the logo, the resolved Source line, the +# error message (written to stdout via the interaction service), and the non-zero +# exit code (asserted by echoing $? into the recording). Hermetic: the path simply +# does not exist. Produces error-flow.gif + error-flow.txt. + +Output error-flow.gif +Output error-flow.txt + +Set Shell bash +Set FontSize 18 +Set Width 1100 +Set Height 850 +Set Padding 14 +Set TypingSpeed 50ms +Set CursorBlink false + +# --- setup (hidden) --- +Hide +Type `export PS1="$ "` Enter +Type `export PATH=/src/test/e2e/bin:$PATH` Enter +Type `cd /tmp && rm -rf work && mkdir work && cd work` Enter +Type `clear` Enter +Show + +# --- the demo --- +Type `skillz add ./missing` Enter + +# Assert the failure UX *and* the non-zero exit: echo $? straight after the +# command (no command runs in between, so $? is still skillz's) so the golden +# pins "exit: 1". A regression that prints the error but exits 0 would change +# this line β€” caught by the diff, and by the marker ([skillz exit: 1]) too. +Wait+Screen@30s /Local path does not exist/ +Sleep 800ms +Type `echo "[skillz exit: $?]"` Enter +Wait+Screen@30s /skillz exit: 1/ +Sleep 1000ms +Screenshot error-flow-final.png +Sleep 500ms diff --git a/test/e2e/extract-frame.sh b/test/e2e/extract-frame.sh new file mode 100755 index 0000000..8fbd416 --- /dev/null +++ b/test/e2e/extract-frame.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Extract the final completed-state frame from a VHS .txt capture. +# +# extract-frame.sh [marker] +# +# `Output foo.txt` in a tape concatenates every rendered frame, separated by a +# horizontal rule (a line of box-drawing dashes). For a stable golden we keep +# only the last frame that reached the flow's success marker, with trailing +# whitespace and surrounding blank lines trimmed. +# +# `marker` is a literal substring the completed frame must contain (e.g. "Done!" +# for an install, "Successfully removed" for a removal). When omitted/empty the +# last non-blank frame is used. +# +# Exit non-zero (instead of silently falling back) when the capture is missing or +# empty, or when a non-empty marker is never seen. This makes a broken recording +# or a wrong/typo'd marker fail loudly rather than (a) passing on the wrong frame +# or (b) committing an empty golden via `run.sh --update`. +set -euo pipefail + +TXT="${1:-/dev/stdin}" +MARKER="${2:-}" + +if [[ "$TXT" != "/dev/stdin" && ! -s "$TXT" ]]; then + echo "extract-frame: capture is missing or empty: $TXT" >&2 + exit 1 +fi + +awk -v marker="$MARKER" ' + function store() { + if (buf ~ /[^[:space:]]/) { + anylast = buf # last non-blank frame (fallback) + if (marker == "" || index(buf, marker) > 0) { + last = buf # last frame matching the marker + seen = 1 + } + } + buf = "" + } + /^────/ { store(); next } + { line = $0; sub(/[ \t\r]+$/, "", line); buf = buf line "\n" } + END { + store() + # A required marker that never appeared means the flow did not reach its + # success state (or the marker is wrong) β€” fail rather than emit a stale frame. + if (marker != "" && !seen) { + print "extract-frame: success marker never appeared: " marker > "/dev/stderr" + exit 3 + } + out = (last != "" ? last : anylast) + sub(/^\n+/, "", out) + sub(/\n+$/, "\n", out) + printf "%s", out + } +' "$TXT" diff --git a/test/e2e/global-flow.gif b/test/e2e/global-flow.gif new file mode 100644 index 0000000..1f857d3 Binary files /dev/null and b/test/e2e/global-flow.gif differ diff --git a/test/e2e/global-flow.golden.txt b/test/e2e/global-flow.golden.txt new file mode 100644 index 0000000..ae5b353 --- /dev/null +++ b/test/e2e/global-flow.golden.txt @@ -0,0 +1,23 @@ +$ skillz add ./sample-skills --agent claude-code --global --skill commit-helper -y + + /$$$$$$ /$$ /$$ /$$ /$$ + /$$__ $$| $$ |__/| $$| $$ +| $$ \__/| $$ /$$ /$$| $$| $$ /$$$$$$$$ +| $$$$$$ | $$ /$$/| $$| $$| $$|____ /$$/ + \____ $$| $$$$$$/ | $$| $$| $$ /$$$$/ + /$$ \ $$| $$_ $$ | $$| $$| $$ /$$__/ +| $$$$$$/| $$ \ $$| $$| $$| $$ /$$$$$$$$ + \______/ |__/ \__/|__/|__/|__/|________/ +Source: /tmp/work/sample-skills +Found 3 skill(s) + +β”Œβ”€Installation Summary────────────────────────────────────────────────────────────────────────┐ +β”‚ Copied: Claude Code β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +β”Œβ”€Installed 1 skill(s)────────────────────────────────────────────────────────────────────────┐ +β”‚ βœ“ commit-helper β”‚ +β”‚ β†’ /tmp/home/.claude/skills/commit-helper β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +Done! Review skills before use; they run with full agent permissions. +$ diff --git a/test/e2e/global-flow.tape b/test/e2e/global-flow.tape new file mode 100644 index 0000000..051ca64 --- /dev/null +++ b/test/e2e/global-flow.tape @@ -0,0 +1,41 @@ +# End-to-end recording of `skillz add -g` (global scope install). +# +# Non-interactive (--agent + --skill + -y skip every prompt; -g skips the scope +# prompt). Validates the global install path β€” skills land under $HOME rather +# than the project. HOME is pinned to /tmp/home (distinct from the /tmp/work cwd) +# so the rendered global path differs from the project path β€” otherwise this flow +# is indistinguishable from copy-flow. A single skill is pinned for a byte-stable +# report. +# +# Produces global-flow.gif (demo) + global-flow.txt (final-frame golden source). + +Output global-flow.gif +Output global-flow.txt + +Set Shell bash +Set FontSize 18 +Set Width 1100 +Set Height 850 +Set Padding 14 +Set TypingSpeed 50ms +Set CursorBlink false + +# --- setup (hidden) --- +# Pin HOME to a dir distinct from the cwd so the global skills directory +# ($HOME/.claude/skills) renders a fixed path that is NOT the project path. +Hide +Type `export PS1="$ "` Enter +Type `export PATH=/src/test/e2e/bin:$PATH` Enter +Type `cd /tmp && rm -rf work && mkdir work && cd work` Enter +Type `export HOME=/tmp/home && mkdir -p /tmp/home` Enter +Type `cp -r /src/test/fixtures/sample-skills .` Enter +Type `clear` Enter +Show + +# --- the demo --- +Type `skillz add ./sample-skills --agent claude-code --global --skill commit-helper -y` Enter + +Wait+Screen@30s /Done/ +Sleep 1800ms +Screenshot global-flow-final.png +Sleep 500ms diff --git a/test/e2e/init-flow.gif b/test/e2e/init-flow.gif new file mode 100644 index 0000000..88fafa2 Binary files /dev/null and b/test/e2e/init-flow.gif differ diff --git a/test/e2e/init-flow.golden.txt b/test/e2e/init-flow.golden.txt new file mode 100644 index 0000000..c077722 --- /dev/null +++ b/test/e2e/init-flow.golden.txt @@ -0,0 +1,23 @@ +$ skillz init my-skill + + /$$$$$$ /$$ /$$ /$$ /$$ + /$$__ $$| $$ |__/| $$| $$ +| $$ \__/| $$ /$$ /$$| $$| $$ /$$$$$$$$ +| $$$$$$ | $$ /$$/| $$| $$| $$|____ /$$/ + \____ $$| $$$$$$/ | $$| $$| $$ /$$$$/ + /$$ \ $$| $$_ $$ | $$| $$| $$ /$$__/ +| $$$$$$/| $$ \ $$| $$| $$| $$ /$$$$$$$$ + \______/ |__/ \__/|__/|__/|__/|________/ +Initialized skill: my-skill + +Created: + my-skill/SKILL.md + +Next steps: + 1. Edit my-skill/SKILL.md to define your skill instructions + 2. Update the name and description in the frontmatter + +Publishing: + GitHub: Push to a repo, then skillz add / + URL: Host the file, then skillz add https://example.com/my-skill/SKILL.md +$ diff --git a/test/e2e/init-flow.tape b/test/e2e/init-flow.tape new file mode 100644 index 0000000..1d40562 --- /dev/null +++ b/test/e2e/init-flow.tape @@ -0,0 +1,31 @@ +# End-to-end recording of `skillz init` (scaffold a new skill). +# +# Non-interactive: writes /SKILL.md and prints the next-steps summary. +# Produces init-flow.gif (demo) + init-flow.txt (final-frame golden source). + +Output init-flow.gif +Output init-flow.txt + +Set Shell bash +Set FontSize 18 +Set Width 1100 +Set Height 850 +Set Padding 14 +Set TypingSpeed 50ms +Set CursorBlink false + +# --- setup (hidden) --- +Hide +Type `export PS1="$ "` Enter +Type `export PATH=/src/test/e2e/bin:$PATH` Enter +Type `cd /tmp && rm -rf work && mkdir work && cd work` Enter +Type `clear` Enter +Show + +# --- the demo --- +Type `skillz init my-skill` Enter + +Wait+Screen@30s /Initialized skill/ +Sleep 1500ms +Screenshot init-flow-final.png +Sleep 500ms diff --git a/test/e2e/list-flow.gif b/test/e2e/list-flow.gif new file mode 100644 index 0000000..49c2432 Binary files /dev/null and b/test/e2e/list-flow.gif differ diff --git a/test/e2e/list-flow.golden.txt b/test/e2e/list-flow.golden.txt new file mode 100644 index 0000000..13439a2 --- /dev/null +++ b/test/e2e/list-flow.golden.txt @@ -0,0 +1,6 @@ +$ skillz list +Project Skills + +Skill Path Agents +commit-helper ./.claude/skills/commit-helper Claude Code +$ diff --git a/test/e2e/list-flow.tape b/test/e2e/list-flow.tape new file mode 100644 index 0000000..eb50a4f --- /dev/null +++ b/test/e2e/list-flow.tape @@ -0,0 +1,38 @@ +# End-to-end recording of `skillz list` (show installed skills). +# +# Hidden setup installs one skill to a concrete agent (claude-code) so the table +# has a real, deterministic row (Skill / Path / Agents). A single skill keeps the +# output byte-stable. Produces list-flow.gif + list-flow.txt. + +Output list-flow.gif +Output list-flow.txt + +Set Shell bash +Set FontSize 18 +Set Width 1100 +Set Height 850 +Set Padding 14 +Set TypingSpeed 50ms +Set CursorBlink false + +# --- setup (hidden): install one skill so `list` has something to show --- +# HOME is pinned to a dir distinct from the cwd so the listed path renders as the +# cwd-relative form (./.claude/...) deterministically, not the container's HOME. +Hide +Type `export PS1="$ "` Enter +Type `export PATH=/src/test/e2e/bin:$PATH` Enter +Type `export HOME=/tmp/home && mkdir -p /tmp/home` Enter +Type `cd /tmp && rm -rf work && mkdir work && cd work` Enter +Type `cp -r /src/test/fixtures/sample-skills .` Enter +Type `skillz add ./sample-skills --agent claude-code --skill commit-helper -y` Enter +Wait+Screen@30s /Done/ +Type `clear` Enter +Show + +# --- the demo --- +Type `skillz list` Enter + +Wait+Screen@30s /Project Skills/ +Sleep 1500ms +Screenshot list-flow-final.png +Sleep 500ms diff --git a/test/e2e/remove-flow.gif b/test/e2e/remove-flow.gif new file mode 100644 index 0000000..8337c66 Binary files /dev/null and b/test/e2e/remove-flow.gif differ diff --git a/test/e2e/remove-flow.golden.txt b/test/e2e/remove-flow.golden.txt new file mode 100644 index 0000000..704488f --- /dev/null +++ b/test/e2e/remove-flow.golden.txt @@ -0,0 +1,5 @@ +$ skillz remove +Are you sure you want to remove 3 skill(s) [changelog-updater, commit-helper, pr-describer]? [ +y/n] (n): y +Successfully removed 3 skill(s) +$ diff --git a/test/e2e/remove-flow.tape b/test/e2e/remove-flow.tape new file mode 100644 index 0000000..4b4b980 --- /dev/null +++ b/test/e2e/remove-flow.tape @@ -0,0 +1,59 @@ +# End-to-end recording of `skillz remove` (interactive removal). +# +# Hidden setup installs the three sample skills. The demo then drives the real +# interactive TUI: the multi-select picker (skills are listed sorted, so the +# order is deterministic), the y/n confirmation (default no), and the removal +# summary. Produces remove-flow.gif + remove-flow.txt. + +Output remove-flow.gif +Output remove-flow.txt + +Set Shell bash +Set FontSize 18 +Set Width 1100 +Set Height 850 +Set Padding 14 +Set TypingSpeed 50ms +Set CursorBlink false + +# --- setup (hidden): install three skills so there is something to remove --- +Hide +Type `export PS1="$ "` Enter +Type `export PATH=/src/test/e2e/bin:$PATH` Enter +Type `cd /tmp && rm -rf work && mkdir work && cd work` Enter +Type `cp -r /src/test/fixtures/sample-skills .` Enter +Type `skillz add ./sample-skills -y` Enter +Wait+Screen@30s /Done/ +Type `clear` Enter +Show + +# --- the demo --- +Type `skillz remove` Enter + +# Multi-select: pick all three (sorted: changelog-updater, commit-helper, pr-describer). +Wait+Screen@30s /Select skills to remove/ +Sleep 800ms +Space +Sleep 350ms +Down +Sleep 250ms +Space +Sleep 350ms +Down +Sleep 250ms +Space +Sleep 600ms +Enter + +# Confirmation prompt (default no): type y to confirm. +Wait+Screen@30s /Are you sure/ +Sleep 700ms +Type `y` +Sleep 350ms +Enter + +# Result. +Wait+Screen@30s /Successfully removed/ +Sleep 1800ms +Screenshot remove-flow-final.png +Sleep 500ms diff --git a/test/e2e/run.sh b/test/e2e/run.sh new file mode 100755 index 0000000..9dd7f23 --- /dev/null +++ b/test/e2e/run.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash +# Record the skillz CLI flows with VHS and verify each against its golden frame. +# +# ./run.sh record every flow, diff each final frame vs its golden (CI / test) +# ./run.sh --update record every flow, overwrite goldens + committed GIFs (accept output) +# ./run.sh remove update record only the named flows (any subset of the list below) +# ./run.sh --update init update a single flow +# +# Each flow is driven by test/e2e/-flow.tape and checked against +# test/e2e/-flow.golden.txt. One VHS run per flow produces both the demo +# GIF (-flow.gif) and the text capture that is reduced to a deterministic +# final frame and diffed. +# +# Inputs (the repo) are mounted read-only into the pinned VHS container; the +# container bundles ttyd + ffmpeg + fonts so nothing needs installing on the host. +# The published binary and raw recordings land in gitignored dirs (bin/, out/). +# +# Results for changed/new flows are collected under out/report/ for CI to upload +# as artifacts and surface in a PR comment: +# out/report/status.tsv \t (every recorded flow) +# out/report/changed.txt one per line (FAIL or NEW only) +# out/report/-flow.gif / .frame.txt / .golden.txt / .diff +# +# Exit code: 0 if every recorded flow PASSed; 1 if any FAILed or was NEW (no +# golden yet); 2 on usage/setup error; 3 if docker is unavailable. --update +# exits 0 when every targeted flow recorded successfully, or 1 if any recording +# failed (its golden is left unchanged). +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +OUT_DIR="$SCRIPT_DIR/out" +BIN_DIR="$SCRIPT_DIR/bin" +REPORT_DIR="$OUT_DIR/report" + +# Pinned by digest so a new VHS release can't silently shift the rendered output +# (and thus the goldens). Bump deliberately, then re-run with --update. Tag: v0.11.0. +VHS_IMAGE="${VHS_IMAGE:-ghcr.io/charmbracelet/vhs@sha256:9d5fc3dc0c160b0fb1d2212baff07e6bdf3fa9438c504a3237484567302fcf93}" + +# Every flow, in record order, with the literal marker the completed final frame +# must contain (see extract-frame.sh). Keep this list and ALL_FLOWS in sync. +declare -A MARKERS=( + [add]="Done!" + [copy]="Done!" + [global]="Done!" + [init]="Initialized skill" + [list]="Project Skills" + [remove]="Successfully removed" + [update]="No project skills to update." + [error]="[skillz exit: 1]" +) +ALL_FLOWS=(add copy global init list remove update error) + +# --- args: optional --update plus an optional subset of flow names ------------ +UPDATE=0 +FLOWS=() +for arg in "$@"; do + case "$arg" in + --update) UPDATE=1 ;; + -*) echo "unknown option: $arg" >&2; exit 2 ;; + *) + if [[ -z "${MARKERS[$arg]+x}" ]]; then + echo "unknown flow: $arg (known: ${ALL_FLOWS[*]})" >&2 + exit 2 + fi + FLOWS+=("$arg") + ;; + esac +done +[[ ${#FLOWS[@]} -eq 0 ]] && FLOWS=("${ALL_FLOWS[@]}") + +if ! command -v docker >/dev/null 2>&1; then + echo "docker is required to run the VHS recordings" >&2 + exit 3 +fi + +# 1. Publish a self-contained linux-x64 binary (glibc β€” matches the Debian-based +# VHS image). Skip if already present unless REBUILD=1 or --update β€” an update +# must never bake a stale binary's output into the committed goldens. +if [[ ! -x "$BIN_DIR/skillz" || "${REBUILD:-0}" == "1" || "$UPDATE" == "1" ]]; then + echo "==> publishing skillz (linux-x64, self-contained)" + dotnet publish "$REPO_ROOT/src/Skillz/Skillz.csproj" \ + -c Release -f net10.0 -r linux-x64 --self-contained \ + -p:PublishAot=false -p:PublishSingleFile=true -p:PublishTrimmed=false \ + -o "$BIN_DIR" >/dev/null +fi + +mkdir -p "$OUT_DIR" +rm -rf "$REPORT_DIR" +mkdir -p "$REPORT_DIR" +: > "$REPORT_DIR/status.tsv" +: > "$REPORT_DIR/changed.txt" + +overall=0 +declare -a SUMMARY=() + +for flow in "${FLOWS[@]}"; do + tape="$SCRIPT_DIR/${flow}-flow.tape" + golden="$SCRIPT_DIR/${flow}-flow.golden.txt" + frame="$OUT_DIR/${flow}-flow.frame.txt" + + if [[ ! -f "$tape" ]]; then + echo "==> $flow: missing tape ($tape)" >&2 + exit 2 + fi + + # 2. Record. The tape writes -flow.gif + -flow.txt into OUT_DIR + # (mounted as the VHS workdir at /vhs). A single flow's failure (docker/VHS + # error, or a Wait sentinel that times out) must NOT abort the batch under + # `set -e`: capture it, record the flow as FAIL, and carry on to the rest. + echo "==> recording ${flow}-flow" + rec_ok=1 + if ! docker run --rm --shm-size=512m \ + -v "$REPO_ROOT":/src:ro \ + -v "$OUT_DIR":/vhs \ + "$VHS_IMAGE" "/src/test/e2e/${flow}-flow.tape"; then + rec_ok=0 + echo " recording FAILED (docker/VHS error)" >&2 + fi + + # 3. Reduce the multi-frame capture to the deterministic final frame. + # extract-frame.sh exits non-zero if the capture is missing/empty or the + # success marker never appeared; treat that as a failed recording too. + extract_ok=0 + if [[ "$rec_ok" == "1" ]] \ + && bash "$SCRIPT_DIR/extract-frame.sh" "$OUT_DIR/${flow}-flow.txt" "${MARKERS[$flow]}" > "$frame"; then + extract_ok=1 + else + : > "$frame" + echo " frame extraction FAILED (empty capture or missing marker '${MARKERS[$flow]}')" >&2 + fi + recorded_ok=0 + if [[ "$extract_ok" == "1" && -s "$frame" ]]; then + recorded_ok=1 + fi + + # 4. Update or verify. + if [[ "$UPDATE" == "1" ]]; then + if [[ "$recorded_ok" == "1" ]]; then + cp "$frame" "$golden" + cp "$OUT_DIR/${flow}-flow.gif" "$SCRIPT_DIR/${flow}-flow.gif" + echo " updated golden + GIF" + SUMMARY+=("$flow UPDATED") + else + overall=1 + echo " NOT updated β€” recording failed, golden left unchanged" >&2 + SUMMARY+=("$flow FAILED") + fi + continue + fi + + if [[ "$recorded_ok" != "1" ]]; then + status=FAIL + elif [[ ! -f "$golden" ]]; then + status=NEW + elif diff -u "$golden" "$frame" > "$REPORT_DIR/${flow}-flow.diff" 2>&1; then + status=PASS + rm -f "$REPORT_DIR/${flow}-flow.diff" + else + status=FAIL + fi + + printf '%s\t%s\n' "$flow" "$status" >> "$REPORT_DIR/status.tsv" + SUMMARY+=("$flow $status") + + if [[ "$status" != "PASS" ]]; then + overall=1 + echo "$flow" >> "$REPORT_DIR/changed.txt" + cp -f "$OUT_DIR/${flow}-flow.gif" "$REPORT_DIR/${flow}-flow.gif" 2>/dev/null || true + cp -f "$frame" "$REPORT_DIR/${flow}-flow.frame.txt" 2>/dev/null || true + [[ -f "$golden" ]] && cp -f "$golden" "$REPORT_DIR/${flow}-flow.golden.txt" + echo " $status (recorded; see report)" + else + echo " PASS" + fi +done + +echo +echo "==> summary" +for line in "${SUMMARY[@]}"; do + echo " $line" +done + +if [[ "$UPDATE" == "1" ]]; then + exit 0 +fi + +if [[ "$overall" -ne 0 ]]; then + echo "==> FAIL: $(wc -l < "$REPORT_DIR/changed.txt" | tr -d ' ') flow(s) changed or new (re-run with --update to accept)" >&2 +else + echo "==> PASS: all ${#FLOWS[@]} flow(s) match their golden" +fi +exit "$overall" diff --git a/test/e2e/update-flow.gif b/test/e2e/update-flow.gif new file mode 100644 index 0000000..d9c64b7 Binary files /dev/null and b/test/e2e/update-flow.gif differ diff --git a/test/e2e/update-flow.golden.txt b/test/e2e/update-flow.golden.txt new file mode 100644 index 0000000..cda8f5e --- /dev/null +++ b/test/e2e/update-flow.golden.txt @@ -0,0 +1,13 @@ +$ skillz update +Checking for skill updates... + +Global Skills +No global skills tracked in lock file. +Install skills with: skillz add -g + +Project Skills +No project skills to update. +Install project skills with: skillz add + + +$ diff --git a/test/e2e/update-flow.tape b/test/e2e/update-flow.tape new file mode 100644 index 0000000..11a0189 --- /dev/null +++ b/test/e2e/update-flow.tape @@ -0,0 +1,47 @@ +# End-to-end recording of `skillz update` (interactive update check). +# +# Hidden setup installs the three sample skills from a LOCAL source (so the lock +# records them) β€” local skills are never network-checked, which keeps the flow +# hermetic and deterministic. The demo drives the interactive scope picker +# (Project / Global / Both), selects Both, and shows the "nothing to update" +# result. Produces update-flow.gif + update-flow.txt. + +Output update-flow.gif +Output update-flow.txt + +Set Shell bash +Set FontSize 18 +Set Width 1100 +Set Height 850 +Set Padding 14 +Set TypingSpeed 50ms +Set CursorBlink false + +# --- setup (hidden): install three local skills so the lock is populated --- +Hide +Type `export PS1="$ "` Enter +Type `export PATH=/src/test/e2e/bin:$PATH` Enter +Type `cd /tmp && rm -rf work && mkdir work && cd work` Enter +Type `cp -r /src/test/fixtures/sample-skills .` Enter +Type `skillz add ./sample-skills -y` Enter +Wait+Screen@30s /Done/ +Type `clear` Enter +Show + +# --- the demo --- +Type `skillz update` Enter + +# Scope picker (single-select): default is Project; arrow down twice to Both. +Wait+Screen@30s /Update scope/ +Sleep 800ms +Down +Sleep 300ms +Down +Sleep 300ms +Enter + +# Result. +Wait+Screen@30s /No project skills to update/ +Sleep 1500ms +Screenshot update-flow-final.png +Sleep 500ms diff --git a/test/fixtures/sample-skills/skills/changelog-updater/SKILL.md b/test/fixtures/sample-skills/skills/changelog-updater/SKILL.md new file mode 100644 index 0000000..84fec6c --- /dev/null +++ b/test/fixtures/sample-skills/skills/changelog-updater/SKILL.md @@ -0,0 +1,12 @@ +--- +name: changelog-updater +description: Maintain a Keep a Changelog-formatted changelog from merged pull requests. +--- + +# Changelog Updater + +Keep `CHANGELOG.md` current using the Keep a Changelog format. + +1. Read merged PRs since the last release. +2. Sort entries into Added / Changed / Fixed / Removed. +3. Update the `Unreleased` section. diff --git a/test/fixtures/sample-skills/skills/commit-helper/SKILL.md b/test/fixtures/sample-skills/skills/commit-helper/SKILL.md new file mode 100644 index 0000000..e141ae7 --- /dev/null +++ b/test/fixtures/sample-skills/skills/commit-helper/SKILL.md @@ -0,0 +1,12 @@ +--- +name: commit-helper +description: Write clear, conventional commit messages from staged changes. +--- + +# Commit Helper + +Summarize staged changes into a concise, conventional commit message. + +1. Inspect the staged diff. +2. Choose a type (`feat`, `fix`, `chore`, ...) and an optional scope. +3. Write a one-line summary in the imperative mood, then an optional body. diff --git a/test/fixtures/sample-skills/skills/pr-describer/SKILL.md b/test/fixtures/sample-skills/skills/pr-describer/SKILL.md new file mode 100644 index 0000000..d346e5c --- /dev/null +++ b/test/fixtures/sample-skills/skills/pr-describer/SKILL.md @@ -0,0 +1,12 @@ +--- +name: pr-describer +description: Generate a pull request description from a branch diff. +--- + +# PR Describer + +Turn a branch's changes into a reviewer-friendly pull request description. + +1. Diff the branch against its base. +2. Group changes by intent. +3. Produce a summary, a "what changed" list, and a testing section.