diff --git a/.gitignore b/.gitignore
index 1f2db025b4f..38a2eb1c390 100644
--- a/.gitignore
+++ b/.gitignore
@@ -36,3 +36,4 @@ packages/example/build
 packages/example-without-zod/build
 packages/docs/out
 packages/cloudrun/container/ensure-browser.mjs
+packages/skills-evals/.runs
diff --git a/bun.lock b/bun.lock
index e9ef0ccb744..c4485300af2 100644
--- a/bun.lock
+++ b/bun.lock
@@ -1468,6 +1468,23 @@
         "remotion": "workspace:*",
       },
     },
+    "packages/skills-evals": {
+      "name": "@remotion/skills-evals",
+      "version": "4.0.460",
+      "dependencies": {
+        "@remotion/skills": "workspace:*",
+        "react": "catalog:",
+        "react-dom": "catalog:",
+      },
+      "devDependencies": {
+        "@remotion/eslint-config-internal": "workspace:*",
+        "@types/bun": "catalog:",
+        "@types/react": "catalog:",
+        "@types/react-dom": "catalog:",
+        "@typescript/native-preview": "catalog:",
+        "eslint": "catalog:",
+      },
+    },
     "packages/starburst": {
       "name": "@remotion/starburst",
       "version": "4.0.461",
@@ -3979,6 +3996,8 @@
 
     "@remotion/skills": ["@remotion/skills@workspace:packages/skills"],
 
+    "@remotion/skills-evals": ["@remotion/skills-evals@workspace:packages/skills-evals"],
+
     "@remotion/starburst": ["@remotion/starburst@workspace:packages/starburst"],
 
     "@remotion/streaming": ["@remotion/streaming@workspace:packages/streaming"],
diff --git a/package.json b/package.json
index 268ef6b3246..c2efe5e9ebd 100644
--- a/package.json
+++ b/package.json
@@ -85,6 +85,7 @@
 			"!packages/template-audiogram/whisper.cpp",
 			"!packages/template-audiogram/whisper.cpp/**",
 			"!packages/template-vercel/**",
+			"!packages/skills-evals/.runs/**",
 			"!packages/bugs"
 		],
 		"catalog": {
diff --git a/packages/cli/src/list-of-remotion-packages.ts b/packages/cli/src/list-of-remotion-packages.ts
index 888aebbff94..5f6317eec64 100644
--- a/packages/cli/src/list-of-remotion-packages.ts
+++ b/packages/cli/src/list-of-remotion-packages.ts
@@ -65,6 +65,7 @@ export const listOfRemotionPackages = [
 	'@remotion/serverless',
 	'@remotion/serverless-client',
 	'@remotion/skills',
+	'@remotion/skills-evals',
 	'@remotion/studio-server',
 	'@remotion/studio-shared',
 	'@remotion/studio',
diff --git a/packages/create-video/src/list-of-remotion-packages.ts b/packages/create-video/src/list-of-remotion-packages.ts
index 888aebbff94..5f6317eec64 100644
--- a/packages/create-video/src/list-of-remotion-packages.ts
+++ b/packages/create-video/src/list-of-remotion-packages.ts
@@ -65,6 +65,7 @@ export const listOfRemotionPackages = [
 	'@remotion/serverless',
 	'@remotion/serverless-client',
 	'@remotion/skills',
+	'@remotion/skills-evals',
 	'@remotion/studio-server',
 	'@remotion/studio-shared',
 	'@remotion/studio',
diff --git a/packages/skills-evals/README.md b/packages/skills-evals/README.md
new file mode 100644
index 00000000000..4510034e8b8
--- /dev/null
+++ b/packages/skills-evals/README.md
@@ -0,0 +1,5 @@
+# @remotion/skills-evals
+
+## Usage
+
+This is an internal package and has no documentation.
diff --git a/packages/skills-evals/eslint.config.mjs b/packages/skills-evals/eslint.config.mjs
new file mode 100644
index 00000000000..04fd4fac110
--- /dev/null
+++ b/packages/skills-evals/eslint.config.mjs
@@ -0,0 +1,7 @@
+import {remotionFlatConfig} from '@remotion/eslint-config-internal';
+
+const config = remotionFlatConfig({react: false});
+
+export default {
+	...config,
+};
diff --git a/packages/skills-evals/package.json b/packages/skills-evals/package.json
new file mode 100644
index 00000000000..8e6f424d2b2
--- /dev/null
+++ b/packages/skills-evals/package.json
@@ -0,0 +1,29 @@
+{
+	"repository": {
+		"url": "https://github.com/remotion-dev/remotion/tree/main/packages/skills-evals"
+	},
+	"name": "@remotion/skills-evals",
+	"version": "4.0.461",
+	"private": true,
+	"type": "module",
+	"scripts": {
+		"dev": "bun src/server.tsx",
+		"eval": "bun src/cli.ts",
+		"format": "oxfmt src scenarios.ts",
+		"formatting": "oxfmt src scenarios.ts --check",
+		"lint": "eslint src && tsgo"
+	},
+	"dependencies": {
+		"@remotion/skills": "workspace:*",
+		"react": "catalog:",
+		"react-dom": "catalog:"
+	},
+	"devDependencies": {
+		"@remotion/eslint-config-internal": "workspace:*",
+		"@types/bun": "catalog:",
+		"@types/react": "catalog:",
+		"@types/react-dom": "catalog:",
+		"@typescript/native-preview": "catalog:",
+		"eslint": "catalog:"
+	}
+}
diff --git a/packages/skills-evals/scenarios.ts b/packages/skills-evals/scenarios.ts
new file mode 100644
index 00000000000..480d865e2a9
--- /dev/null
+++ b/packages/skills-evals/scenarios.ts
@@ -0,0 +1,8 @@
+export type SkillEvalScenario = {
+	id: string;
+	model: string;
+	prompt: string;
+	timeoutMs?: number;
+};
+
+export const scenarios: SkillEvalScenario[] = [];
diff --git a/packages/skills-evals/src/app/comparison-data.ts b/packages/skills-evals/src/app/comparison-data.ts
new file mode 100644
index 00000000000..5dcb0080069
--- /dev/null
+++ b/packages/skills-evals/src/app/comparison-data.ts
@@ -0,0 +1,67 @@
+import {existsSync} from 'node:fs';
+import {readFile} from 'node:fs/promises';
+import {join} from 'node:path';
+import {listFilesRecursively, readJson, sanitizePathPart} from '../files';
+import type {SkillEvalComparison, SkillEvalManifest} from '../manifest';
+import {comparisonsRoot} from './shared';
+
+export type ComparisonWithManifests = {
+	afterManifest: SkillEvalManifest;
+	beforeManifest: SkillEvalManifest;
+	comparison: SkillEvalComparison;
+	skillDiff: string;
+};
+
+export const loadComparisons = async () => {
+	const files = (await listFilesRecursively(comparisonsRoot)).filter((file) =>
+		file.endsWith('/comparison.json'),
+	);
+	const comparisons = await Promise.all(
+		files.map((file) => readJson<SkillEvalComparison>(file)),
+	);
+
+	comparisons.sort((a, b) => b.completedAt.localeCompare(a.completedAt));
+
+	return comparisons;
+};
+
+export const getLatestComparisonByScenario = async () => {
+	const latest = new Map<string, SkillEvalComparison>();
+
+	for (const comparison of await loadComparisons()) {
+		if (!latest.has(comparison.scenarioId)) {
+			latest.set(comparison.scenarioId, comparison);
+		}
+	}
+
+	return latest;
+};
+
+export const loadComparison = async (
+	scenarioId: string,
+	comparisonId: string,
+): Promise<ComparisonWithManifests | null> => {
+	const comparisonPath = join(
+		comparisonsRoot,
+		sanitizePathPart(scenarioId),
+		comparisonId,
+		'comparison.json',
+	);
+
+	if (!existsSync(comparisonPath)) {
+		return null;
+	}
+
+	const comparison = await readJson<SkillEvalComparison>(comparisonPath);
+	const [beforeManifest, afterManifest, skillDiff] = await Promise.all([
+		readJson<SkillEvalManifest>(comparison.before.manifestPath),
+		readJson<SkillEvalManifest>(comparison.after.manifestPath),
+		readFile(comparison.skillDiffPath, 'utf-8'),
+	]);
+
+	return {afterManifest, beforeManifest, comparison, skillDiff};
+};
+
+export const getPreferredArtifact = (manifest: SkillEvalManifest) =>
+	manifest.artifacts.find((artifact) => artifact.type === 'video') ??
+	manifest.artifacts[0];
diff --git a/packages/skills-evals/src/app/comparison.tsx b/packages/skills-evals/src/app/comparison.tsx
new file mode 100644
index 00000000000..d6e6199b7c2
--- /dev/null
+++ b/packages/skills-evals/src/app/comparison.tsx
@@ -0,0 +1,221 @@
+import type {SkillEvalManifest} from '../manifest';
+import {
+	getPreferredArtifact,
+	type ComparisonWithManifests,
+} from './comparison-data';
+import {formatDate, Header, page, Pill, toFileUrl} from './shared';
+
+const Artifact = ({manifest}: {manifest: SkillEvalManifest}) => {
+	const artifact = getPreferredArtifact(manifest);
+
+	if (!artifact) {
+		return (
+			<div className="grid aspect-video place-items-center rounded-xl bg-zinc-900 text-[0.8125rem] text-zinc-400">
+				No visual artifact found
+			</div>
+		);
+	}
+
+	const href = toFileUrl(artifact.path);
+
+	if (artifact.type === 'image') {
+		return (
+			<a href={href}>
+				<img
+					alt={artifact.relativePath}
+					className="aspect-video w-full rounded-xl bg-zinc-900 object-contain"
+					src={href}
+				/>
+			</a>
+		);
+	}
+
+	return (
+		<video
+			className="aspect-video w-full rounded-xl bg-zinc-900 object-contain"
+			controls
+			preload="metadata"
+			src={href}
+		/>
+	);
+};
+
+const RunPanel = ({
+	label,
+	manifest,
+	manifestPath,
+}: {
+	label: string;
+	manifest: SkillEvalManifest;
+	manifestPath: string;
+}) => {
+	const artifact = getPreferredArtifact(manifest);
+
+	return (
+		<section className="rounded-2xl border border-zinc-200 bg-white p-3">
+			<div className="mb-2 flex items-center justify-between gap-3">
+				<h2 className="text-[0.9375rem] font-semibold">{label}</h2>
+				<div className="flex flex-wrap items-center gap-3">
+					<a
+						className="text-[0.8125rem] text-zinc-600"
+						href={toFileUrl(manifest.pi.htmlExport)}
+					>
+						Pi export
+					</a>
+					<a
+						className="text-[0.8125rem] text-zinc-600"
+						href={toFileUrl(manifestPath)}
+					>
+						Manifest
+					</a>
+				</div>
+			</div>
+			<Artifact manifest={manifest} />
+			<div className="mt-2 overflow-hidden text-ellipsis whitespace-nowrap text-xs text-zinc-500">
+				{artifact?.relativePath ?? 'No artifact'}
+			</div>
+		</section>
+	);
+};
+
+const ComparisonDiffScript = () => (
+	<>
+		<script src="https://cdn.jsdelivr.net/npm/diff@5/dist/diff.min.js" />
+		<script
+			dangerouslySetInnerHTML={{
+				__html: `
+(() => {
+	const rawDiff = document.getElementById('raw-diff');
+	const renderedDiff = document.getElementById('rendered-diff');
+
+	if (!rawDiff || !renderedDiff || !window.Diff?.parsePatch) {
+		return;
+	}
+
+	const patches = window.Diff.parsePatch(rawDiff.textContent || '');
+
+	if (patches.length === 0) {
+		return;
+	}
+
+	const lineClass = (line) => {
+		if (line.startsWith('+')) {
+			return 'bg-emerald-50 text-green-800';
+		}
+
+		if (line.startsWith('-')) {
+			return 'bg-red-50 text-red-800';
+		}
+
+		return 'text-zinc-700';
+	};
+
+	renderedDiff.replaceChildren(
+		...patches.map((patch) => {
+			const file = document.createElement('section');
+			file.className = 'border-t border-zinc-200 py-2 first:border-t-0';
+			const header = document.createElement('div');
+			header.className = 'px-3 pb-2 font-semibold text-zinc-800';
+			header.textContent = patch.newFileName || patch.oldFileName || 'Diff';
+			file.append(header);
+
+			for (const hunk of patch.hunks) {
+				const hunkHeader = document.createElement('div');
+				hunkHeader.className = 'bg-zinc-100 px-3 py-0.5 text-zinc-500 whitespace-pre';
+				hunkHeader.textContent = '@@ -' + hunk.oldStart + ',' + hunk.oldLines + ' +' + hunk.newStart + ',' + hunk.newLines + ' @@';
+				file.append(hunkHeader);
+
+				for (const line of hunk.lines) {
+					const row = document.createElement('div');
+					row.className = 'grid grid-cols-[22px_1fr] min-w-max px-3 whitespace-pre ' + lineClass(line);
+					const prefix = document.createElement('span');
+					prefix.className = 'text-zinc-400 select-none';
+					prefix.textContent = line[0] || ' ';
+					const content = document.createElement('span');
+					content.textContent = line.slice(1);
+					row.append(prefix, content);
+					file.append(row);
+				}
+			}
+
+			return file;
+		}),
+	);
+	renderedDiff.hidden = false;
+	rawDiff.hidden = true;
+})();
+`,
+			}}
+		/>
+	</>
+);
+
+export const renderComparison = (comparisonData: ComparisonWithManifests) => {
+	const {afterManifest, beforeManifest, comparison, skillDiff} = comparisonData;
+
+	return page({
+		children: (
+			<>
+				<Header
+					action={
+						<div className="flex flex-wrap items-center gap-3">
+							<Pill>{formatDate(comparison.completedAt)}</Pill>
+							<a
+								className="text-[0.8125rem] text-zinc-600"
+								href={`/scenarios/${encodeURIComponent(comparison.scenarioId)}`}
+							>
+								Scenario
+							</a>
+						</div>
+					}
+					eyebrow="Comparison"
+					subtitle={beforeManifest.model}
+					title={comparison.scenarioId}
+				/>
+				<main className="grid min-w-0 gap-4">
+					<div className="grid grid-cols-2 gap-3 max-lg:grid-cols-1">
+						<RunPanel
+							label="Before"
+							manifest={beforeManifest}
+							manifestPath={comparison.before.manifestPath}
+						/>
+						<RunPanel
+							label="After"
+							manifest={afterManifest}
+							manifestPath={comparison.after.manifestPath}
+						/>
+					</div>
+					<details
+						className="min-w-0 rounded-2xl border border-zinc-200 bg-white p-4"
+						open
+					>
+						<summary className="cursor-pointer text-sm font-semibold">
+							Skill diff
+						</summary>
+						<div
+							className="mt-3 max-h-130 max-w-full min-w-0 overflow-auto rounded-xl border border-zinc-200 bg-zinc-50 font-mono text-xs"
+							hidden
+							id="rendered-diff"
+						/>
+						<pre
+							className="mt-3 max-h-130 overflow-auto whitespace-pre-wrap rounded-xl border border-zinc-200 bg-zinc-100 p-3 text-xs text-zinc-700"
+							id="raw-diff"
+						>
+							{skillDiff}
+						</pre>
+					</details>
+					<details className="min-w-0 rounded-2xl border border-zinc-200 bg-white p-4">
+						<summary className="cursor-pointer text-sm font-semibold">
+							Prompt
+						</summary>
+						<pre className="mt-3 max-h-75 overflow-auto whitespace-pre-wrap rounded-xl border border-zinc-200 bg-zinc-100 p-3 text-xs text-zinc-700">
+							{afterManifest.prompt}
+						</pre>
+					</details>
+				</main>
+				<ComparisonDiffScript />
+			</>
+		),
+		title: `${comparison.scenarioId} Comparison`,
+	});
+};
diff --git a/packages/skills-evals/src/app/files.ts b/packages/skills-evals/src/app/files.ts
new file mode 100644
index 00000000000..474cb2f75f7
--- /dev/null
+++ b/packages/skills-evals/src/app/files.ts
@@ -0,0 +1,44 @@
+import {existsSync} from 'node:fs';
+import {realpath, stat} from 'node:fs/promises';
+import {extname, isAbsolute, relative, resolve} from 'node:path';
+import {notFound, runsRoot} from './shared';
+
+export const serveRunFile = async (relativePath: string) => {
+	const file = resolve(runsRoot, relativePath);
+
+	if (!existsSync(runsRoot)) {
+		return notFound();
+	}
+
+	const resolvedRunsRoot = await realpath(runsRoot);
+
+	if (!existsSync(file)) {
+		return notFound();
+	}
+
+	const realFile = await realpath(file);
+	const relativeToRunsRoot = relative(resolvedRunsRoot, realFile);
+
+	if (relativeToRunsRoot.startsWith('..') || isAbsolute(relativeToRunsRoot)) {
+		return notFound();
+	}
+
+	if (!(await stat(realFile)).isFile()) {
+		return notFound();
+	}
+
+	const contentType =
+		{
+			'.html': 'text/html; charset=utf-8',
+			'.json': 'application/json',
+			'.jsonl': 'application/x-ndjson',
+			'.log': 'text/plain; charset=utf-8',
+			'.mp4': 'video/mp4',
+			'.png': 'image/png',
+			'.webm': 'video/webm',
+		}[extname(realFile)] ?? 'application/octet-stream';
+
+	return new Response(Bun.file(realFile), {
+		headers: {'content-type': contentType},
+	});
+};
diff --git a/packages/skills-evals/src/app/home.tsx b/packages/skills-evals/src/app/home.tsx
new file mode 100644
index 00000000000..362cb4eb650
--- /dev/null
+++ b/packages/skills-evals/src/app/home.tsx
@@ -0,0 +1,61 @@
+import {scenarios, type SkillEvalScenario} from '../../scenarios';
+import type {SkillEvalComparison} from '../manifest';
+import {getLatestComparisonByScenario} from './comparison-data';
+import {formatDate, page, Pill} from './shared';
+
+const ScenarioCard = ({
+	latest,
+	scenario,
+}: {
+	latest: SkillEvalComparison | undefined;
+	scenario: SkillEvalScenario;
+}) => (
+	<a
+		className="block rounded-2xl border border-zinc-200 bg-white p-4 hover:border-zinc-300 hover:shadow-[0_12px_30px_rgba(24,24,27,0.06)]"
+		href={`/scenarios/${encodeURIComponent(scenario.id)}`}
+	>
+		<div className="flex items-start justify-between gap-3">
+			<div className="min-w-0">
+				<h2 className="text-[0.9375rem] font-semibold">{scenario.id}</h2>
+				<p className="text-sm text-zinc-500">{scenario.model}</p>
+			</div>
+			<Pill>{latest ? 'Ready' : 'New'}</Pill>
+		</div>
+		<p className="mt-4 text-xs text-zinc-400">
+			{latest
+				? `Last run ${formatDate(latest.completedAt)}`
+				: 'No comparisons yet.'}
+		</p>
+	</a>
+);
+
+export const renderHome = async () => {
+	const latest = await getLatestComparisonByScenario();
+
+	return page({
+		children: (
+			<>
+				<main className="grid gap-4">
+					<section className="max-w-3xl">
+						<h2 className="text-sm font-semibold text-zinc-700">Workflow</h2>
+						<p className="mt-1 text-sm leading-6 text-zinc-500">
+							When skills match HEAD, run a scenario as visual validation. When
+							you have local skill changes, run a comparison to see how those
+							changes affect the resulting videos.
+						</p>
+					</section>
+					<section className="grid grid-cols-[repeat(auto-fit,minmax(300px,1fr))] gap-3">
+						{scenarios.map((scenario) => (
+							<ScenarioCard
+								key={scenario.id}
+								latest={latest.get(scenario.id)}
+								scenario={scenario}
+							/>
+						))}
+					</section>
+				</main>
+			</>
+		),
+		title: 'Remotion Skills Evals',
+	});
+};
diff --git a/packages/skills-evals/src/app/jobs.ts b/packages/skills-evals/src/app/jobs.ts
new file mode 100644
index 00000000000..cfb3129e08b
--- /dev/null
+++ b/packages/skills-evals/src/app/jobs.ts
@@ -0,0 +1,251 @@
+import {basename} from 'node:path';
+import {scenarios, type SkillEvalScenario} from '../../scenarios';
+import {
+	runSkillEvalComparison,
+	type SkillEvalComparisonEvent,
+	type SkillEvalComparisonRunLabel,
+} from '../compare';
+import {runSkillEval, type SkillEvalPhase} from '../run-skill-eval';
+import {toComparisonUrl} from './shared';
+
+export type Job = {
+	id: string;
+	logs: string[];
+	runs: Record<SkillEvalComparisonRunLabel, JobRun>;
+	scenarioId: string;
+	startedAt: string;
+	status: 'running' | 'completed' | 'failed' | 'skipped';
+	comparisonUrl?: string;
+	error?: string;
+	message?: string;
+	resultUrl?: string;
+};
+
+export type JobRun = {
+	logs: string[];
+	message: string;
+	status: 'completed' | 'failed' | 'running' | 'waiting';
+};
+
+const jobs = new Map<string, Job>();
+
+const phaseLabels: Record<SkillEvalPhase, string> = {
+	install: 'Installing dependencies',
+	pi: 'Generating project',
+	'pi-export': 'Exporting Pi session',
+	'pi-render': 'Rendering artifact',
+};
+
+const createJobRun = (): JobRun => ({
+	logs: [],
+	message: 'Waiting to start',
+	status: 'waiting',
+});
+
+export const createJobRuns = (): Record<
+	SkillEvalComparisonRunLabel,
+	JobRun
+> => ({
+	after: createJobRun(),
+	before: createJobRun(),
+});
+
+export const formatRunLabel = (label: SkillEvalComparisonRunLabel) =>
+	label === 'before' ? 'Before' : 'After';
+
+const trimLog = (logs: string[], maxLength: number) => {
+	if (logs.length > maxLength) {
+		logs.splice(0, logs.length - maxLength);
+	}
+};
+
+export const getScenario = (id: string) =>
+	scenarios.find((scenario) => scenario.id === id) ?? null;
+
+export const getJob = (id: string) => jobs.get(id) ?? null;
+
+export const getActiveJob = (scenarioId: string) =>
+	[...jobs.values()].find(
+		(job) => job.scenarioId === scenarioId && job.status === 'running',
+	);
+
+export const startComparison = (scenario: SkillEvalScenario) => {
+	const existingJob = getActiveJob(scenario.id);
+
+	if (existingJob) {
+		return existingJob;
+	}
+
+	const job: Job = {
+		id: `${Date.now()}-${scenario.id}`,
+		logs: [],
+		message: 'Preparing comparison...',
+		runs: createJobRuns(),
+		scenarioId: scenario.id,
+		startedAt: new Date().toISOString(),
+		status: 'running',
+	};
+
+	jobs.set(job.id, job);
+
+	const appendLog = (chunk: string) => {
+		job.logs.push(chunk);
+		trimLog(job.logs, 1000);
+	};
+
+	const updateJobMessage = () => {
+		const running = Object.entries(job.runs)
+			.filter(([, run]) => run.status === 'running')
+			.map(([label]) => formatRunLabel(label as SkillEvalComparisonRunLabel));
+
+		if (running.length > 0) {
+			job.message = `${running.join(' and ')} ${
+				running.length === 1 ? 'is' : 'are'
+			} generating.`;
+			return;
+		}
+
+		const failed = Object.entries(job.runs).find(
+			([, run]) => run.status === 'failed',
+		);
+
+		if (failed) {
+			job.message = `${formatRunLabel(
+				failed[0] as SkillEvalComparisonRunLabel,
+			)} failed.`;
+		}
+	};
+
+	const handleEvent = (event: SkillEvalComparisonEvent) => {
+		if (event.type === 'message') {
+			appendLog(event.message);
+			job.message = event.message.trim();
+			return;
+		}
+
+		const run = job.runs[event.label];
+
+		if (event.type === 'run-start') {
+			run.status = 'running';
+			run.message = `${formatRunLabel(event.label)} is generating.`;
+			updateJobMessage();
+			return;
+		}
+
+		if (event.type === 'run-phase') {
+			run.status = 'running';
+			run.message = `${phaseLabels[event.event.phase]}${
+				event.event.status === 'completed' ? ' complete' : ''
+			}`;
+			updateJobMessage();
+			return;
+		}
+
+		if (event.type === 'run-output') {
+			run.status = 'running';
+			run.message = `${phaseLabels[event.output.phase]} (${event.output.stream})`;
+			run.logs.push(event.output.chunk);
+			trimLog(run.logs, 1000);
+			updateJobMessage();
+			return;
+		}
+
+		if (event.type === 'run-complete') {
+			run.status = 'completed';
+			run.message = `${formatRunLabel(event.label)} complete.`;
+			updateJobMessage();
+			return;
+		}
+
+		run.status = 'failed';
+		run.message = event.error;
+		updateJobMessage();
+	};
+
+	const comparisonPromise = runSkillEvalComparison(scenario, {
+		onEvent: handleEvent,
+	})
+		.then((result) => {
+			if (result.skipped) {
+				job.message = result.reason;
+				job.status = 'skipped';
+				return;
+			}
+
+			job.comparisonUrl = toComparisonUrl(result.comparison);
+			job.message = 'Comparison complete.';
+			job.status = 'completed';
+		})
+		.catch((error: unknown) => {
+			job.error = error instanceof Error ? error.message : String(error);
+			job.message = job.error;
+			job.status = 'failed';
+		});
+	comparisonPromise.catch(() => undefined);
+
+	return job;
+};
+
+export const startRun = (scenario: SkillEvalScenario) => {
+	const existingJob = getActiveJob(scenario.id);
+
+	if (existingJob) {
+		return existingJob;
+	}
+
+	const job: Job = {
+		id: `${Date.now()}-${scenario.id}`,
+		logs: [],
+		message: 'Preparing run...',
+		runs: createJobRuns(),
+		scenarioId: scenario.id,
+		startedAt: new Date().toISOString(),
+		status: 'running',
+	};
+	const run = job.runs.after;
+
+	job.runs.before.message = 'Not needed for a plain run';
+	job.runs.before.status = 'completed';
+	jobs.set(job.id, job);
+
+	const runPromise = runSkillEval({
+		...scenario,
+		onPhase: (event) => {
+			run.status = 'running';
+			run.message = `${phaseLabels[event.phase]}${
+				event.status === 'completed' ? ' complete' : ''
+			}`;
+			job.message = 'Running scenario...';
+		},
+		onOutput: (output) => {
+			run.status = 'running';
+			run.message = `${phaseLabels[output.phase]} (${output.stream})`;
+			run.logs.push(output.chunk);
+			trimLog(run.logs, 1000);
+			job.message = 'Running scenario...';
+		},
+		skillSnapshot: {
+			isWorkingTree: true,
+			label: 'after',
+		},
+	})
+		.then((result) => {
+			run.status = 'completed';
+			run.message = 'Run complete.';
+			job.message = 'Run complete.';
+			job.resultUrl = `/runs/${encodeURIComponent(
+				result.manifest.id,
+			)}/${encodeURIComponent(basename(result.manifest.runDir))}`;
+			job.status = 'completed';
+		})
+		.catch((error: unknown) => {
+			job.error = error instanceof Error ? error.message : String(error);
+			job.message = job.error;
+			job.status = 'failed';
+			run.status = 'failed';
+			run.message = job.error;
+		});
+	runPromise.catch(() => undefined);
+
+	return job;
+};
diff --git a/packages/skills-evals/src/app/routes.ts b/packages/skills-evals/src/app/routes.ts
new file mode 100644
index 00000000000..30af4da5baf
--- /dev/null
+++ b/packages/skills-evals/src/app/routes.ts
@@ -0,0 +1,102 @@
+import type {BunRequest} from 'bun';
+import {renderComparison} from './comparison';
+import {loadComparison} from './comparison-data';
+import {serveRunFile} from './files';
+import {renderHome} from './home';
+import {getJob, getScenario, startComparison, startRun} from './jobs';
+import {loadRun, renderRun} from './run';
+import {renderScenario} from './scenario';
+import {htmlResponse, json, notFound} from './shared';
+
+export const routes = {
+	'/': async () => htmlResponse(await renderHome()),
+
+	'/api/compare/:scenarioId': {
+		POST: (request: BunRequest<'/api/compare/:scenarioId'>) => {
+			const scenario = getScenario(request.params.scenarioId);
+
+			if (!scenario) {
+				return json({error: 'Unknown scenario'}, {status: 404});
+			}
+
+			return json(startComparison(scenario));
+		},
+	},
+
+	'/api/jobs/:jobId': (request: BunRequest<'/api/jobs/:jobId'>) => {
+		const job = getJob(request.params.jobId);
+
+		if (!job) {
+			return json({error: 'Unknown job'}, {status: 404});
+		}
+
+		return json(job);
+	},
+
+	'/api/run/:scenarioId': {
+		POST: (request: BunRequest<'/api/run/:scenarioId'>) => {
+			const scenario = getScenario(request.params.scenarioId);
+
+			if (!scenario) {
+				return json({error: 'Unknown scenario'}, {status: 404});
+			}
+
+			return json(startRun(scenario));
+		},
+	},
+
+	'/comparisons/:scenarioId/:comparisonId': async (
+		request: BunRequest<'/comparisons/:scenarioId/:comparisonId'>,
+	) => {
+		const comparisonData = await loadComparison(
+			request.params.scenarioId,
+			request.params.comparisonId,
+		);
+
+		if (!comparisonData) {
+			return notFound();
+		}
+
+		return htmlResponse(renderComparison(comparisonData));
+	},
+
+	'/files/*': (request: Request) => {
+		const {pathname} = new URL(request.url);
+		let relativePath: string;
+
+		try {
+			relativePath = decodeURIComponent(pathname.replace(/^\/files\//, ''));
+		} catch {
+			return new Response('Bad request', {status: 400});
+		}
+
+		return serveRunFile(relativePath);
+	},
+
+	'/scenarios/:scenarioId': async (
+		request: BunRequest<'/scenarios/:scenarioId'>,
+	) => {
+		const scenario = getScenario(request.params.scenarioId);
+
+		if (!scenario) {
+			return notFound();
+		}
+
+		return htmlResponse(await renderScenario(scenario));
+	},
+
+	'/runs/:scenarioId/:runId': async (
+		request: BunRequest<'/runs/:scenarioId/:runId'>,
+	) => {
+		const manifest = await loadRun(
+			request.params.scenarioId,
+			request.params.runId,
+		);
+
+		if (!manifest) {
+			return notFound();
+		}
+
+		return htmlResponse(renderRun(manifest));
+	},
+};
diff --git a/packages/skills-evals/src/app/run.tsx b/packages/skills-evals/src/app/run.tsx
new file mode 100644
index 00000000000..f1e446b8032
--- /dev/null
+++ b/packages/skills-evals/src/app/run.tsx
@@ -0,0 +1,101 @@
+import {existsSync} from 'node:fs';
+import {join} from 'node:path';
+import {readJson, sanitizePathPart} from '../files';
+import type {SkillEvalManifest} from '../manifest';
+import {getPreferredArtifact} from './comparison-data';
+import {Card, Header, page, runsRoot, toFileUrl} from './shared';
+
+export const loadRun = async (scenarioId: string, runId: string) => {
+	const manifestPath = join(
+		runsRoot,
+		sanitizePathPart(scenarioId),
+		runId,
+		'manifest.json',
+	);
+
+	if (!existsSync(manifestPath)) {
+		return null;
+	}
+
+	const manifest = await readJson<SkillEvalManifest>(manifestPath);
+
+	return manifest;
+};
+
+const Artifact = ({manifest}: {manifest: SkillEvalManifest}) => {
+	const artifact = getPreferredArtifact(manifest);
+
+	if (!artifact) {
+		return (
+			<div className="grid aspect-video place-items-center rounded-xl bg-zinc-900 text-[0.8125rem] text-zinc-400">
+				No visual artifact found
+			</div>
+		);
+	}
+
+	const href = toFileUrl(artifact.path);
+
+	if (artifact.type === 'image') {
+		return (
+			<a href={href}>
+				<img
+					alt={artifact.relativePath}
+					className="aspect-video w-full rounded-xl bg-zinc-900 object-contain"
+					src={href}
+				/>
+			</a>
+		);
+	}
+
+	return (
+		<video
+			className="aspect-video w-full rounded-xl bg-zinc-900 object-contain"
+			controls
+			preload="metadata"
+			src={href}
+		/>
+	);
+};
+
+export const renderRun = (manifest: SkillEvalManifest) =>
+	page({
+		children: (
+			<>
+				<Header
+					action={
+						<div className="flex flex-wrap items-center gap-3">
+							<a
+								className="text-[0.8125rem] text-zinc-600"
+								href={`/scenarios/${encodeURIComponent(manifest.id)}`}
+							>
+								Scenario
+							</a>
+							<a
+								className="text-[0.8125rem] text-zinc-600"
+								href={toFileUrl(manifest.pi.htmlExport)}
+							>
+								Pi export
+							</a>
+						</div>
+					}
+					eyebrow="Run"
+					subtitle={manifest.model}
+					title={manifest.id}
+				/>
+				<main className="grid min-w-0 gap-4">
+					<Card>
+						<Artifact manifest={manifest} />
+					</Card>
+					<details className="min-w-0 rounded-2xl border border-zinc-200 bg-white p-4">
+						<summary className="cursor-pointer text-sm font-semibold">
+							Prompt
+						</summary>
+						<pre className="mt-3 max-h-75 overflow-auto whitespace-pre-wrap rounded-xl border border-zinc-200 bg-zinc-100 p-3 text-xs text-zinc-700">
+							{manifest.prompt}
+						</pre>
+					</details>
+				</main>
+			</>
+		),
+		title: `${manifest.id} Run`,
+	});
diff --git a/packages/skills-evals/src/app/scenario.tsx b/packages/skills-evals/src/app/scenario.tsx
new file mode 100644
index 00000000000..b0bc92c1725
--- /dev/null
+++ b/packages/skills-evals/src/app/scenario.tsx
@@ -0,0 +1,372 @@
+import {basename, join} from 'node:path';
+import type {SkillEvalScenario} from '../../scenarios';
+import {runCommand} from '../command';
+import {listFilesRecursively, readJson, sanitizePathPart} from '../files';
+import type {SkillEvalComparison, SkillEvalManifest} from '../manifest';
+import {loadComparisons} from './comparison-data';
+import {
+	createJobRuns,
+	formatRunLabel,
+	getActiveJob,
+	type Job,
+	type JobRun,
+} from './jobs';
+import {
+	Card,
+	formatDate,
+	Header,
+	page,
+	repoRoot,
+	runsRoot,
+	toComparisonUrl,
+} from './shared';
+
+type SkillDiffState = {
+	hasChanges: boolean;
+	message: string;
+	title: string;
+};
+
+type ScenarioRunListItem = {
+	completedAt: string;
+	href: string;
+	metadata: string;
+};
+
+const getSkillDiffState = async (): Promise<SkillDiffState> => {
+	const result = await runCommand({
+		command: ['git', 'status', '--porcelain', '--', 'packages/skills/skills'],
+		cwd: repoRoot,
+	});
+
+	if (result.exitCode === 0) {
+		return {
+			hasChanges: result.stdout.trim().length > 0,
+			message: result.stdout.trim()
+				? 'Skills differ from HEAD. Run a before/after comparison against HEAD.'
+				: 'Skills match HEAD. This will run the scenario without a diff to review.',
+			title: result.stdout.trim()
+				? 'Skills changed from HEAD'
+				: 'No skill changes',
+		};
+	}
+
+	return {
+		hasChanges: false,
+		message: `Could not inspect skill diff. The scenario can still run without a comparison. ${result.stderr}`,
+		title: 'Skill diff unavailable',
+	};
+};
+
+const loadPlainRuns = async (
+	scenarioId: string,
+): Promise<SkillEvalManifest[]> => {
+	const manifestFiles = (
+		await listFilesRecursively(join(runsRoot, sanitizePathPart(scenarioId)))
+	).filter((file) => file.endsWith('/manifest.json'));
+
+	return Promise.all(
+		manifestFiles.map((file) => readJson<SkillEvalManifest>(file)),
+	);
+};
+
+const toRunListItems = ({
+	comparisons,
+	runs,
+}: {
+	comparisons: SkillEvalComparison[];
+	runs: SkillEvalManifest[];
+}) => {
+	const items: ScenarioRunListItem[] = [
+		...comparisons.map((comparison) => ({
+			completedAt: comparison.completedAt,
+			href: toComparisonUrl(comparison),
+			metadata: `${comparison.before.hash} -> ${comparison.after.hash}`,
+		})),
+		...runs.map((run) => ({
+			completedAt: run.completedAt,
+			href: `/runs/${encodeURIComponent(run.id)}/${encodeURIComponent(
+				basename(run.runDir),
+			)}`,
+			metadata: run.skillSnapshot.hash,
+		})),
+	];
+
+	return items.sort((a, b) => b.completedAt.localeCompare(a.completedAt));
+};
+
+const ScenarioRuns = ({items}: {items: ScenarioRunListItem[]}) => {
+	if (items.length === 0) {
+		return <p className="text-sm text-zinc-500">No runs yet.</p>;
+	}
+
+	return (
+		<div className="mt-3 grid">
+			{items.map((item) => (
+				<a
+					className="flex items-center justify-between gap-4 border-t border-zinc-100 py-3 first:border-t-0 first:pt-0 last:pb-0"
+					href={item.href}
+					key={item.href}
+				>
+					<div>
+						<h3 className="text-sm font-semibold text-zinc-800">
+							{formatDate(item.completedAt)}
+						</h3>
+						<p className="mt-1 text-[0.8125rem] text-zinc-500">
+							{item.metadata}
+						</p>
+					</div>
+				</a>
+			))}
+		</div>
+	);
+};
+
+const ScenarioPageScript = ({activeJob}: {activeJob: Job | undefined}) => (
+	<script
+		dangerouslySetInnerHTML={{
+			__html: `
+const button = document.getElementById('run-comparison');
+const panel = document.getElementById('active-job');
+const log = document.getElementById('job-log');
+const status = document.getElementById('job-status');
+const runStatus = {
+	before: document.getElementById('before-run-status'),
+	after: document.getElementById('after-run-status'),
+};
+const runLog = {
+	before: document.getElementById('before-run-log'),
+	after: document.getElementById('after-run-log'),
+};
+const activeJobId = ${JSON.stringify(activeJob?.id ?? null)};
+
+const renderJob = (job) => {
+	panel.hidden = false;
+	status.textContent = job.message || job.status;
+
+	if (job.logs?.length) {
+		log.hidden = false;
+		log.textContent = job.logs.join('');
+		log.scrollTop = log.scrollHeight;
+	}
+
+	for (const label of ['before', 'after']) {
+		const run = job.runs?.[label];
+
+		if (!run) {
+			continue;
+		}
+
+		runStatus[label].textContent = run.message || run.status;
+		runStatus[label].dataset.status = run.status;
+
+		if (run.logs?.length) {
+			runLog[label].hidden = false;
+			runLog[label].textContent = run.logs.join('');
+			runLog[label].scrollTop = runLog[label].scrollHeight;
+		}
+	}
+};
+
+const poll = async (jobId) => {
+	const response = await fetch('/api/jobs/' + encodeURIComponent(jobId));
+	const job = await response.json();
+	renderJob(job);
+
+	if (job.status === 'completed' && job.comparisonUrl) {
+		location.href = job.comparisonUrl;
+		return;
+	}
+
+	if (job.status === 'completed' && job.resultUrl) {
+		location.href = job.resultUrl;
+		return;
+	}
+
+	if (job.status === 'failed' || job.status === 'skipped') {
+		button.disabled = false;
+		return;
+	}
+
+	setTimeout(() => poll(jobId), 2000);
+};
+
+if (activeJobId) {
+	button.disabled = true;
+	log.hidden = false;
+	poll(activeJobId);
+}
+
+button?.addEventListener('click', async () => {
+	button.disabled = true;
+	panel.hidden = false;
+	status.textContent = 'Starting comparison...';
+	for (const label of ['before', 'after']) {
+		runStatus[label].textContent = 'Waiting to start';
+		runStatus[label].dataset.status = 'waiting';
+		runLog[label].hidden = true;
+		runLog[label].textContent = '';
+	}
+	const response = await fetch(button.dataset.actionUrl, {
+		method: 'POST',
+	});
+	const job = await response.json();
+	poll(job.id);
+});
+`,
+		}}
+	/>
+);
+
+const RunProgressCard = ({
+	jobRun,
+	label,
+}: {
+	jobRun: JobRun;
+	label: 'after' | 'before';
+}) => (
+	<section className="min-w-0 rounded-xl border border-zinc-200 bg-zinc-50 p-3">
+		<div className="flex items-center justify-between gap-3">
+			<h3 className="text-sm font-semibold">{formatRunLabel(label)}</h3>
+			<span
+				className="rounded-full bg-white px-2 py-1 text-xs text-zinc-500 data-[status=completed]:text-emerald-700 data-[status=failed]:text-red-700 data-[status=running]:text-yellow-700"
+				data-status={jobRun.status}
+				id={`${label}-run-status`}
+			>
+				{jobRun.message}
+			</span>
+		</div>
+		<pre
+			className="mt-3 max-h-70 overflow-auto whitespace-pre-wrap rounded-lg border border-zinc-200 bg-white p-3 text-xs text-zinc-700"
+			hidden={jobRun.logs.length === 0}
+			id={`${label}-run-log`}
+		>
+			{jobRun.logs.join('')}
+		</pre>
+	</section>
+);
+
+const ActiveJobPanel = ({
+	hasSkillChanges,
+	job,
+}: {
+	hasSkillChanges: boolean;
+	job: Job | undefined;
+}) => {
+	const runs = job?.runs ?? createJobRuns();
+
+	return (
+		<section
+			className="min-w-0 rounded-2xl border border-zinc-200 bg-white p-4"
+			hidden={!job}
+			id="active-job"
+		>
+			<div>
+				<h2 className="text-[0.9375rem] font-semibold">Active run</h2>
+				<p className="text-sm text-zinc-500">
+					{hasSkillChanges
+						? 'Before and after run in parallel once the skill diff is ready.'
+						: 'Scenario run without a before/after comparison.'}
+				</p>
+			</div>
+			<p className="mt-3 text-[0.8125rem] text-yellow-700" id="job-status">
+				{job?.message ?? ''}
+			</p>
+			<pre
+				className="mt-3 max-h-40 overflow-auto whitespace-pre-wrap rounded-xl border border-zinc-200 bg-zinc-100 p-3 text-xs text-zinc-700"
+				hidden={!job?.logs.length}
+				id="job-log"
+			>
+				{job?.logs.join('') ?? ''}
+			</pre>
+			<div className="mt-3 grid grid-cols-2 gap-3 max-lg:grid-cols-1">
+				<RunProgressCard jobRun={runs.before} label="before" />
+				<RunProgressCard jobRun={runs.after} label="after" />
+			</div>
+		</section>
+	);
+};
+
+export const renderScenario = async (scenario: SkillEvalScenario) => {
+	const activeJob = getActiveJob(scenario.id);
+	const skillDiffState = await getSkillDiffState();
+	const comparisons = (await loadComparisons()).filter(
+		(comparison) => comparison.scenarioId === scenario.id,
+	);
+	const runs = await loadPlainRuns(scenario.id);
+	const runItems = toRunListItems({comparisons, runs});
+
+	return page({
+		children: (
+			<>
+				<Header
+					action={
+						<button
+							className="rounded-full bg-zinc-900 px-3 py-2 text-[0.8125rem] font-semibold text-white hover:bg-zinc-800 disabled:bg-zinc-300 disabled:text-zinc-500"
+							data-action-url={
+								skillDiffState.hasChanges
+									? `/api/compare/${encodeURIComponent(scenario.id)}`
+									: `/api/run/${encodeURIComponent(scenario.id)}`
+							}
+							data-scenario={scenario.id}
+							id="run-comparison"
+						>
+							{skillDiffState.hasChanges ? 'Run comparison' : 'Run'}
+						</button>
+					}
+					subtitle={scenario.model}
+					title={scenario.id}
+				/>
+				<main className="grid min-w-0 gap-4">
+					<section
+						className={`min-w-0 rounded-2xl border p-4 ${
+							skillDiffState.hasChanges
+								? 'border-amber-200 bg-amber-50'
+								: 'border-zinc-200 bg-white'
+						}`}
+					>
+						<div className="flex items-start justify-between gap-4 max-md:flex-col">
+							<div>
+								<h2 className="text-[0.9375rem] font-semibold">
+									{skillDiffState.title}
+								</h2>
+								<p
+									className={`mt-1 text-sm ${
+										skillDiffState.hasChanges
+											? 'text-amber-800'
+											: 'text-zinc-500'
+									}`}
+								>
+									{skillDiffState.message}
+								</p>
+							</div>
+							{skillDiffState.hasChanges ? (
+								<span className="rounded-full bg-amber-100 px-2 py-1 text-xs text-amber-800">
+									Compare against HEAD
+								</span>
+							) : null}
+						</div>
+					</section>
+					<details className="min-w-0 rounded-2xl border border-zinc-200 bg-white p-4">
+						<summary className="cursor-pointer text-sm font-semibold">
+							Prompt
+						</summary>
+						<pre className="mt-3 max-h-75 overflow-auto whitespace-pre-wrap rounded-xl border border-zinc-200 bg-zinc-100 p-3 text-xs text-zinc-700">
+							{scenario.prompt}
+						</pre>
+					</details>
+					<ActiveJobPanel
+						hasSkillChanges={skillDiffState.hasChanges}
+						job={activeJob}
+					/>
+					<Card>
+						<h2 className="text-[0.9375rem] font-semibold">Runs</h2>
+						<ScenarioRuns items={runItems} />
+					</Card>
+				</main>
+				<ScenarioPageScript activeJob={activeJob} />
+			</>
+		),
+		title: scenario.id,
+	});
+};
diff --git a/packages/skills-evals/src/app/shared.tsx b/packages/skills-evals/src/app/shared.tsx
new file mode 100644
index 00000000000..044d7f2ea64
--- /dev/null
+++ b/packages/skills-evals/src/app/shared.tsx
@@ -0,0 +1,132 @@
+import {dirname, join, relative, resolve} from 'node:path';
+import {fileURLToPath} from 'node:url';
+import {renderToStaticMarkup} from 'react-dom/server';
+import type {SkillEvalComparison} from '../manifest';
+
+export const packageRoot = resolve(
+	dirname(fileURLToPath(import.meta.url)),
+	'..',
+	'..',
+);
+export const repoRoot = resolve(packageRoot, '..', '..');
+export const runsRoot = join(packageRoot, '.runs');
+export const comparisonsRoot = join(runsRoot, 'comparisons');
+export const port = Number(process.env.PORT ?? 4321);
+export const origin = `http://localhost:${port}`;
+
+const AppLayout = ({children}: {children: React.ReactNode}) => (
+	<div className="isolate mx-auto max-w-6xl px-6 py-6 max-md:px-4">
+		<nav className="mb-8 border-b border-zinc-200 pb-4">
+			<a
+				className="text-[0.8125rem] font-medium text-zinc-500 hover:text-zinc-900"
+				href="/"
+			>
+				Scenarios
+			</a>
+		</nav>
+		{children}
+	</div>
+);
+
+export const page = ({
+	children,
+	title,
+}: {
+	children: React.ReactNode;
+	title: string;
+}) =>
+	`<!doctype html>${renderToStaticMarkup(
+		<html>
+			<head>
+				<meta charSet="utf-8" />
+				<meta name="viewport" content="width=device-width, initial-scale=1" />
+				<title>{title}</title>
+				<script src="https://cdn.jsdelivr.net/npm/@tailwindcss/browser@4" />
+			</head>
+			<body className="bg-zinc-50 text-zinc-900 antialiased">
+				<AppLayout>{children}</AppLayout>
+			</body>
+		</html>,
+	)}`;
+
+export const Header = ({
+	action,
+	eyebrow,
+	subtitle,
+	title,
+}: {
+	action?: React.ReactNode;
+	eyebrow?: string;
+	subtitle: string;
+	title: string;
+}) => (
+	<header className="mb-6">
+		<div className="min-w-0">
+			{eyebrow ? (
+				<p className="mb-2 text-[0.8125rem] font-medium text-zinc-400">
+					{eyebrow}
+				</p>
+			) : null}
+			<div className="flex items-center justify-between gap-5 max-md:flex-col max-md:items-start">
+				<h1 className="text-3xl font-semibold tracking-tight text-balance">
+					{title}
+				</h1>
+				{action}
+			</div>
+			<p className="text-sm text-zinc-500 text-pretty">{subtitle}</p>
+		</div>
+	</header>
+);
+
+export const Card = ({children}: {children: React.ReactNode}) => (
+	<section className="min-w-0 rounded-2xl border border-zinc-200 bg-white p-4">
+		{children}
+	</section>
+);
+
+export const Pill = ({children}: {children: React.ReactNode}) => (
+	<span className="rounded-full bg-zinc-100 px-2 py-1 text-xs text-zinc-600">
+		{children}
+	</span>
+);
+
+export const formatDate = (value: string) =>
+	new Intl.DateTimeFormat('en', {
+		dateStyle: 'medium',
+		timeStyle: 'medium',
+	}).format(new Date(value));
+
+export const toFileUrl = (file: string) => {
+	const relativePath = relative(runsRoot, file);
+
+	if (relativePath.startsWith('..')) {
+		throw new Error(`Cannot serve file outside .runs: ${file}`);
+	}
+
+	return `/files/${relativePath.split(/[\\/]/).map(encodeURIComponent).join('/')}`;
+};
+
+export const toComparisonUrl = (comparison: SkillEvalComparison) =>
+	`/comparisons/${encodeURIComponent(comparison.scenarioId)}/${encodeURIComponent(
+		comparison.id,
+	)}`;
+
+export const json = (value: unknown, init?: ResponseInit) =>
+	new Response(JSON.stringify(value), {
+		...init,
+		headers: {
+			'content-type': 'application/json',
+			...init?.headers,
+		},
+	});
+
+export const htmlResponse = (value: string, init?: ResponseInit) =>
+	new Response(value, {
+		...init,
+		headers: {
+			'content-type': 'text/html; charset=utf-8',
+			...init?.headers,
+		},
+	});
+
+export const notFound = () => htmlResponse('Not found', {status: 404});
diff --git a/packages/skills-evals/src/cli.ts b/packages/skills-evals/src/cli.ts
new file mode 100644
index 00000000000..82297a756b1
--- /dev/null
+++ b/packages/skills-evals/src/cli.ts
@@ -0,0 +1,72 @@
+import {scenarios} from '../scenarios';
+import {runSkillEvalComparison} from './compare';
+import {runSkillEval} from './run-skill-eval';
+
+const serverUrl = 'http://localhost:4321';
+
+const getScenario = (id: string | undefined) => {
+	if (!id) {
+		if (scenarios.length === 1) {
+			return scenarios[0];
+		}
+
+		throw new Error('Pass a scenario id.');
+	}
+
+	const scenario = scenarios.find((candidate) => candidate.id === id);
+
+	if (!scenario) {
+		throw new Error(`Unknown scenario "${id}". Run "bun run eval list".`);
+	}
+
+	return scenario;
+};
+
+const main = async () => {
+	const command = process.argv[2];
+
+	if (command === 'dev') {
+		await import('./server');
+	} else if (command === 'compare') {
+		const scenario = getScenario(process.argv[3]);
+		process.stdout.write(`Comparing ${scenario.id} with ${scenario.model}\n`);
+		const result = await runSkillEvalComparison(scenario, {
+			onLog: (chunk) => process.stdout.write(chunk),
+		});
+
+		if (result.skipped) {
+			process.stdout.write(`${result.reason}\n`);
+			return;
+		}
+
+		process.stdout.write(
+			`${scenario.id}: ${result.comparison.before.hash} -> ${result.comparison.after.hash}\n`,
+		);
+		process.stdout.write(`Preview: ${serverUrl}\n`);
+	} else if (command === 'list') {
+		for (const scenario of scenarios) {
+			process.stdout.write(`${scenario.id}\t${scenario.model}\n`);
+		}
+	} else if (command === 'run') {
+		const scenarioId = process.argv[3];
+		if (!scenarioId) {
+			throw new Error('Pass a scenario id or --all.');
+		}
+
+		const scenariosToRun =
+			scenarioId === '--all' ? scenarios : [getScenario(scenarioId)];
+
+		for (const scenario of scenariosToRun) {
+			process.stdout.write(`Running ${scenario.id} with ${scenario.model}\n`);
+			const result = await runSkillEval(scenario);
+			process.stdout.write(`Wrote ${result.manifestPath}\n`);
+		}
+	} else {
+		process.stdout.write(
+			'Usage: bun run eval <list|run|compare|dev> [scenario-id|--all]\n',
+		);
+		process.exit(command ? 1 : 0);
+	}
+};
+
+main();
diff --git a/packages/skills-evals/src/command.ts b/packages/skills-evals/src/command.ts
new file mode 100644
index 00000000000..ef385066e9b
--- /dev/null
+++ b/packages/skills-evals/src/command.ts
@@ -0,0 +1,188 @@
+export type CommandResult = {
+	aborted: boolean;
+	command: string[];
+	cwd: string;
+	exitCode: number;
+	stdout: string;
+	stderr: string;
+	timedOut: boolean;
+	durationMs: number;
+};
+
+export type CommandOutput = {
+	chunk: string;
+	stream: 'stderr' | 'stdout';
+};
+
+const collectStream = async ({
+	onOutput,
+	redactOutput,
+	stream,
+	streamName,
+}: {
+	onOutput?: (output: CommandOutput) => void;
+	redactOutput: (value: string) => string;
+	stream: ReadableStream<Uint8Array>;
+	streamName: CommandOutput['stream'];
+}) => {
+	const reader = stream.getReader();
+	const decoder = new TextDecoder();
+	let output = '';
+
+	while (true) {
+		const {done, value} = await reader.read();
+
+		if (done) {
+			break;
+		}
+
+		const chunk = decoder.decode(value, {stream: true});
+		const redactedChunk = redactOutput(chunk);
+		output += chunk;
+		onOutput?.({chunk: redactedChunk, stream: streamName});
+	}
+
+	const finalChunk = decoder.decode();
+	if (finalChunk) {
+		const redactedChunk = redactOutput(finalChunk);
+		output += finalChunk;
+		onOutput?.({chunk: redactedChunk, stream: streamName});
+	}
+
+	return redactOutput(output);
+};
+
+const sensitiveEnvKey = /(token|key|secret|password|credential|auth|cookie)/i;
+
+const createCommandEnv = (env: Record<string, string | undefined> = {}) => ({
+	...process.env,
+	...env,
+});
+
+const createOutputRedactor = (env: Record<string, string | undefined>) => {
+	const sensitiveValues = Object.entries(env)
+		.filter(
+			([key, value]) => sensitiveEnvKey.test(key) && value && value.length > 7,
+		)
+		.map(([, value]) => value as string)
+		.sort((a, b) => b.length - a.length);
+
+	return (value: string) => {
+		let redacted = value;
+
+		for (const sensitiveValue of sensitiveValues) {
+			redacted = redacted.split(sensitiveValue).join('[redacted]');
+		}
+
+		return redacted;
+	};
+};
+
+export const runCommand = async ({
+	command,
+	cwd,
+	env,
+	onOutput,
+	signal,
+	timeoutMs,
+}: {
+	command: string[];
+	cwd: string;
+	env?: Record<string, string | undefined>;
+	onOutput?: (output: CommandOutput) => void;
+	signal?: AbortSignal;
+	timeoutMs?: number;
+}): Promise<CommandResult> => {
+	const startedAt = Date.now();
+	const commandEnv = createCommandEnv(env);
+	const redactOutput = createOutputRedactor(commandEnv);
+	const subprocess = Bun.spawn(command, {
+		cwd,
+		detached: process.platform !== 'win32',
+		env: commandEnv,
+		stdout: 'pipe',
+		stderr: 'pipe',
+	});
+	let aborted = false;
+	let timedOut = false;
+	let forceKillTimeout: ReturnType<typeof setTimeout> | null = null;
+	const killSubprocess = (killSignal: 'SIGKILL' | 'SIGTERM') => {
+		try {
+			if (process.platform !== 'win32') {
+				process.kill(-subprocess.pid, killSignal);
+				return;
+			}
+		} catch {
+			// Fall back to killing the direct child if the process group is gone.
+		}
+
+		subprocess.kill(killSignal);
+	};
+
+	const terminateSubprocess = () => {
+		killSubprocess('SIGTERM');
+		forceKillTimeout = setTimeout(() => {
+			killSubprocess('SIGKILL');
+		}, 2000);
+	};
+
+	const onAbort = () => {
+		aborted = true;
+		terminateSubprocess();
+	};
+
+	if (signal?.aborted) {
+		onAbort();
+	} else {
+		signal?.addEventListener('abort', onAbort, {once: true});
+	}
+
+	const timeout = timeoutMs
+		? setTimeout(() => {
+				timedOut = true;
+				terminateSubprocess();
+			}, timeoutMs)
+		: null;
+
+	const [stdout, stderr, exitCode] = await Promise.all([
+		collectStream({
+			onOutput,
+			redactOutput,
+			stream: subprocess.stdout,
+			streamName: 'stdout',
+		}),
+		collectStream({
+			onOutput,
+			redactOutput,
+			stream: subprocess.stderr,
+			streamName: 'stderr',
+		}),
+		subprocess.exited,
+	]);
+
+	if (timeout) {
+		clearTimeout(timeout);
+	}
+
+	if (forceKillTimeout) {
+		clearTimeout(forceKillTimeout);
+	}
+
+	signal?.removeEventListener('abort', onAbort);
+
+	return {
+		aborted,
+		command,
+		cwd,
+		exitCode,
+		stdout,
+		stderr,
+		timedOut,
+		durationMs: Date.now() - startedAt,
+	};
+};
+
+export const commandToString = (command: string[]) =>
+	command
+		.map((part) => (/\s/.test(part) ? JSON.stringify(part) : part))
+		.join(' ');
diff --git a/packages/skills-evals/src/compare.ts b/packages/skills-evals/src/compare.ts
new file mode 100644
index 00000000000..a686c3fb175
--- /dev/null
+++ b/packages/skills-evals/src/compare.ts
@@ -0,0 +1,293 @@
+import {mkdir, rm, writeFile} from 'node:fs/promises';
+import {dirname, join, resolve} from 'node:path';
+import {fileURLToPath} from 'node:url';
+import type {SkillEvalScenario} from '../scenarios';
+import {runCommand} from './command';
+import {
+	copyDirectory,
+	createTimestamp,
+	sanitizePathPart,
+	writeJson,
+} from './files';
+import type {SkillEvalComparison} from './manifest';
+import {
+	runSkillEval,
+	type SkillEvalOutput,
+	type SkillEvalPhaseEvent,
+} from './run-skill-eval';
+
+export type SkillEvalComparisonRunLabel = 'after' | 'before';
+
+export type SkillEvalComparisonEvent =
+	| {
+			message: string;
+			type: 'message';
+	  }
+	| {
+			label: SkillEvalComparisonRunLabel;
+			type: 'run-start';
+	  }
+	| {
+			event: SkillEvalPhaseEvent;
+			label: SkillEvalComparisonRunLabel;
+			type: 'run-phase';
+	  }
+	| {
+			label: SkillEvalComparisonRunLabel;
+			output: SkillEvalOutput;
+			type: 'run-output';
+	  }
+	| {
+			label: SkillEvalComparisonRunLabel;
+			type: 'run-complete';
+	  }
+	| {
+			error: string;
+			label: SkillEvalComparisonRunLabel;
+			type: 'run-error';
+	  };
+
+type SkillEvalComparisonResult =
+	| {
+			skipped: false;
+			comparison: SkillEvalComparison;
+	  }
+	| {
+			reason: string;
+			skipped: true;
+	  };
+
+type SkillEvalComparisonOptions = {
+	onEvent?: (event: SkillEvalComparisonEvent) => void;
+	onLog?: (chunk: string) => void;
+};
+
+type BeforeSkillsSource = {
+	gitRef: 'HEAD';
+	source: 'head';
+};
+
+const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const repoRoot = resolve(packageRoot, '..', '..');
+const runsRoot = resolve(packageRoot, '.runs');
+const skillsSource = resolve(packageRoot, '..', 'skills', 'skills');
+const comparisonsRoot = join(runsRoot, 'comparisons');
+
+const getErrorMessage = (error: unknown) =>
+	error instanceof Error ? error.message : String(error);
+
+const copyGitSkills = async ({gitRef, to}: {gitRef: string; to: string}) => {
+	await mkdir(to, {recursive: true});
+	const archive = await runCommand({
+		command: [
+			'sh',
+			'-c',
+			'git archive "$1":packages/skills/skills | tar -x -C "$2"',
+			'sh',
+			gitRef,
+			to,
+		],
+		cwd: repoRoot,
+	});
+
+	if (archive.exitCode !== 0) {
+		throw new Error(
+			`Could not materialize skills at ${gitRef}: ${archive.stderr}`,
+		);
+	}
+};
+
+const prepareBeforeSkills = async ({
+	beforeSkillsPath,
+}: {
+	beforeSkillsPath: string;
+}): Promise<BeforeSkillsSource> => {
+	await copyGitSkills({gitRef: 'HEAD', to: beforeSkillsPath});
+	return {
+		gitRef: 'HEAD',
+		source: 'head',
+	};
+};
+
+export const runSkillEvalComparison = async (
+	scenario: SkillEvalScenario,
+	options: SkillEvalComparisonOptions = {},
+): Promise<SkillEvalComparisonResult> => {
+	const comparisonId = `${createTimestamp()}--${sanitizePathPart(scenario.id)}`;
+	const comparisonDir = join(
+		comparisonsRoot,
+		sanitizePathPart(scenario.id),
+		comparisonId,
+	);
+	const beforeSkillsPath = join(comparisonDir, 'before-skills');
+	const afterSkillsPath = join(comparisonDir, 'after-skills');
+	const skillDiffPath = join(comparisonDir, 'skills.diff');
+	const createdAt = new Date().toISOString();
+	const emitMessage = (message: string) => {
+		options.onEvent?.({message, type: 'message'});
+		options.onLog?.(message);
+	};
+
+	await rm(comparisonDir, {force: true, recursive: true});
+	await mkdir(comparisonDir, {recursive: true});
+
+	emitMessage(`[compare] Preparing ${scenario.id}\n`);
+	const beforeSource = await prepareBeforeSkills({
+		beforeSkillsPath,
+	});
+
+	await copyDirectory(skillsSource, afterSkillsPath);
+
+	emitMessage('[compare] Diffing skill snapshots\n');
+	const diff = await runCommand({
+		command: [
+			'git',
+			'diff',
+			'--no-index',
+			'--',
+			beforeSkillsPath,
+			afterSkillsPath,
+		],
+		cwd: repoRoot,
+	});
+
+	if (diff.exitCode === 0) {
+		await rm(comparisonDir, {force: true, recursive: true});
+		return {
+			reason: 'No skill changes since HEAD.',
+			skipped: true,
+		};
+	}
+
+	if (diff.exitCode !== 1) {
+		throw new Error(`Could not diff skill snapshots: ${diff.stderr}`);
+	}
+
+	await writeFile(skillDiffPath, `${diff.stdout}${diff.stderr}`);
+
+	const forwardOutput =
+		(label: SkillEvalComparisonRunLabel) => (output: SkillEvalOutput) => {
+			options.onEvent?.({label, output, type: 'run-output'});
+			options.onLog?.(
+				`[${label} ${output.phase} ${output.stream}] ${output.chunk}`,
+			);
+		};
+
+	const forwardPhase =
+		(label: SkillEvalComparisonRunLabel) => (event: SkillEvalPhaseEvent) => {
+			options.onEvent?.({event, label, type: 'run-phase'});
+			options.onLog?.(`[${label} ${event.phase}] ${event.status}\n`);
+		};
+
+	const abortControllers: Record<SkillEvalComparisonRunLabel, AbortController> =
+		{
+			after: new AbortController(),
+			before: new AbortController(),
+		};
+	let firstSnapshotError: unknown = null;
+	const runSnapshot = async (label: SkillEvalComparisonRunLabel) => {
+		emitMessage(`[compare] Running ${label} snapshot\n`);
+		options.onEvent?.({label, type: 'run-start'});
+
+		try {
+			const result = await runSkillEval({
+				...scenario,
+				onOutput: forwardOutput(label),
+				onPhase: forwardPhase(label),
+				runLabel: label,
+				runRoot: join(comparisonDir, 'runs'),
+				signal: abortControllers[label].signal,
+				skillSnapshot:
+					label === 'before'
+						? {
+								gitRef: beforeSource.gitRef,
+								label,
+							}
+						: {
+								isWorkingTree: true,
+								label,
+							},
+				skillsSourcePath:
+					label === 'before' ? beforeSkillsPath : afterSkillsPath,
+			});
+
+			options.onEvent?.({label, type: 'run-complete'});
+			return result;
+		} catch (error) {
+			options.onEvent?.({
+				error: getErrorMessage(error),
+				label,
+				type: 'run-error',
+			});
+			throw error;
+		}
+	};
+
+	const runSnapshotWithSiblingCancellation = async (
+		label: SkillEvalComparisonRunLabel,
+		sibling: SkillEvalComparisonRunLabel,
+	) => {
+		try {
+			return await runSnapshot(label);
+		} catch (error) {
+			if (!firstSnapshotError) {
+				firstSnapshotError = error;
+				abortControllers[sibling].abort();
+			}
+
+			throw error;
+		}
+	};
+
+	const [beforeResult, afterResult] = await Promise.allSettled([
+		runSnapshotWithSiblingCancellation('before', 'after'),
+		runSnapshotWithSiblingCancellation('after', 'before'),
+	]);
+
+	if (firstSnapshotError) {
+		throw firstSnapshotError;
+	}
+
+	if (beforeResult.status === 'rejected') {
+		throw beforeResult.reason;
+	}
+
+	if (afterResult.status === 'rejected') {
+		throw afterResult.reason;
+	}
+
+	const before = beforeResult.value;
+	const after = afterResult.value;
+	const completedAt = new Date().toISOString();
+	const comparison: SkillEvalComparison = {
+		after: {
+			hash: after.manifest.skillSnapshot.hash,
+			isWorkingTree: true,
+			label: 'after',
+			manifestPath: after.manifestPath,
+			skillsPath: afterSkillsPath,
+		},
+		before: {
+			gitRef: beforeSource.gitRef,
+			hash: before.manifest.skillSnapshot.hash,
+			label: 'before',
+			manifestPath: before.manifestPath,
+			source: beforeSource.source,
+			skillsPath: beforeSkillsPath,
+		},
+		completedAt,
+		comparisonDir,
+		createdAt,
+		id: comparisonId,
+		scenarioId: scenario.id,
+		skillDiffPath,
+	};
+
+	await writeJson(join(comparisonDir, 'comparison.json'), comparison);
+	emitMessage('[compare] Comparison complete\n');
+
+	return {
+		comparison,
+		skipped: false,
+	};
+};
diff --git a/packages/skills-evals/src/files.ts b/packages/skills-evals/src/files.ts
new file mode 100644
index 00000000000..bc77b6024ba
--- /dev/null
+++ b/packages/skills-evals/src/files.ts
@@ -0,0 +1,69 @@
+import {existsSync} from 'node:fs';
+import {cp, lstat, readdir, readFile, rm, writeFile} from 'node:fs/promises';
+import {join} from 'node:path';
+
+export const createTimestamp = () =>
+	new Date().toISOString().replace(/[:.]/g, '-');
+
+export const sanitizePathPart = (value: string) => {
+	return value
+		.toLowerCase()
+		.replace(/[^a-z0-9._-]+/g, '-')
+		.replace(/^-+|-+$/g, '')
+		.slice(0, 80);
+};
+
+export const copyDirectory = async (from: string, to: string) => {
+	await rm(to, {force: true, recursive: true});
+	await cp(from, to, {
+		recursive: true,
+		filter: async (source) => {
+			const pathParts = source.split(/[\\/]/);
+
+			if ((await lstat(source)).isSymbolicLink()) {
+				return false;
+			}
+
+			return (
+				!pathParts.includes('node_modules') && !pathParts.includes('.DS_Store')
+			);
+		},
+	});
+};
+
+export const readJson = async <T>(file: string): Promise<T> => {
+	return JSON.parse(await readFile(file, 'utf-8')) as T;
+};
+
+export const writeJson = async (file: string, value: unknown) => {
+	await writeFile(file, `${JSON.stringify(value, null, 2)}\n`);
+};
+
+export const listFilesRecursively = async (dir: string): Promise<string[]> => {
+	if (!existsSync(dir)) {
+		return [];
+	}
+
+	const entries = await readdir(dir, {withFileTypes: true});
+	const files = await Promise.all(
+		entries.map((entry) => {
+			const absolutePath = join(dir, entry.name);
+
+			if (entry.isSymbolicLink()) {
+				return [];
+			}
+
+			if (entry.isDirectory()) {
+				if (entry.name === 'node_modules') {
+					return [];
+				}
+
+				return listFilesRecursively(absolutePath);
+			}
+
+			return [absolutePath];
+		}),
+	);
+
+	return files.flat().sort();
+};
diff --git a/packages/skills-evals/src/index.ts b/packages/skills-evals/src/index.ts
new file mode 100644
index 00000000000..dd146d21d98
--- /dev/null
+++ b/packages/skills-evals/src/index.ts
@@ -0,0 +1,9 @@
+export {runSkillEvalComparison} from './compare';
+export {runSkillEval} from './run-skill-eval';
+export {scenarios} from '../scenarios';
+export type {
+	SkillEvalComparison,
+	SkillEvalManifest,
+	SkillEvalResult,
+} from './manifest';
+export type {SkillEvalScenario} from '../scenarios';
diff --git a/packages/skills-evals/src/manifest.ts b/packages/skills-evals/src/manifest.ts
new file mode 100644
index 00000000000..e1c8e2b06c3
--- /dev/null
+++ b/packages/skills-evals/src/manifest.ts
@@ -0,0 +1,80 @@
+import type {SkillEvalScenario} from '../scenarios';
+import type {CommandResult} from './command';
+
+export type SkillSnapshot = {
+	sourcePath: string;
+	sandboxPath: string;
+	hash: string;
+	label?: 'before' | 'after';
+	gitRef?: string;
+	comparisonId?: string;
+	isWorkingTree?: boolean;
+};
+
+export type VisualArtifact = {
+	path: string;
+	relativePath: string;
+	sizeInBytes: number;
+	type: 'image' | 'video';
+};
+
+export type LoggedCommand = Pick<
+	CommandResult,
+	'command' | 'cwd' | 'durationMs' | 'exitCode'
+> & {
+	stdoutPath: string;
+	stderrPath: string;
+};
+
+export type SkillEvalManifest = {
+	id: string;
+	model: string;
+	prompt: string;
+	createdAt: string;
+	completedAt: string;
+	runDir: string;
+	projectRoot: string;
+	skillSnapshot: SkillSnapshot;
+	pi: {
+		sessionFile: string;
+		htmlExport: string;
+	};
+	artifacts: VisualArtifact[];
+	commands: {
+		install: LoggedCommand;
+		pi: LoggedCommand;
+		piRender: LoggedCommand;
+		piExport: LoggedCommand;
+	};
+};
+
+export type SkillEvalResult = {
+	manifest: SkillEvalManifest;
+	manifestPath: string;
+	scenario: SkillEvalScenario;
+};
+
+export type SkillEvalComparison = {
+	id: string;
+	scenarioId: string;
+	createdAt: string;
+	completedAt: string;
+	comparisonDir: string;
+	skillDiffPath: string;
+	before: {
+		label: 'before';
+		source: 'head';
+		skillsPath: string;
+		hash: string;
+		gitRef?: string;
+		comparisonId?: string;
+		manifestPath: string;
+	};
+	after: {
+		label: 'after';
+		skillsPath: string;
+		hash: string;
+		isWorkingTree: boolean;
+		manifestPath: string;
+	};
+};
diff --git a/packages/skills-evals/src/pi-stream-extension.ts b/packages/skills-evals/src/pi-stream-extension.ts
new file mode 100644
index 00000000000..694b83c45e1
--- /dev/null
+++ b/packages/skills-evals/src/pi-stream-extension.ts
@@ -0,0 +1,243 @@
+// Note: this is a 100% AI slop extension to stream eval progress to stderr.
+// If it breaks - safe to remove or fix. It's just a cosmetic thing (live updating logs)
+
+type ExtensionAPI = {
+	getFlag: (name: string) => string | undefined;
+	on: (
+		event: string,
+		handler: (event: unknown) => void | Promise<void>,
+	) => void;
+	registerFlag: (
+		name: string,
+		options: {
+			default: string;
+			description: string;
+			type: 'string';
+		},
+	) => void;
+};
+
+type StreamOption = 'message' | 'thinking' | 'tools';
+
+type AssistantMessageEvent = {
+	delta?: unknown;
+	type: string;
+};
+
+type ToolEvent = {
+	args?: unknown;
+	isError?: unknown;
+	partialResult?: unknown;
+	result?: unknown;
+	toolName?: unknown;
+};
+
+const isRecord = (value: unknown): value is Record<string, unknown> =>
+	typeof value === 'object' && value !== null;
+
+const getAssistantMessageEvent = (
+	event: unknown,
+): AssistantMessageEvent | null => {
+	if (!isRecord(event)) {
+		return null;
+	}
+
+	const {assistantMessageEvent} = event;
+	if (
+		!isRecord(assistantMessageEvent) ||
+		typeof assistantMessageEvent.type !== 'string'
+	) {
+		return null;
+	}
+
+	return assistantMessageEvent as AssistantMessageEvent;
+};
+
+const getToolEvent = (event: unknown): ToolEvent | null =>
+	isRecord(event) ? event : null;
+
+const toText = (value: unknown) =>
+	typeof value === 'string' ? value : JSON.stringify(value, null, 2);
+
+const truncate = (value: string, maxLength: number) =>
+	value.length > maxLength ? `${value.slice(0, maxLength)}...` : value;
+
+const parseJsonOrText = (value: string): unknown => {
+	try {
+		return JSON.parse(value);
+	} catch {
+		return value;
+	}
+};
+
+const parseArgs = (args: unknown): Record<string, unknown> | null => {
+	if (!args) {
+		return null;
+	}
+
+	if (typeof args === 'string') {
+		try {
+			const parsed = JSON.parse(args);
+
+			return isRecord(parsed) ? parsed : null;
+		} catch {
+			return null;
+		}
+	}
+
+	return isRecord(args) ? args : null;
+};
+
+const extractText = (result: unknown): string | null => {
+	const value = typeof result === 'string' ? parseJsonOrText(result) : result;
+
+	if (!isRecord(value) || !Array.isArray(value.content)) {
+		return typeof value === 'string' ? value : null;
+	}
+
+	const texts = value.content
+		.filter(
+			(item) =>
+				isRecord(item) && item.type === 'text' && typeof item.text === 'string',
+		)
+		.map((item) => item.text);
+
+	return texts.length > 0 ? texts.join('\n') : null;
+};
+
+const formatToolArgs = (toolName: string, args: unknown) => {
+	const parsed = parseArgs(args);
+
+	if (!parsed) {
+		return truncate(toText(args), 500);
+	}
+
+	if (toolName === 'bash' && typeof parsed.command === 'string') {
+		return `$ ${parsed.command}`;
+	}
+
+	if (toolName === 'read' && typeof parsed.path === 'string') {
+		return parsed.path;
+	}
+
+	if (toolName === 'edit' || toolName === 'write') {
+		return typeof parsed.path === 'string'
+			? parsed.path
+			: truncate(toText(parsed), 500);
+	}
+
+	return truncate(
+		Object.entries(parsed)
+			.map(([key, value]) => `${key}: ${truncate(toText(value), 120)}`)
+			.join(', '),
+		500,
+	);
+};
+
+const formatToolResult = (result: unknown) => {
+	const text = extractText(result);
+
+	return truncate(text ?? toText(result), 1500);
+};
+
+const getStreamOptions = (pi: ExtensionAPI) => {
+	const flag = pi.getFlag('eval-stream');
+
+	if (!flag || flag === 'off') {
+		return new Set<StreamOption>();
+	}
+
+	const parts = flag.split(',').map((part) => part.trim().toLowerCase());
+
+	if (parts.includes('all')) {
+		return new Set<StreamOption>(['message', 'thinking', 'tools']);
+	}
+
+	const validOptions: StreamOption[] = ['message', 'thinking', 'tools'];
+
+	return new Set(
+		parts.filter((part): part is StreamOption =>
+			validOptions.includes(part as StreamOption),
+		),
+	);
+};
+
+const writeLog = (message: string) => {
+	process.stderr.write(`${message}\n`);
+};
+
+export default function evalStreamExtension(pi: ExtensionAPI): void {
+	pi.registerFlag('eval-stream', {
+		default: 'off',
+		description:
+			'Stream eval progress to stderr. Comma-separated values: message, thinking, tools, all',
+		type: 'string',
+	});
+
+	pi.on('message_update', (event) => {
+		const options = getStreamOptions(pi);
+		const assistantEvent = getAssistantMessageEvent(event);
+
+		if (!assistantEvent) {
+			return;
+		}
+
+		if (assistantEvent.type === 'thinking_delta' && options.has('thinking')) {
+			process.stderr.write(String(assistantEvent.delta ?? ''));
+			return;
+		}
+
+		if (assistantEvent.type === 'text_delta' && options.has('message')) {
+			process.stderr.write(String(assistantEvent.delta ?? ''));
+		}
+	});
+
+	pi.on('tool_execution_start', (event) => {
+		const options = getStreamOptions(pi);
+		const toolEvent = getToolEvent(event);
+
+		if (!options.has('tools') || !toolEvent) {
+			return;
+		}
+
+		const toolName =
+			typeof toolEvent.toolName === 'string' ? toolEvent.toolName : 'tool';
+
+		writeLog(`[tool] calling ${toolName}`);
+		writeLog(`[tool] args ${formatToolArgs(toolName, toolEvent.args)}`);
+	});
+
+	pi.on('tool_execution_update', (event) => {
+		const options = getStreamOptions(pi);
+		const toolEvent = getToolEvent(event);
+
+		if (!options.has('tools') || !toolEvent?.partialResult) {
+			return;
+		}
+
+		const toolName =
+			typeof toolEvent.toolName === 'string' ? toolEvent.toolName : 'tool';
+		const text = extractText(toolEvent.partialResult);
+		const size = text
+			? `${text.length} chars`
+			: `${toText(toolEvent.partialResult).length} chars`;
+
+		writeLog(`[tool] running ${toolName} (${size})`);
+	});
+
+	pi.on('tool_execution_end', (event) => {
+		const options = getStreamOptions(pi);
+		const toolEvent = getToolEvent(event);
+
+		if (!options.has('tools') || !toolEvent) {
+			return;
+		}
+
+		const toolName =
+			typeof toolEvent.toolName === 'string' ? toolEvent.toolName : 'tool';
+		const status = toolEvent.isError ? 'error' : 'result';
+
+		writeLog(`[tool] ${status} ${toolName}`);
+		writeLog(formatToolResult(toolEvent.result));
+	});
+}
diff --git a/packages/skills-evals/src/pi.ts b/packages/skills-evals/src/pi.ts
new file mode 100644
index 00000000000..3ba280c91bb
--- /dev/null
+++ b/packages/skills-evals/src/pi.ts
@@ -0,0 +1,203 @@
+import {stat} from 'node:fs/promises';
+import {dirname, join} from 'node:path';
+import {fileURLToPath} from 'node:url';
+import {
+	commandToString,
+	runCommand,
+	type CommandOutput,
+	type CommandResult,
+} from './command';
+import {listFilesRecursively} from './files';
+
+export type PiRun = {
+	command: CommandResult;
+	sessionFile: string;
+};
+
+export type PiExport = {
+	command: CommandResult;
+	htmlExport: string;
+};
+
+type PiAvailability = {
+	message?: string;
+	ok: boolean;
+};
+
+const piSetupMessage =
+	'Pi is not available. Install it from https://pi.dev, then run `pi config` to configure a provider before running skills evals.';
+const evalStreamExtensionPath = join(
+	dirname(fileURLToPath(import.meta.url)),
+	'pi-stream-extension.ts',
+);
+let piAvailabilityPromise: Promise<PiAvailability> | null = null;
+
+export const checkPiAvailable = ({cwd}: {cwd: string}) => {
+	piAvailabilityPromise ??= runCommand({
+		command: ['pi', '--version'],
+		cwd,
+	})
+		.then((result) => {
+			if (result.exitCode === 0) {
+				return {ok: true};
+			}
+
+			return {
+				message: `${piSetupMessage}\n\n\`pi --version\` failed with exit code ${result.exitCode}:\n${result.stderr}`,
+				ok: false,
+			};
+		})
+		.catch((error: unknown) => ({
+			message: `${piSetupMessage}\n\n${error instanceof Error ? error.message : String(error)}`,
+			ok: false,
+		}));
+
+	return piAvailabilityPromise;
+};
+
+const assertPiAvailable = async ({cwd}: {cwd: string}) => {
+	const availability = await checkPiAvailable({cwd});
+
+	if (!availability.ok) {
+		throw new Error(availability.message ?? piSetupMessage);
+	}
+};
+
+const findNewestSessionFile = async (sessionDir: string) => {
+	const sessionFiles = (await listFilesRecursively(sessionDir)).filter((file) =>
+		file.endsWith('.jsonl'),
+	);
+
+	if (sessionFiles.length === 0) {
+		throw new Error(`Pi did not write a session jsonl file to ${sessionDir}.`);
+	}
+
+	const filesWithModifiedTime = await Promise.all(
+		sessionFiles.map(async (file) => ({
+			file,
+			mtimeMs: (await stat(file)).mtimeMs,
+		})),
+	);
+
+	filesWithModifiedTime.sort((a, b) => b.mtimeMs - a.mtimeMs);
+
+	return filesWithModifiedTime[0].file;
+};
+
+export const runPi = async ({
+	onOutput,
+	onPhase,
+	projectRoot,
+	signal,
+	sessionFile,
+	sessionDir,
+	model,
+	prompt,
+	timeoutMs,
+}: {
+	onOutput?: (output: CommandOutput) => void;
+	onPhase?: (status: 'completed' | 'started') => void;
+	projectRoot: string;
+	signal?: AbortSignal;
+	sessionFile?: string;
+	sessionDir: string;
+	model: string;
+	prompt: string;
+	timeoutMs?: number;
+}): Promise<PiRun> => {
+	await assertPiAvailable({cwd: projectRoot});
+
+	const command = [
+		'pi',
+		'--extension',
+		evalStreamExtensionPath,
+		'--eval-stream',
+		'all',
+		'--model',
+		model,
+		'--session-dir',
+		sessionDir,
+		...(sessionFile ? ['--session', sessionFile] : []),
+		'-p',
+		prompt,
+	];
+	onPhase?.('started');
+	const result = await runCommand({
+		command,
+		cwd: projectRoot,
+		onOutput,
+		signal,
+		timeoutMs,
+	});
+
+	if (result.exitCode !== 0) {
+		if (result.timedOut) {
+			throw new Error(`Pi timed out after ${result.durationMs}ms.`);
+		}
+
+		if (result.aborted) {
+			throw new Error('Pi was cancelled.');
+		}
+
+		throw new Error(
+			`Pi failed (${result.exitCode}): ${commandToString(result.command)}\n${result.stderr}`,
+		);
+	}
+
+	onPhase?.('completed');
+
+	return {
+		command: result,
+		sessionFile: await findNewestSessionFile(sessionDir),
+	};
+};
+
+export const exportPiSession = async ({
+	htmlExport,
+	onOutput,
+	onPhase,
+	projectRoot,
+	signal,
+	sessionFile,
+	timeoutMs,
+}: {
+	htmlExport: string;
+	onOutput?: (output: CommandOutput) => void;
+	onPhase?: (status: 'completed' | 'started') => void;
+	projectRoot: string;
+	signal?: AbortSignal;
+	sessionFile: string;
+	timeoutMs?: number;
+}): Promise<PiExport> => {
+	onPhase?.('started');
+	const exportCommand = await runCommand({
+		command: ['pi', '--export', sessionFile, htmlExport],
+		cwd: projectRoot,
+		onOutput,
+		signal,
+		timeoutMs,
+	});
+
+	if (exportCommand.exitCode !== 0) {
+		if (exportCommand.timedOut) {
+			throw new Error(
+				`Pi export timed out after ${exportCommand.durationMs}ms.`,
+			);
+		}
+
+		if (exportCommand.aborted) {
+			throw new Error('Pi export was cancelled.');
+		}
+
+		throw new Error(
+			`Pi export failed (${exportCommand.exitCode}): ${exportCommand.stderr}`,
+		);
+	}
+
+	onPhase?.('completed');
+
+	return {
+		command: exportCommand,
+		htmlExport,
+	};
+};
diff --git a/packages/skills-evals/src/run-skill-eval.ts b/packages/skills-evals/src/run-skill-eval.ts
new file mode 100644
index 00000000000..0d77e5f1596
--- /dev/null
+++ b/packages/skills-evals/src/run-skill-eval.ts
@@ -0,0 +1,418 @@
+import {createHash} from 'node:crypto';
+import {mkdir, readFile, rm, stat, writeFile} from 'node:fs/promises';
+import {dirname, extname, join, relative, resolve} from 'node:path';
+import {fileURLToPath} from 'node:url';
+import type {SkillEvalScenario} from '../scenarios';
+import type {CommandResult} from './command';
+import {runCommand, type CommandOutput} from './command';
+import {
+	copyDirectory,
+	createTimestamp,
+	listFilesRecursively,
+	readJson,
+	sanitizePathPart,
+	writeJson,
+} from './files';
+import type {
+	LoggedCommand,
+	SkillEvalManifest,
+	SkillEvalResult,
+	SkillSnapshot,
+	VisualArtifact,
+} from './manifest';
+import {exportPiSession, runPi} from './pi';
+
+export type SkillEvalPhase = 'install' | 'pi' | 'pi-render' | 'pi-export';
+
+export type SkillEvalOutput = CommandOutput & {
+	phase: SkillEvalPhase;
+};
+
+export type SkillEvalPhaseEvent = {
+	phase: SkillEvalPhase;
+	status: 'completed' | 'started';
+};
+
+type SkillEvalScenarioInput = SkillEvalScenario & {
+	onOutput?: (output: SkillEvalOutput) => void;
+	onPhase?: (event: SkillEvalPhaseEvent) => void;
+	runRoot?: string;
+	runLabel?: 'before' | 'after';
+	signal?: AbortSignal;
+	skillsSourcePath?: string;
+	skillSnapshot?: Partial<
+		Pick<SkillSnapshot, 'comparisonId' | 'gitRef' | 'isWorkingTree' | 'label'>
+	>;
+};
+
+type PackageJson = {
+	name?: string;
+	dependencies?: Record<string, string>;
+	devDependencies?: Record<string, string>;
+	[key: string]: unknown;
+};
+
+const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const repoRoot = resolve(packageRoot, '..', '..');
+const runsRoot = resolve(packageRoot, '.runs');
+const templateBlank = resolve(packageRoot, '..', 'template-blank');
+const skillsSource = resolve(packageRoot, '..', 'skills', 'skills');
+const remotionPackagesInBlankTemplate = [
+	'@remotion/cli',
+	'@remotion/eslint-config-flat',
+	'remotion',
+];
+const visualArtifactExtensions = new Set([
+	'.gif',
+	'.jpeg',
+	'.jpg',
+	'.mov',
+	'.mp4',
+	'.png',
+	'.webm',
+]);
+const visualArtifactTypePriority: Record<VisualArtifact['type'], number> = {
+	video: 0,
+	image: 1,
+};
+const createRenderPrompt = (expectedArtifactPath: string) =>
+	`Generate and render the final video artifact that can be reviewed.
+
+Render the final MP4 to this exact path:
+${expectedArtifactPath}
+
+Do not leave the final artifact only in /tmp or another temporary directory. Temporary files are fine, but the reviewed MP4 must exist at the path above. Do not stop after producing only a still image or screenshot.`;
+
+const hashDirectory = async (dir: string) => {
+	const hash = createHash('sha256');
+	const files = await listFilesRecursively(dir);
+
+	for (const file of files) {
+		const stats = await stat(file);
+
+		if (!stats.isFile()) {
+			continue;
+		}
+
+		hash.update(relative(dir, file));
+		hash.update('\0');
+		hash.update((await readFile(file)).toString('base64'));
+		hash.update('\0');
+	}
+
+	return hash.digest('hex').slice(0, 12);
+};
+
+let publishedRemotionVersionPromise: Promise<string> | null = null;
+
+const getPublishedRemotionVersion = () => {
+	publishedRemotionVersionPromise ??= (async () => {
+		const override = process.env.REMOTION_SKILLS_EVALS_REMOTION_VERSION;
+
+		if (override) {
+			return override;
+		}
+
+		const result = await runCommand({
+			command: ['bun', 'pm', 'view', 'remotion', 'version'],
+			cwd: repoRoot,
+			timeoutMs: 30_000,
+		});
+
+		if (result.exitCode !== 0) {
+			throw new Error(
+				`Could not resolve the latest published Remotion version: ${result.stderr}`,
+			);
+		}
+
+		return result.stdout.trim();
+	})();
+
+	return publishedRemotionVersionPromise;
+};
+
+const patchDependencyMap = (
+	dependencies: Record<string, string> | undefined,
+	version: string,
+) => {
+	if (!dependencies) {
+		return dependencies;
+	}
+
+	return Object.fromEntries(
+		Object.entries(dependencies).map(([name, dependencyVersion]) => [
+			name,
+			remotionPackagesInBlankTemplate.includes(name)
+				? version
+				: dependencyVersion,
+		]),
+	);
+};
+
+const copyBlankTemplate = async ({
+	projectRoot,
+	projectName,
+}: {
+	projectRoot: string;
+	projectName: string;
+}) => {
+	await copyDirectory(templateBlank, projectRoot);
+
+	const packageJsonFile = join(projectRoot, 'package.json');
+	const packageJson = await readJson<PackageJson>(packageJsonFile);
+	const version = await getPublishedRemotionVersion();
+
+	await writeJson(packageJsonFile, {
+		...packageJson,
+		name: projectName,
+		dependencies: patchDependencyMap(packageJson.dependencies, version),
+		devDependencies: patchDependencyMap(packageJson.devDependencies, version),
+	});
+
+	await writeFile(
+		join(projectRoot, 'README.md'),
+		`# ${projectName}\n\nGenerated by @remotion/skills-evals.\n`,
+	);
+};
+
+const copySkillsForPiDiscovery = async ({
+	projectRoot,
+	skillSnapshot,
+	skillsSourcePath,
+}: {
+	projectRoot: string;
+	skillSnapshot?: SkillEvalScenarioInput['skillSnapshot'];
+	skillsSourcePath?: string;
+}): Promise<SkillSnapshot> => {
+	const projectSkillsRoot = join(projectRoot, '.pi', 'skills');
+	const sandboxPath = join(projectSkillsRoot, 'remotion');
+	const sourcePath = skillsSourcePath ?? skillsSource;
+
+	await mkdir(projectSkillsRoot, {recursive: true});
+	await copyDirectory(join(sourcePath, 'remotion'), sandboxPath);
+
+	return {
+		...skillSnapshot,
+		sourcePath,
+		sandboxPath,
+		hash: await hashDirectory(sandboxPath),
+	};
+};
+
+const discoverVisualArtifacts = async (
+	projectRoot: string,
+): Promise<VisualArtifact[]> => {
+	const files = await listFilesRecursively(projectRoot);
+	const artifacts = await Promise.all(
+		files
+			.filter((file) =>
+				visualArtifactExtensions.has(extname(file).toLowerCase()),
+			)
+			.map(async (file) => {
+				const stats = await stat(file);
+				const extension = extname(file).toLowerCase();
+
+				return {
+					path: file,
+					relativePath: relative(projectRoot, file),
+					sizeInBytes: stats.size,
+					type:
+						extension === '.png' ||
+						extension === '.jpg' ||
+						extension === '.jpeg'
+							? 'image'
+							: 'video',
+				} satisfies VisualArtifact;
+			}),
+	);
+
+	return artifacts.sort(
+		(a, b) =>
+			visualArtifactTypePriority[a.type] - visualArtifactTypePriority[b.type] ||
+			a.relativePath.localeCompare(b.relativePath),
+	);
+};
+
+const logCommand = async ({
+	command,
+	logDir,
+	name,
+}: {
+	command: CommandResult;
+	logDir: string;
+	name: string;
+}): Promise<LoggedCommand> => {
+	const stdoutPath = join(logDir, `${name}.stdout.log`);
+	const stderrPath = join(logDir, `${name}.stderr.log`);
+
+	await writeFile(stdoutPath, command.stdout);
+	await writeFile(stderrPath, command.stderr);
+
+	return {
+		command: command.command,
+		cwd: command.cwd,
+		durationMs: command.durationMs,
+		exitCode: command.exitCode,
+		stdoutPath,
+		stderrPath,
+	};
+};
+
+const describeCommandFailure = (name: string, command: CommandResult) => {
+	if (command.timedOut) {
+		return `${name} timed out after ${command.durationMs}ms.`;
+	}
+
+	if (command.aborted) {
+		return `${name} was cancelled.`;
+	}
+
+	return `${name} failed (${command.exitCode}): ${command.stderr}`;
+};
+
+const runPhase = async ({
+	command,
+	cwd,
+	input,
+	phase,
+	timeoutMs,
+}: {
+	command: string[];
+	cwd: string;
+	input: SkillEvalScenarioInput;
+	phase: SkillEvalPhase;
+	timeoutMs?: number;
+}) => {
+	input.onPhase?.({phase, status: 'started'});
+
+	const result = await runCommand({
+		command,
+		cwd,
+		onOutput: (output) => input.onOutput?.({...output, phase}),
+		signal: input.signal,
+		timeoutMs,
+	});
+
+	input.onPhase?.({phase, status: 'completed'});
+
+	return result;
+};
+
+export const runSkillEval = async (
+	input: SkillEvalScenarioInput,
+): Promise<SkillEvalResult> => {
+	const runRoot = input.runRoot ?? runsRoot;
+	const runLabel = input.runLabel ? `${input.runLabel}--` : '';
+	const runDir = join(
+		runRoot,
+		sanitizePathPart(input.id),
+		`${createTimestamp()}--${runLabel}${sanitizePathPart(input.model)}`,
+	);
+	const projectRoot = join(runDir, 'project');
+	const expectedArtifactPath = join(projectRoot, 'out', 'skills-eval.mp4');
+	const sessionDir = join(runDir, 'pi-session');
+	const logDir = join(runDir, 'logs');
+	const createdAt = new Date().toISOString();
+
+	await rm(runDir, {force: true, recursive: true});
+	await mkdir(runDir, {recursive: true});
+	await mkdir(logDir, {recursive: true});
+	await mkdir(dirname(expectedArtifactPath), {recursive: true});
+	await copyBlankTemplate({
+		projectName: `skills-eval-${sanitizePathPart(input.id)}`,
+		projectRoot,
+	});
+	const skillSnapshot = await copySkillsForPiDiscovery({
+		projectRoot,
+		skillSnapshot: input.skillSnapshot,
+		skillsSourcePath: input.skillsSourcePath,
+	});
+	const install = await runPhase({
+		command: ['bun', 'install'],
+		cwd: projectRoot,
+		input,
+		phase: 'install',
+	});
+
+	if (install.exitCode !== 0) {
+		throw new Error(describeCommandFailure('bun install', install));
+	}
+
+	const pi = await runPi({
+		model: input.model,
+		onOutput: (output) => input.onOutput?.({...output, phase: 'pi'}),
+		onPhase: (status) => input.onPhase?.({phase: 'pi', status}),
+		projectRoot,
+		prompt: input.prompt,
+		signal: input.signal,
+		sessionDir,
+		timeoutMs: input.timeoutMs,
+	});
+	const piRender = await runPi({
+		model: input.model,
+		onOutput: (output) => input.onOutput?.({...output, phase: 'pi-render'}),
+		onPhase: (status) => input.onPhase?.({phase: 'pi-render', status}),
+		projectRoot,
+		prompt: createRenderPrompt(expectedArtifactPath),
+		signal: input.signal,
+		sessionDir,
+		sessionFile: pi.sessionFile,
+		timeoutMs: input.timeoutMs,
+	});
+	const piExport = await exportPiSession({
+		htmlExport: join(sessionDir, 'session.html'),
+		onOutput: (output) => input.onOutput?.({...output, phase: 'pi-export'}),
+		onPhase: (status) => input.onPhase?.({phase: 'pi-export', status}),
+		projectRoot,
+		signal: input.signal,
+		sessionFile: piRender.sessionFile,
+		timeoutMs: input.timeoutMs,
+	});
+	const artifacts = await discoverVisualArtifacts(projectRoot);
+
+	if (artifacts.length === 0) {
+		throw new Error(
+			`No visual artifacts were found in ${relative(repoRoot, projectRoot)}.`,
+		);
+	}
+
+	const completedAt = new Date().toISOString();
+	const manifest = {
+		id: input.id,
+		model: input.model,
+		prompt: input.prompt,
+		createdAt,
+		completedAt,
+		runDir,
+		projectRoot,
+		skillSnapshot,
+		pi: {
+			sessionFile: piRender.sessionFile,
+			htmlExport: piExport.htmlExport,
+		},
+		artifacts,
+		commands: {
+			install: await logCommand({command: install, logDir, name: 'install'}),
+			pi: await logCommand({command: pi.command, logDir, name: 'pi'}),
+			piRender: await logCommand({
+				command: piRender.command,
+				logDir,
+				name: 'pi-render',
+			}),
+			piExport: await logCommand({
+				command: piExport.command,
+				logDir,
+				name: 'pi-export',
+			}),
+		},
+	} satisfies SkillEvalManifest;
+	const manifestPath = join(runDir, 'manifest.json');
+
+	await writeJson(manifestPath, manifest);
+
+	return {
+		manifest,
+		manifestPath,
+		scenario: input,
+	};
+};
diff --git a/packages/skills-evals/src/server.tsx b/packages/skills-evals/src/server.tsx
new file mode 100644
index 00000000000..b66ac1f75bb
--- /dev/null
+++ b/packages/skills-evals/src/server.tsx
@@ -0,0 +1,58 @@
+import {routes} from './app/routes';
+import {notFound, origin, port, repoRoot} from './app/shared';
+import {runCommand} from './command';
+import {checkPiAvailable} from './pi';
+
+const server = Bun.serve({
+	error: (error) => {
+		process.stderr.write(
+			`Skills eval server error: ${
+				error instanceof Error ? error.stack : String(error)
+			}\n`,
+		);
+
+		return new Response('Internal Server Error', {status: 500});
+	},
+	fetch: () => notFound(),
+	hostname: '127.0.0.1',
+	port,
+	routes,
+});
+
+process.stdout.write(`Skills eval server running at ${origin}\n`);
+
+checkPiAvailable({cwd: repoRoot})
+	.then((availability) => {
+		if (!availability.ok) {
+			process.stderr.write(`${availability.message}\n`);
+		}
+	})
+	.catch((error: unknown) => {
+		process.stderr.write(
+			`Could not check Pi availability: ${
+				error instanceof Error ? error.message : String(error)
+			}\n`,
+		);
+	});
+
+if (process.platform === 'darwin' && process.stdout.isTTY && !process.env.CI) {
+	const openBrowser = async () => {
+		await runCommand({
+			command: ['open', origin],
+			cwd: repoRoot,
+		});
+	};
+
+	openBrowser().catch((error: unknown) => {
+		process.stderr.write(
+			`Could not open browser: ${
+				error instanceof Error ? error.message : String(error)
+			}\n`,
+		);
+	});
+}
+
+process.on('SIGINT', () => {
+	server.stop();
+	process.exit(0);
+});
diff --git a/packages/skills-evals/tsconfig.json b/packages/skills-evals/tsconfig.json
new file mode 100644
index 00000000000..06d5bd16cb0
--- /dev/null
+++ b/packages/skills-evals/tsconfig.json
@@ -0,0 +1,13 @@
+{
+	"extends": "../tsconfig.settings.json",
+	"compilerOptions": {
+		"rootDir": ".",
+		"outDir": "dist",
+		"noEmit": true,
+		"types": ["bun"],
+		"target": "ES2022",
+		"module": "es2020",
+		"moduleResolution": "bundler"
+	},
+	"include": ["src", "scenarios.ts"]
+}
diff --git a/packages/studio-shared/src/package-info.ts b/packages/studio-shared/src/package-info.ts
index 0c72b808ffa..ecd5c8d0629 100644
--- a/packages/studio-shared/src/package-info.ts
+++ b/packages/studio-shared/src/package-info.ts
@@ -64,6 +64,7 @@ export const packages = [
 	'serverless',
 	'serverless-client',
 	'skills',
+	'skills-evals',
 	'studio-server',
 	'studio-shared',
 	'studio',
@@ -200,6 +201,7 @@ export const descriptions: {[key in Pkgs]: string | null} = {
 	'lambda-ruby': null,
 	'player-example': null,
 	skills: null,
+	'skills-evals': null,
 	'discord-poster': null,
 	'docusaurus-plugin': null,
 	'animated-emoji': 'Google Fonts Animated Emojis as Remotion components',
@@ -289,6 +291,7 @@ export const installableMap: {[key in Pkgs]: boolean} = {
 	shapes: true,
 	skia: true,
 	skills: false,
+	'skills-evals': false,
 	'promo-pages': false,
 	streaming: false,
 	serverless: false,
@@ -396,6 +399,7 @@ export const apiDocs: {[key in Pkgs]: string | null} = {
 	streaming: null,
 	serverless: null,
 	skills: null,
+	'skills-evals': null,
 	studio: 'https://www.remotion.dev/docs/studio/api',
 	'timeline-utils': null,
 	transitions: 'https://www.remotion.dev/transitions',