chore: refresh eval results

README.md

@@ -143,10 +143,10 @@ Skills come from [`supabase/agent-skills`](https://github.com/supabase/agent-ski
 
 To use a skill in an experiment, reference its directory name in the experiment's `skills` array.
 
-Both runtimes load skills lazily ([progressive disclosure](https://ai-sdk.dev/cookbook/guides/agent-skills)): only each skill's name+description is in the system prompt, and the agent pulls a skill's full instructions on demand. They differ only in how the body is fetched, because the tools-mode agent has no filesystem:
+Skills are always loaded lazily ([progressive disclosure](https://ai-sdk.dev/cookbook/guides/agent-skills)) — a skill's full instructions are pulled on demand, never preloaded. How that happens depends on the harness:
 
-- **Local-stack (sandbox) mode:** skills are installed into the workspace with [Vercel's `skills` CLI](https://github.com/vercel-labs/skills) (baked into the sandbox image, sourced from the local `skills/` directory — never the network), into every project scope the CLI harnesses discover natively: `.claude/skills/` for Claude Code, `.agents/skills/` for Codex and OpenCode. Each CLI discovers the scope it reads and surfaces those skills to the model itself. The framework also still injects a listing naming them, so a CLI harness hears about them twice; removing that is a separate change.
-- **Tools mode:** no filesystem, so a `load_skill` tool returns a skill's full instructions when the agent calls it with the skill's name.
+- **CLI harnesses (Claude Code, Codex, OpenCode)** use their own built-in skills mechanism. Skills are installed into the sandbox workspace with [Vercel's `skills` CLI](https://github.com/vercel-labs/skills) (baked into the sandbox image, sourced from the local `skills/` directory — never the network), for each harness's own project scope: `.claude/skills/` for Claude Code, `.agents/skills/` for Codex and OpenCode. Each CLI then discovers, advertises and loads the skills itself. The framework injects nothing — an agent's real-world skill-following behaviour is part of what an eval measures.
+- **The in-process `ai-sdk` harness** has no such mechanism, so the framework supplies one. In local-stack mode it lists each skill's name+description in the system prompt and the agent reads `.claude/skills/<name>/SKILL.md` with its file tools. In tools mode there is no filesystem at all, so a `load_skill` tool returns a skill's full instructions when the agent calls it with the skill's name.
 
 ## Framework Checks
 

apps/framework/harness/run-eval.ts

@@ -29,6 +29,7 @@ import {
 } from '../lib/cli-args.js';
 import { bootPlatformBackend } from './platform-backend.js';
 import { viteBuild, vitestRun } from './project-runner.js';
+import { buildSystemPrompt } from './system-prompt.js';
 import {
   buildDocsResult,
   buildSkillResult,
@@ -252,9 +253,9 @@ function buildLoadSkillTool(skills: readonly ToolsSkill[]): ToolSet {
 }
 
 /**
- * Local-stack skill sources: resolve each skill name to its host directory so
- * the sandbox can install it with Vercel's `skills` CLI; the agent then
- * discovers each skill by reading its SKILL.md with its file tools. The
+ * Sandbox skill sources: resolve each skill name to its host directory so the
+ * sandbox can install it with Vercel's `skills` CLI, which places it in every
+ * CLI harness's native project scope for that harness to discover. The
  * `skills/` entries are symlinks into the agent-skills submodule; realpath them
  * so `docker cp` copies real files, not dangling links. Missing skills are
  * skipped with a warning.
@@ -313,31 +314,6 @@ function readSessionSeedArgs(ev: EvalManifest) {
   };
 }
 
-function basePromptFor(mode: EvalMode): string {
-  if (mode === 'local-stack') {
-    return (
-      'You are an agent solving a Supabase eval task in a Linux workspace. ' +
-      'Use the provided tools to inspect and modify the workspace and run commands. ' +
-      'When you are done, end your turn with a short summary of what you did.'
-    );
-  }
-  return (
-    'You are an agent solving a Supabase eval task. ' +
-    'Use the provided tools to inspect and modify the project. ' +
-    'When you are done, end your turn with a short summary of what you did ' +
-    '(or for audit tasks, your findings).'
-  );
-}
-
-function buildSystemPrompt(
-  mode: EvalMode,
-  addendum?: string,
-  skillContext?: string
-): string {
-  const blocks = [basePromptFor(mode), addendum, skillContext].filter(Boolean);
-  return blocks.join('\n\n');
-}
-
 /**
  * Adapt a `{ close() }` resource to `AsyncDisposable` so it can be bound with
  * `await using` — cleanup then runs on scope exit (normal fall-through, `continue`,
@@ -367,6 +343,13 @@ async function runOne(
     transcript: TranscriptPart[];
     agentReport: string;
     stoppedReason: string;
+    /**
+     * The exact system prompt handed to the agent (`''` when it got none). CLI
+     * harnesses receive theirs as a file in the sandbox scratch dir, outside the
+     * exported workspace, so recording it here is the only way to verify from a
+     * run artifact what the agent was actually told.
+     */
+    systemPrompt: string;
   }
 > {
   const prompt = parseEvalMarkdown(
@@ -398,6 +381,7 @@ async function runOne(
   let lastTranscript: TranscriptPart[] = [];
   let lastAgentReport = '';
   let lastStoppedReason = 'not_started';
+  let lastSystemPrompt = '';
 
   for (let attempt = 1; attempt <= RUNS; attempt += 1) {
     if (ev.mode === 'local-stack') {
@@ -452,8 +436,13 @@ async function runOne(
         })
       );
 
+      const systemPrompt = buildSystemPrompt(
+        exp.agent.id,
+        'local-stack',
+        session.promptAddendum
+      );
       const run = await exp.agent.run({
-        systemPrompt: buildSystemPrompt('local-stack', session.promptAddendum),
+        systemPrompt,
         userPrompt: prompt,
         tools: session.tools,
         sandbox: session.sandbox,
@@ -464,6 +453,7 @@ async function runOne(
       lastTranscript = run.transcript;
       lastAgentReport = run.agentReport;
       lastStoppedReason = run.stoppedReason;
+      lastSystemPrompt = systemPrompt;
 
       // Export the agent's workspace to the host so scorers can run host
       // tooling (vite/vitest from the repo root) against the produced files
@@ -505,6 +495,7 @@ async function runOne(
           transcript: run.transcript,
           agentReport: run.agentReport,
           stoppedReason: run.stoppedReason,
+          systemPrompt,
         };
       }
       logRetryAttempt(expName, ev, attempt, last);
@@ -519,7 +510,6 @@ async function runOne(
     await using cliSandbox = agentRunsInSandbox
       ? disposable(
           await createBareSandbox({
-            agent: exp.agent.id,
             skills: skillSources,
             mounts: supabaseMcpServerMounts(),
           })
@@ -532,14 +522,16 @@ async function runOne(
       })
     );
 
-    // CLI agents read their installed skills from disk (the bare sandbox folds
-    // the discovery listing into its promptAddendum). In-process agents have
-    // no filesystem, so their skills are advertised in the prompt and pulled
-    // on demand via the load_skill tool.
+    // CLI agents discover their installed skills themselves — the skills CLI
+    // put them in every harness's native project scope, so each one advertises
+    // and loads them in its own words and the bare sandbox contributes nothing
+    // here. In-process agents have no filesystem, so their skills are advertised
+    // in the prompt and pulled on demand via the load_skill tool.
     const skillsPrompt = agentRunsInSandbox
-      ? cliSandbox!.promptAddendum
+      ? undefined
       : buildToolsSkillsPrompt(toolsSkills);
     const systemPrompt = buildSystemPrompt(
+      exp.agent.id,
       'tools',
       session.promptAddendum,
       skillsPrompt
@@ -556,6 +548,7 @@ async function runOne(
     lastTranscript = run.transcript;
     lastAgentReport = run.agentReport;
     lastStoppedReason = run.stoppedReason;
+    lastSystemPrompt = systemPrompt;
     last = await (scorer as ToolScorer)({
       ...session.scoringContext,
       toolCalls: run.toolCalls,
@@ -577,6 +570,7 @@ async function runOne(
         transcript: run.transcript,
         agentReport: run.agentReport,
         stoppedReason: run.stoppedReason,
+        systemPrompt,
       };
     }
     logRetryAttempt(expName, ev, attempt, last);
@@ -591,6 +585,7 @@ async function runOne(
     transcript: lastTranscript,
     agentReport: lastAgentReport,
     stoppedReason: lastStoppedReason,
+    systemPrompt: lastSystemPrompt,
   };
 }
 
@@ -847,9 +842,16 @@ async function main() {
   }
 }
 
-main()
-  .then(() => process.exit(0))
-  .catch((e) => {
-    console.error(e);
-    process.exit(1);
-  });
+// Only when this file is the entry point. Importing it (a unit test reaching for
+// one of its helpers) must not dispatch a run or call process.exit.
+if (
+  process.argv[1] &&
+  import.meta.url === pathToFileURL(process.argv[1]).href
+) {
+  main()
+    .then(() => process.exit(0))
+    .catch((e) => {
+      console.error(e);
+      process.exit(1);
+    });
+}

apps/framework/harness/system-prompt.test.ts

@@ -0,0 +1,111 @@
+import { describe, expect, it } from 'vitest';
+import type { AgentHarnessId } from '@supabase-evals/core';
+import {
+  buildSkillsPrompt,
+  buildToolSurfaceAddendum,
+  type SkillEntry,
+} from '@supabase-evals/sandbox';
+import { buildSystemPrompt } from './system-prompt.js';
+import type { EvalMode } from './types.js';
+
+const CLI_AGENTS: AgentHarnessId[] = ['claude-code', 'codex', 'opencode'];
+const MODES: EvalMode[] = ['tools', 'local-stack'];
+
+describe('buildSystemPrompt', () => {
+  it('gives the ai-sdk agent task framing in both modes', () => {
+    // ai-sdk is the one harness with no system prompt of its own, so it's the
+    // one harness the framework has to supply one for.
+    for (const mode of MODES) {
+      expect(buildSystemPrompt('ai-sdk', mode)).toContain(
+        'Use the provided tools'
+      );
+    }
+  });
+
+  it('gives no framing of our own to any CLI harness', () => {
+    // CLI harnesses ship their own system prompt; we're measuring that.
+    for (const agent of CLI_AGENTS) {
+      for (const mode of MODES) {
+        expect(buildSystemPrompt(agent, mode)).toBe('');
+      }
+    }
+  });
+
+  it('refuses blocks a caller hands it for a CLI harness', () => {
+    // The producers gate their own output, so a non-empty block here means an
+    // experiment is misconfigured. Dropping it would be as silent as injecting
+    // it, and the block may be load-bearing: `executorMcpServer`'s addendum is
+    // the pause/resume protocol its tools require.
+    for (const agent of CLI_AGENTS) {
+      for (const mode of MODES) {
+        expect(() => buildSystemPrompt(agent, mode, 'Addendum.')).toThrow(
+          /must receive no system prompt/
+        );
+        expect(() =>
+          buildSystemPrompt(agent, mode, undefined, 'Skills listing.')
+        ).toThrow(/must receive no system prompt/);
+        // Empty and absent blocks are the normal case, not a misconfiguration.
+        expect(buildSystemPrompt(agent, mode, '', '')).toBe('');
+      }
+    }
+  });
+
+  it('keeps the runtime blocks for ai-sdk, in order, after the base prompt', () => {
+    const base = buildSystemPrompt('ai-sdk', 'local-stack');
+    expect(
+      buildSystemPrompt('ai-sdk', 'local-stack', 'Addendum.', 'Skills listing.')
+    ).toBe(`${base}\n\nAddendum.\n\nSkills listing.`);
+  });
+
+  it('assembles to nothing at all for a CLI harness, even with skills', () => {
+    // The real block producers, not stand-ins: with skills installed, a CLI
+    // harness must still receive an entirely empty system prompt. Codex and
+    // OpenCode find the skills through their own project-scope discovery and
+    // describe them to the model themselves.
+    const skills: SkillEntry[] = [
+      {
+        name: 'supabase',
+        description: 'Use for Supabase tasks.',
+        dir: '.claude/skills/supabase',
+      },
+    ];
+    for (const agent of CLI_AGENTS) {
+      expect(
+        buildSystemPrompt(
+          agent,
+          'local-stack',
+          buildToolSurfaceAddendum(agent),
+          buildSkillsPrompt(agent, skills)
+        )
+      ).toBe('');
+    }
+    // ai-sdk has no such mechanism — it only learns about skills from us.
+    const aiSdk = buildSystemPrompt(
+      'ai-sdk',
+      'local-stack',
+      buildToolSurfaceAddendum('ai-sdk'),
+      buildSkillsPrompt('ai-sdk', skills)
+    );
+    expect(aiSdk).toContain('## Available skills');
+    expect(aiSdk).toContain('- supabase: Use for Supabase tasks.');
+  });
+
+  it('never tells any agent how to end its turn', () => {
+    // Stopping behaviour is part of what an eval measures, so the harness must
+    // not coach it (e.g. "end your turn with a short summary").
+    for (const agent of [...CLI_AGENTS, 'ai-sdk' as const]) {
+      for (const mode of MODES) {
+        const prompt = buildSystemPrompt(agent, mode);
+        expect(prompt).not.toMatch(/summary/i);
+        expect(prompt).not.toMatch(/end your turn/i);
+      }
+    }
+  });
+
+  it('drops empty blocks instead of leaving blank gaps', () => {
+    expect(buildSystemPrompt('ai-sdk', 'tools', '', 'Skills listing.')).toBe(
+      `${buildSystemPrompt('ai-sdk', 'tools')}\n\nSkills listing.`
+    );
+    expect(buildSystemPrompt('ai-sdk', 'tools', '', '')).not.toMatch(/\n\n$/);
+  });
+});

apps/framework/harness/system-prompt.ts

@@ -0,0 +1,76 @@
+/**
+ * System-prompt assembly, per agent harness.
+ *
+ * An eval measures out-of-the-box agent behaviour, so the harness injects as
+ * little prompt of its own as it can get away with: only the ai-sdk agent gets
+ * any base framing, because it is the only harness with no system prompt of its
+ * own (`aiSdkAgent` hands `systemPrompt` straight to the model's `system`). CLI
+ * agents ship their own coding-agent prompt, tool guidance, and stopping
+ * behaviour — and codex/opencode have no system-prompt flag at all, so anything
+ * we pass them lands on the *user* prompt.
+ */
+
+import type { AgentHarnessId } from '@supabase-evals/core';
+import type { EvalMode } from './types.js';
+
+/**
+ * Base framing for the ai-sdk harness: what it can't infer on its own — that it
+ * has tools, and what they act on. Deliberately silent on how to finish a turn
+ * (no "end with a summary"): stopping behaviour is part of what's measured.
+ * Empty for every CLI harness.
+ */
+function basePromptFor(agent: AgentHarnessId, mode: EvalMode): string {
+  if (agent !== 'ai-sdk') return '';
+  if (mode === 'local-stack') {
+    return (
+      'You are an agent solving a Supabase eval task in a Linux workspace. ' +
+      'Use the provided tools to inspect and modify the workspace and run commands.'
+    );
+  }
+  return (
+    'You are an agent solving a Supabase eval task. ' +
+    'Use the provided tools to inspect and modify the project.'
+  );
+}
+
+/**
+ * Assemble the system prompt handed to the agent. Every block is ai-sdk-only —
+ * the base framing, the tool-surface addendum, the skills listing — so a CLI
+ * harness ends up with `''`, and the CLI engine then stages no system-prompt
+ * file at all rather than an empty one.
+ *
+ * The callers' blocks are already gated by their producers, so a non-empty one
+ * arriving here means an experiment is misconfigured. Throw rather than drop it:
+ * dropping is as silent as injecting, and the block may be load-bearing. An MCP
+ * server carrying a `promptAddendum` is the live path in. Only
+ * `executorMcpServer` has one, and its text is the pause/resume protocol its
+ * tools require, not a tool description. A CLI harness paired with it would get
+ * the tools and none of the protocol, then stall on the first paused execution
+ * with a recorded prompt of `''` explaining nothing.
+ */
+export function buildSystemPrompt(
+  agent: AgentHarnessId,
+  mode: EvalMode,
+  addendum?: string,
+  skillContext?: string
+): string {
+  if (agent !== 'ai-sdk') {
+    for (const [arg, block] of [
+      ['addendum', addendum],
+      ['skillContext', skillContext],
+    ] as const) {
+      if (block) {
+        throw new Error(
+          `buildSystemPrompt got a non-empty ${arg} for '${agent}', which must receive ` +
+            'no system prompt. Whichever runtime or MCP server produced it is not gated ' +
+            'on the agent harness.'
+        );
+      }
+    }
+    return '';
+  }
+  const blocks = [basePromptFor(agent, mode), addendum, skillContext].filter(
+    Boolean
+  );
+  return blocks.join('\n\n');
+}