fix(framework): remove the harness system prompt for CLI agents (AI-1034)

README.md

@@ -161,10 +161,7 @@ Skills come from [`supabase/agent-skills`](https://github.com/supabase/agent-ski
 
 To use a skill in an experiment, reference its directory name in the experiment's `skills` array.
 
-Both runtimes load skills lazily ([progressive disclosure](https://ai-sdk.dev/cookbook/guides/agent-skills)): only each skill's name+description is in the system prompt, and the agent pulls a skill's full instructions on demand. They differ only in how the body is fetched, because the tools-mode agent has no filesystem:
-
-- **Local-stack (sandbox) mode:** skills are installed into the workspace with [Vercel's `skills` CLI](https://github.com/vercel-labs/skills) (baked into the sandbox image, sourced from the local `skills/` directory — never the network), into every project scope the CLI harnesses discover natively: `.claude/skills/` for Claude Code, `.agents/skills/` for Codex and OpenCode. Each CLI discovers the scope it reads and surfaces those skills to the model itself. The framework also still injects a listing naming them, so a CLI harness hears about them twice; removing that is a separate change.
-- **Tools mode:** no filesystem, so a `load_skill` tool returns a skill's full instructions when the agent calls it with the skill's name.
+Skills are installed into the sandbox workspace with [Vercel's `skills` CLI](https://github.com/vercel-labs/skills) under each harness's native skills folder (`.claude/skills/` for Claude Code, `.agents/skills/` for Codex and OpenCode). The `ai-sdk` harness exposes a `load_skill` tool to support skill loading.
 
 ## Framework Checks
 

apps/framework/harness/run-eval.ts

@@ -29,6 +29,7 @@ import {
 } from '../lib/cli-args.js';
 import { bootPlatformBackend } from './platform-backend.js';
 import { viteBuild, vitestRun } from './project-runner.js';
+import { buildSystemPrompt } from './system-prompt.js';
 import {
   buildDocsResult,
   buildSkillResult,
@@ -319,31 +320,6 @@ function readSessionSeedArgs(ev: EvalManifest) {
   };
 }
 
-function basePromptFor(mode: EvalMode): string {
-  if (mode === 'local-stack') {
-    return (
-      'You are an agent solving a Supabase eval task in a Linux workspace. ' +
-      'Use the provided tools to inspect and modify the workspace and run commands. ' +
-      'When you are done, end your turn with a short summary of what you did.'
-    );
-  }
-  return (
-    'You are an agent solving a Supabase eval task. ' +
-    'Use the provided tools to inspect and modify the project. ' +
-    'When you are done, end your turn with a short summary of what you did ' +
-    '(or for audit tasks, your findings).'
-  );
-}
-
-function buildSystemPrompt(
-  mode: EvalMode,
-  addendum?: string,
-  skillContext?: string
-): string {
-  const blocks = [basePromptFor(mode), addendum, skillContext].filter(Boolean);
-  return blocks.join('\n\n');
-}
-
 /**
  * Adapt a `{ close() }` resource to `AsyncDisposable` so it can be bound with
  * `await using` — cleanup then runs on scope exit (normal fall-through, `continue`,
@@ -374,6 +350,11 @@ async function runOne(
     transcript: TranscriptPart[];
     agentReport: string;
     stoppedReason: string;
+    /**
+     * The system prompt the harness handed the agent, `''` for a CLI agent.
+     * Recorded so a run artifact shows what the agent was told.
+     */
+    systemPrompt: string;
     usage?: AgentUsage;
     stepCount?: number;
     toolCallCount: number;
@@ -447,8 +428,12 @@ async function runOne(
       })
     );
 
+    const systemPrompt = buildSystemPrompt({
+      agent: exp.agent.id,
+      addendum: session.promptAddendum,
+    });
     const run = await exp.agent.run({
-      systemPrompt: buildSystemPrompt('local-stack', session.promptAddendum),
+      systemPrompt,
       userPrompt: prompt,
       tools: session.tools,
       sandbox: session.sandbox,
@@ -492,6 +477,7 @@ async function runOne(
       transcript: run.transcript,
       agentReport: run.agentReport,
       stoppedReason: run.stoppedReason,
+      systemPrompt,
       usage: run.usage,
       stepCount: run.stepCount,
       toolCallCount: run.toolCalls.length,
@@ -504,7 +490,6 @@ async function runOne(
   await using cliSandbox = agentRunsInSandbox
     ? disposable(
         await createBareSandbox({
-          agent: exp.agent.id,
           skills: skillSources,
           mounts: supabaseMcpServerMounts(),
         })
@@ -517,16 +502,16 @@ async function runOne(
     })
   );
 
-  // In-process agents have no filesystem, so skills are advertised in the
-  // prompt and pulled on demand via the load_skill tool instead.
-  const skillsPrompt = agentRunsInSandbox
-    ? cliSandbox!.promptAddendum
-    : buildToolsSkillsPrompt(toolsSkills);
-  const systemPrompt = buildSystemPrompt(
-    'tools',
-    session.promptAddendum,
-    skillsPrompt
-  );
+  // A CLI agent discovers its installed skills itself. An in-process agent has
+  // no filesystem, so its skills are advertised in the prompt and pulled on
+  // demand via the load_skill tool instead.
+  const systemPrompt = buildSystemPrompt({
+    agent: exp.agent.id,
+    addendum: session.promptAddendum,
+    skillContext: agentRunsInSandbox
+      ? undefined
+      : buildToolsSkillsPrompt(toolsSkills),
+  });
   const run = await exp.agent.run({
     systemPrompt,
     userPrompt: prompt,
@@ -555,6 +540,7 @@ async function runOne(
     transcript: run.transcript,
     agentReport: run.agentReport,
     stoppedReason: run.stoppedReason,
+    systemPrompt,
     usage: run.usage,
     stepCount: run.stepCount,
     toolCallCount: run.toolCalls.length,
@@ -805,9 +791,16 @@ async function main() {
   }
 }
 
-main()
-  .then(() => process.exit(0))
-  .catch((e) => {
-    console.error(e);
-    process.exit(1);
-  });
+// Only when this file is the entry point. Importing it (a unit test reaching
+// for one of its helpers) must not dispatch a run or call process.exit.
+if (
+  process.argv[1] &&
+  import.meta.url === pathToFileURL(process.argv[1]).href
+) {
+  main()
+    .then(() => process.exit(0))
+    .catch((e) => {
+      console.error(e);
+      process.exit(1);
+    });
+}

apps/framework/harness/system-prompt.test.ts

@@ -0,0 +1,91 @@
+import { describe, expect, it } from 'vitest';
+import type { AgentHarnessId } from '@supabase-evals/core';
+import {
+  buildSkillsPrompt,
+  buildToolSurfaceAddendum,
+  type SkillEntry,
+} from '@supabase-evals/sandbox';
+import { buildSystemPrompt } from './system-prompt.js';
+
+const CLI_AGENTS: AgentHarnessId[] = ['claude-code', 'codex', 'opencode'];
+
+const skills: SkillEntry[] = [
+  {
+    name: 'supabase',
+    description: 'Use for Supabase tasks.',
+    dir: '.claude/skills/supabase',
+  },
+];
+
+describe('buildSystemPrompt', () => {
+  it('gives the ai-sdk agent task framing', () => {
+    // ai-sdk is the one harness with no system prompt of its own.
+    expect(buildSystemPrompt({ agent: 'ai-sdk' })).toContain(
+      'Use the provided tools'
+    );
+  });
+
+  it('gives nothing to any CLI harness', () => {
+    // CLI harnesses ship their own system prompt; that is what is measured.
+    for (const agent of CLI_AGENTS) {
+      expect(buildSystemPrompt({ agent })).toBe('');
+    }
+  });
+
+  it('assembles to nothing for a CLI harness, even with skills installed', () => {
+    // The real block producers, not stand-ins: each CLI finds the skills through
+    // its own project-scope discovery and describes them to the model itself.
+    for (const agent of CLI_AGENTS) {
+      expect(
+        buildSystemPrompt({
+          agent,
+          addendum: buildToolSurfaceAddendum(agent),
+          skillContext: buildSkillsPrompt(agent, skills),
+        })
+      ).toBe('');
+    }
+  });
+
+  it('keeps the runtime blocks for ai-sdk, in order, after the base prompt', () => {
+    const base = buildSystemPrompt({ agent: 'ai-sdk' });
+    expect(
+      buildSystemPrompt({
+        agent: 'ai-sdk',
+        addendum: 'Addendum.',
+        skillContext: 'Skills listing.',
+      })
+    ).toBe(`${base}\n\nAddendum.\n\nSkills listing.`);
+    const withSkills = buildSystemPrompt({
+      agent: 'ai-sdk',
+      addendum: buildToolSurfaceAddendum('ai-sdk'),
+      skillContext: buildSkillsPrompt('ai-sdk', skills),
+    });
+    expect(withSkills).toContain('## Available skills');
+    expect(withSkills).toContain('- supabase: Use for Supabase tasks.');
+  });
+
+  it('drops empty blocks instead of leaving blank gaps', () => {
+    const base = buildSystemPrompt({ agent: 'ai-sdk' });
+    expect(
+      buildSystemPrompt({
+        agent: 'ai-sdk',
+        addendum: '',
+        skillContext: 'Skills listing.',
+      })
+    ).toBe(`${base}\n\nSkills listing.`);
+    expect(
+      buildSystemPrompt({ agent: 'ai-sdk', addendum: '', skillContext: '' })
+    ).toBe(base);
+  });
+
+  it('never names Supabase, a project, or how to end the turn', () => {
+    // Issue #164: naming Supabase hands the agent the answer to "which tool";
+    // "project" presumes there is one to modify; stopping coaching shapes the
+    // report the judge reads. All three are part of what is measured.
+    const prompt = buildSystemPrompt({ agent: 'ai-sdk' });
+    expect(prompt).not.toMatch(/supabase/i);
+    expect(prompt).not.toMatch(/project/i);
+    expect(prompt).not.toMatch(/eval/i);
+    expect(prompt).not.toMatch(/summary|end your turn/i);
+  });
+});

apps/framework/harness/system-prompt.ts

@@ -0,0 +1,41 @@
+/**
+ * System-prompt assembly, per agent harness.
+ *
+ * An eval measures the agent as shipped, so the harness adds as little prompt
+ * of its own as it can. Only the ai-sdk agent gets any: it is the one harness
+ * with no system prompt of its own (`aiSdkAgent` hands `systemPrompt` straight
+ * to the model's `system`). Every CLI agent runs with the prompt it ships with,
+ * and `createCliAgent` refuses a non-empty one.
+ */
+
+import type { AgentHarnessId } from '@supabase-evals/core';
+
+/**
+ * Base framing for the ai-sdk harness: only what it cannot infer on its own,
+ * that it has tools and should use them. Deliberately says nothing about
+ * Supabase, a project, or how to finish a turn: which tools the agent reaches
+ * for and when it stops are part of what is measured (see issue #164).
+ */
+const AI_SDK_BASE_PROMPT =
+  'You are an agent. Use the provided tools to complete the task.';
+
+/**
+ * Assemble the system prompt for the agent. Every block is ai-sdk-only, so a
+ * CLI harness gets `''`. The runtime blocks (tool surface, skills listing) are
+ * already empty for a CLI agent at their source; an MCP server's
+ * `promptAddendum` is not, and is left to reach `createCliAgent`, which throws.
+ */
+export function buildSystemPrompt({
+  agent,
+  addendum,
+  skillContext,
+}: {
+  agent: AgentHarnessId;
+  /** Runtime text: the session's tool surface, or the MCP servers' addenda. */
+  addendum?: string;
+  /** The installed-skills listing. */
+  skillContext?: string;
+}): string {
+  const base = agent === 'ai-sdk' ? AI_SDK_BASE_PROMPT : '';
+  return [base, addendum, skillContext].filter(Boolean).join('\n\n');
+}

apps/framework/package.json

@@ -4,12 +4,13 @@
   "version": "0.0.1",
   "type": "module",
   "scripts": {
-    "check": "pnpm typecheck && pnpm test:framework && pnpm test:vercel-runner",
+    "check": "pnpm typecheck && pnpm test && pnpm test:framework && pnpm test:vercel-runner",
     "eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts",
     "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry",
     "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke",
     "eval:vercel": "node --env-file=../../.env --import tsx/esm scripts/run-vercel-evals.ts",
     "typecheck": "tsc --noEmit",
+    "test": "vitest run harness",
     "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts",
     "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts lib/cli-args.test.ts lib/sample-sets.test.ts",
     "export-results": "node --import tsx/esm scripts/export-results.ts",

packages/core/src/agents/claude-code/runner.test.ts

@@ -36,6 +36,35 @@ function streamJson(subtype: string, isError = false): string {
   ].join('\n');
 }
 
+/** The `claude` invocation from one exec, with a fake sandbox. */
+async function captureRunCommand(): Promise<string> {
+  let runCommand = '';
+  await claudeCodeRunner.exec({
+    sandbox: {
+      workspace: '/w',
+      exec: async (cmd) => {
+        if (cmd.includes('/bin/claude')) runCommand = cmd;
+        return ok;
+      },
+      readFile: async () => '',
+    },
+    model: 'claude-sonnet-4-6',
+    apiKey: 'k',
+    userPromptPath: '"$HOME/.eval/user-prompt.txt"',
+    mcpServers: {},
+    timeoutSec: 1,
+  });
+  return runCommand;
+}
+
+describe('claudeCodeRunner.exec', () => {
+  it("pipes the task in and leaves Claude Code's own system prompt intact", async () => {
+    const command = await captureRunCommand();
+    expect(command).toContain('cat "$HOME/.eval/user-prompt.txt" |');
+    expect(command).not.toContain('system-prompt');
+  });
+});
+
 describe('claudeCodeRunner.deriveStopReason', () => {
   const derive = claudeCodeRunner.deriveStopReason!;
 

packages/core/src/agents/claude-code/runner.ts

@@ -40,7 +40,6 @@ export const claudeCodeRunner: AgentRunner<AnthropicModel> = {
     sandbox,
     model,
     apiKey,
-    systemPromptPath,
     userPromptPath,
     mcpServers,
     reasoningEffort,
@@ -68,9 +67,6 @@ export const claudeCodeRunner: AgentRunner<AnthropicModel> = {
       `--model ${shellQuote(model)}`,
       // Reasoning effort for the session; omitted leaves Claude Code's default.
       ...(reasoningEffort ? [`--effort ${shellQuote(reasoningEffort)}`] : []),
-      // Append (not replace), from a file (no ARG_MAX/shell-expansion surface),
-      // so Claude Code keeps its default coding-agent prompt + tool guidance.
-      `--append-system-prompt-file ${systemPromptPath}`,
       ...mcpFlags,
       // The sandbox is the isolation boundary, so skip permission prompts and
       // give the agent its full native toolset (same in both modes).

packages/core/src/agents/codex/runner.test.ts

@@ -1,6 +1,42 @@
 import { describe, expect, it } from 'vitest';
+import type { CommandResult } from '../../index.js';
 import { codexRunner, countModelResponses } from './runner.js';
 
+const ok: CommandResult = { ok: true, exitCode: 0, stdout: '', stderr: '' };
+
+/** The `codex exec` invocation from one exec, with a fake sandbox. */
+async function captureRunCommand(): Promise<string> {
+  let runCommand = '';
+  await codexRunner.exec({
+    sandbox: {
+      workspace: '/w',
+      exec: async (cmd) => {
+        if (cmd.includes(' exec ')) runCommand = cmd;
+        return ok;
+      },
+      readFile: async () => '',
+    },
+    model: 'gpt-5.4',
+    apiKey: 'k',
+    userPromptPath: '"$HOME/.eval/user-prompt.txt"',
+    mcpServers: {},
+    timeoutSec: 1,
+  });
+  return runCommand;
+}
+
+describe('codexRunner.exec', () => {
+  it('sends the task alone on stdin', async () => {
+    // Codex has no system-prompt flag, so anything else here would land on the
+    // user prompt. Nothing is prepended.
+    const command = await captureRunCommand();
+    expect(command.startsWith('cat "$HOME/.eval/user-prompt.txt" |')).toBe(
+      true
+    );
+    expect(command).not.toContain('system-prompt');
+  });
+});
+
 describe('codexRunner.extractUsage', () => {
   const extract = codexRunner.extractUsage!;