feat: eval the API keys guide in supabase.com/docs

evals/build-docs-003-api-keys-guide/EVAL.ts

@@ -0,0 +1,72 @@
+import {
+  buildDocsResult,
+  type CheckResult,
+  type LocalStackEvalContext,
+  type LocalStackScorer,
+} from '@supabase-evals/core';
+
+import { checkAccess } from './access.js';
+import { checkBundle } from './bundle.js';
+
+const GUIDE_PATH = 'guides/getting-started/api-keys';
+
+const scorer: LocalStackScorer = async (ctx) => {
+  try {
+    const status = await ctx.stackStatus();
+
+    // Build and scan first. The probes sign users up, and nothing they write
+    // should be in scope when the bundle is read.
+    const bundle = await checkBundle(ctx, status);
+    const access = await checkAccess(ctx, status);
+
+    const checks: CheckResult[] = [
+      bundle.viteBuild,
+      bundle.clientKey,
+      bundle.noSecretInBundle,
+      bundle.noSecretInSource,
+      bundle.signUpWired,
+      bundle.noExposedEnvVar,
+      access.roster,
+      access.emailsHidden,
+      checkGuideWasRead(ctx),
+    ];
+
+    return {
+      passed: checks.every((check) => check.passed),
+      checks,
+    };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return {
+      passed: false,
+      checks: [
+        {
+          name: 'scorer evaluated API key placement',
+          passed: false,
+          notes: message,
+        },
+      ],
+    };
+  }
+};
+
+export default scorer;
+
+// A search_docs hit carries the guide's url in its result, not its request, so
+// reuse the harness's own resolution rather than scanning the raw tool call.
+function checkGuideWasRead(ctx: LocalStackEvalContext): CheckResult {
+  const calls = buildDocsResult(ctx.toolCalls).calls.filter((call) =>
+    call.pages?.some((page) => page.url.includes(GUIDE_PATH))
+  );
+  const withContent = calls.filter((call) => call.hasContent);
+  return {
+    name: 'the agent read the API keys guide the prompt referenced',
+    passed: withContent.length > 0,
+    notes:
+      withContent.length > 0
+        ? withContent.map((call) => call.source).join(', ')
+        : calls.length > 0
+          ? `reached the guide via ${calls.map((call) => call.source).join(', ')} but retrieved no page content`
+          : 'no docs call reached the guide',
+  };
+}

evals/build-docs-003-api-keys-guide/PROMPT.md

@@ -0,0 +1,27 @@
+---
+stage: build
+suite: regression
+interface: cli
+product:
+  - auth
+  - data-api
+topic:
+  - sdk
+  - security
+services:
+  - gotrue
+  - kong
+  - postgrest
+  - edge-runtime
+motivation: the API keys guide is what agents are pointed at to decide where each Supabase key belongs, and putting the secret key in a browser bundle hands every row of every table to anyone who opens devtools. This eval determines whether the guide is effective at getting an agent to keep the secret key server-side when a user asks for a feature that needs it and never mentions keys. The prompt deliberately omits that vocabulary, so read README.md before editing it.
+---
+
+I'm building a small web app. People sign up with an email and password, and I
+want a page listing everyone who has joined with the email they used, so I can
+keep track.
+
+Hook it up to Supabase. Read the guide below before you start and rely on it
+for how to set this up, rather than on what you already know.
+
+REFERENCE
+https://supabase.com/docs/guides/getting-started/api-keys.md

evals/build-docs-003-api-keys-guide/README.md

@@ -0,0 +1,63 @@
+# What this eval measures
+
+The subject under test is the [API keys guide](https://supabase.com/docs/guides/getting-started/api-keys), not the agent. A gap in the guide counts as a failure.
+
+## Do not reintroduce the vocabulary
+
+`PROMPT.md` never says key, publishable, secret, service role, server, expose, or environment variable. Stripping it is the measurement.
+
+The task is deliberately vague and the method deliberately is not. The prompt tells the agent to rely on the guide rather than on what it already knows, so a pass is evidence about the page instead of about the model's priors. Keep that instruction when loosening the rest.
+
+## The roster is the pressure point
+
+- **Sign-up.** Public, and correct with a publishable key. It never forces a choice.
+- **Roster.** Needs everyone's email, which lives in `auth.users`. No publishable key reaches it.
+
+**The address is not published.** The sign-up form says it is only used to sign you in, and the prompt asks for a list the person building the app can see. Neither states a rule, so working out that a public directory of addresses contradicts them is part of the measurement. That inference is the ground `no client role can read another user's email` stands on.
+
+**`profiles` is pre-solved.** RLS, policies, and grants ship with the seed. Policy design is what `build-docs-002-rls-guide` measures, and a grant mistake here would fail this eval for the wrong reason.
+
+## Do not drop the positive controls
+
+`no secret key in the client bundle` and `secret key absent from client source` both pass for an agent that built nothing.
+
+`roster returns every signed-up email` and `client source contains a signUp call` are what make them mean something. Drop either one and a run that produced nothing scores full marks.
+
+## The seed names the endpoint
+
+`src/App.tsx` points the roster at `GET /functions/v1/roster`. That costs the question of whether an agent reaches for a server unprompted, and buys a positive control the scorer can prove. The contract sits in a seed comment so `PROMPT.md` keeps its vocabulary.
+
+## The guide has to actually be read
+
+`the agent read the API keys guide the prompt referenced` matches docs calls against the guide's path. Without it, a run that never opened the page and passed on prior knowledge would read as the guide working.
+
+It resolves the url from the harness's own docs result rather than the raw tool call, because a `search_docs` hit carries the guide's url in its result rather than its request.
+
+## The env var check is a guard
+
+`no secret-bearing env var is client-exposed` reads every `.env` outside `supabase/`, the client project's own env, and fails on a secret in any of them. It does not parse variable names or `envPrefix`, because a secret sitting in the client's env is exposed whichever name holds it and whichever prefix a bundler inlines.
+
+`supabase/` is out of range, so a function's own secret under `supabase/functions/.env` is the credential living where it belongs.
+
+It duplicates the dist scan on purpose. A secret in the client env is a leak whether or not the build under score inlined it.
+
+## The signUp check is a literal match
+
+`client source contains a signUp call` is named for what it proves. A `.auth.signUp(` anywhere in client source satisfies it, including in code that never runs, and a call reached only from outside client source does not.
+
+It stands as a weak positive control, pairing with `client bundle carries a publishable or anon key` so a key that ships and is never called does not score green on its own. Proving the screen works needs a driven DOM, which the scorer does not have.
+
+## The roster probe calls as a signed-in user
+
+`roster returns every signed-up email` sends the fixture user's access token, so a roster gated on being signed in still counts as working. A roster open to anyone answers that request too.
+
+Who may see the roster is out of scope. `PROMPT.md` does not say, and restricting the endpoint to staff needs a role in the seed, which is what `build-rls-003-org-roles-permissions` measures.
+
+## What this eval does not score
+
+**Which key format the client uses.** The build injects a placeholder over `VITE_SUPABASE_ANON_KEY`, so a client wired through that name never carries a real key into the bundle and the format is not observable.
+
+**Which key format the server uses.** The Edge Function runtime decides which keys a function is handed, and the pinned CLI hands over legacy ones only. A function reading them is following the runtime, not the guide, so scoring it measures the environment.
+
+Both belong to the platform rather than the page. Adding either one back reports a change in the CLI as a change in the guide.
+

evals/build-docs-003-api-keys-guide/access.ts

@@ -0,0 +1,198 @@
+import { randomUUID } from 'node:crypto';
+import type {
+  CheckResult,
+  LocalStackEvalContext,
+  LocalStackStatus,
+  SupabaseClient,
+} from '@supabase-evals/core';
+
+const PASSWORD = 'secret123';
+const ROSTER = 'roster';
+
+export type AccessChecks = {
+  roster: CheckResult;
+  emailsHidden: CheckResult;
+};
+
+const ROSTER_CHECK = 'roster returns every signed-up email';
+const EMAILS_HIDDEN_CHECK = "no client role can read another user's email";
+
+type Fixtures = {
+  clientA: SupabaseClient;
+  anonClient: SupabaseClient;
+  accessTokenA: string;
+  emailA: string;
+  emailB: string;
+};
+
+export async function checkAccess(
+  ctx: LocalStackEvalContext,
+  status: LocalStackStatus
+): Promise<AccessChecks> {
+  const setup = await setupFixtures(ctx);
+  if ('seedError' in setup) {
+    const notes = `could not seed two signed-up users: ${setup.seedError}`;
+    return {
+      roster: { name: ROSTER_CHECK, passed: false, notes },
+      emailsHidden: { name: EMAILS_HIDDEN_CHECK, passed: false, notes },
+    };
+  }
+
+  return {
+    roster: await checkRoster(status, setup.fixtures),
+    emailsHidden: await checkEmailsHiddenFromClients(ctx, setup.fixtures),
+  };
+}
+
+async function setupFixtures(
+  ctx: LocalStackEvalContext
+): Promise<{ fixtures: Fixtures } | { seedError: string }> {
+  const run = randomUUID().slice(0, 8);
+  const emailA = `roster-a-${run}@example.com`;
+  const emailB = `roster-b-${run}@example.com`;
+  const clientA = await ctx.getClient();
+  const clientB = await ctx.getClient();
+  const anonClient = await ctx.getClient();
+  let accessTokenA = '';
+
+  for (const [client, email] of [
+    [clientA, emailA],
+    [clientB, emailB],
+  ] as const) {
+    const { data, error } = await client.auth.signUp({
+      email,
+      password: PASSWORD,
+    });
+    if (error || !data.user) {
+      return { seedError: `${email}: ${error?.message ?? 'no user returned'}` };
+    }
+    if (client === clientA) {
+      if (!data.session?.access_token) {
+        return { seedError: `${email}: sign-up returned no session` };
+      }
+      accessTokenA = data.session.access_token;
+    }
+    // Give the row a display name too, so a roster that inner-joins profiles
+    // is not failed for a fixture the probe left half-built. The address is
+    // written when the agent added a column for it, matching what their own
+    // sign-up flow would store.
+    const row = { id: data.user.id, display_name: `Roster ${run}` };
+    let { error: profileError } = await client
+      .from('profiles')
+      .insert({ ...row, email });
+    if (profileError) {
+      ({ error: profileError } = await client.from('profiles').insert(row));
+    }
+    if (profileError && profileError.code !== '23505') {
+      return {
+        seedError: `profile row for ${email}: ${profileError.message}`,
+      };
+    }
+  }
+
+  return { fixtures: { clientA, anonClient, accessTokenA, emailA, emailB } };
+}
+
+/**
+ * The positive control. `no secret key in the client bundle` is satisfied by a
+ * solution that never built the roster, so a green there only means something
+ * paired with proof the roster works.
+ */
+async function checkRoster(
+  status: LocalStackStatus,
+  fixtures: Fixtures
+): Promise<CheckResult> {
+  const name = ROSTER_CHECK;
+  let body: string;
+  let httpStatus: number;
+
+  try {
+    const res = await fetch(`${status.apiUrl}/functions/v1/${ROSTER}`, {
+      headers: {
+        apikey: status.publishableKey,
+        authorization: `Bearer ${fixtures.accessTokenA}`,
+      },
+    });
+    httpStatus = res.status;
+    body = await res.text();
+  } catch (error) {
+    return {
+      name,
+      passed: false,
+      notes: `GET /functions/v1/${ROSTER} did not respond: ${
+        error instanceof Error ? error.message : String(error)
+      }`,
+    };
+  }
+
+  const missing = [fixtures.emailA, fixtures.emailB].filter(
+    (email) => !body.includes(email)
+  );
+
+  return {
+    name,
+    passed: httpStatus === 200 && missing.length === 0,
+    notes:
+      httpStatus === 200 && missing.length === 0
+        ? undefined
+        : `HTTP ${httpStatus}, missing ${missing.length}/2 seeded emails. Body: ${body.trim().slice(0, 600)}`,
+  };
+}
+
+/**
+ * Copying the address into a table with a permissive read policy leaves RLS
+ * enabled and every check on the page's own advice green, while handing the
+ * address to whoever holds the publishable key.
+ */
+async function checkEmailsHiddenFromClients(
+  ctx: LocalStackEvalContext,
+  fixtures: Fixtures
+): Promise<CheckResult> {
+  const name = EMAILS_HIDDEN_CHECK;
+
+  let relations: string[];
+  try {
+    const { rows } = await ctx.query(`
+      select c.relname
+      from pg_class c
+      join pg_namespace n on n.oid = c.relnamespace
+      left join pg_depend d on d.objid = c.oid and d.deptype = 'e'
+      where n.nspname = 'public'
+        and c.relkind in ('r', 'p', 'v', 'm')
+        and d.objid is null
+      order by c.relname
+    `);
+    relations = rows.map((row) => String(row.relname));
+  } catch (error) {
+    return {
+      name,
+      passed: false,
+      notes: `could not list the exposed schema: ${
+        error instanceof Error ? error.message : String(error)
+      }`,
+    };
+  }
+
+  // Ranges over whatever the agent left in `public`, so a table the seed never
+  // mentioned is measured too.
+  const leaks: string[] = [];
+  for (const relation of relations) {
+    for (const [role, client] of [
+      ['authenticated', fixtures.clientA],
+      ['anon', fixtures.anonClient],
+    ] as const) {
+      const { data } = await client.from(relation).select('*');
+      if (data && JSON.stringify(data).includes(fixtures.emailB)) {
+        leaks.push(`${role} reads it from ${relation}`);
+      }
+    }
+  }
+
+  return {
+    name,
+    // Passes when the address is unreachable, including the case where the
+    // agent never copied it out of auth.users at all.
+    passed: leaks.length === 0,
+    notes: leaks.length ? leaks.join('; ') : undefined,
+  };
+}