diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index 227a32c3..1c10f6c2 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -297,6 +297,356 @@ "attempts": 2, "sourcePath": "claude-code-sonnet-5/build-docs-002-rls-guide.json" }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-docs-003-api-keys-guide", + "stage": "build", + "product": [ + "auth", + "data-api" + ], + "topic": [ + "sdk", + "security" + ], + "suite": "regression", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "vite build passed", + "passed": true + }, + { + "name": "client bundle carries a publishable or anon key", + "passed": true + }, + { + "name": "no secret key in the client bundle", + "passed": true + }, + { + "name": "secret key absent from client source", + "passed": true + }, + { + "name": "client source contains a signUp call", + "passed": true + }, + { + "name": "no secret-bearing env var is client-exposed", + "passed": true + }, + { + "name": "roster returns every signed-up email", + "passed": true + }, + { + "name": "no client role can read another user's email", + "passed": true + }, + { + "name": "the agent read the API keys guide the prompt referenced", + "passed": true, + "notes": "web_fetch" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "Summarize the current guidance on API keys: publishable key vs secret key vs legacy anon/service_role keys, how to fetch them, and which one to use in frontend vs backend code (e.g. Edge Functions).", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys.md" + } + ], + "resultChars": 1400 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"default environment variables available in Edge Functions SUPABASE_URL SUPABASE_SERVICE_ROLE_KEY SUPABASE_SECRET_KEY\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", + "title": "Inspecting edge function environment variables" + } + ], + "resultChars": 38255 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable key secret key vs anon key service_role key migration new API keys\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/rotating-anon-service-and-jwt-secrets-1Jq6yd", + "title": "Rotating Anon, Service, and JWT Secrets" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + } + ], + "resultChars": 63574 + } + ] + }, + "prompt": "I'm building a small web app. People sign up with an email and password, and I\nwant a page listing everyone who has joined with the email they used, so I can\nkeep track.\n\nHook it up to Supabase. Read the guide below before you start and rely on it\nfor how to set this up, rather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/getting-started/api-keys.md", + "promptSourcePath": "evals/build-docs-003-api-keys-guide/PROMPT.md", + "run": 1, + "sourcePath": "claude-code-sonnet-5/build-docs-003-api-keys-guide/run-1/result.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-docs-003-api-keys-guide", + "stage": "build", + "product": [ + "auth", + "data-api" + ], + "topic": [ + "sdk", + "security" + ], + "suite": "regression", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "vite build passed", + "passed": true + }, + { + "name": "client bundle carries a publishable or anon key", + "passed": true + }, + { + "name": "no secret key in the client bundle", + "passed": true + }, + { + "name": "secret key absent from client source", + "passed": true + }, + { + "name": "client source contains a signUp call", + "passed": true + }, + { + "name": "no secret-bearing env var is client-exposed", + "passed": true + }, + { + "name": "roster returns every signed-up email", + "passed": true + }, + { + "name": "no client role can read another user's email", + "passed": true + }, + { + "name": "the agent read the API keys guide the prompt referenced", + "passed": true, + "notes": "web_fetch" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "Summarize the guidance on API keys: publishable vs secret keys, legacy anon/service_role keys, which key to use where (frontend vs backend), and any migration/security recommendations.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys.md" + } + ], + "resultChars": 1794 + } + ] + }, + "prompt": "I'm building a small web app. People sign up with an email and password, and I\nwant a page listing everyone who has joined with the email they used, so I can\nkeep track.\n\nHook it up to Supabase. Read the guide below before you start and rely on it\nfor how to set this up, rather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/getting-started/api-keys.md", + "promptSourcePath": "evals/build-docs-003-api-keys-guide/PROMPT.md", + "run": 2, + "sourcePath": "claude-code-sonnet-5/build-docs-003-api-keys-guide/run-2/result.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-docs-003-api-keys-guide", + "stage": "build", + "product": [ + "auth", + "data-api" + ], + "topic": [ + "sdk", + "security" + ], + "suite": "regression", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "vite build passed", + "passed": true + }, + { + "name": "client bundle carries a publishable or anon key", + "passed": true + }, + { + "name": "no secret key in the client bundle", + "passed": true + }, + { + "name": "secret key absent from client source", + "passed": true + }, + { + "name": "client source contains a signUp call", + "passed": true + }, + { + "name": "no secret-bearing env var is client-exposed", + "passed": true + }, + { + "name": "roster returns every signed-up email", + "passed": true + }, + { + "name": "no client role can read another user's email", + "passed": true + }, + { + "name": "the agent read the API keys guide the prompt referenced", + "passed": true, + "notes": "web_fetch" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "Summarize the key concepts: what are publishable/anon keys vs secret/service_role keys, where each should be used (client vs server), how to fetch them via CLI or dashboard, and any current naming conventions (new API keys vs legacy JWT-based keys). Include exact env var names and CLI commands mentioned.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys.md" + } + ], + "resultChars": 1359 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Edge Functions default environment variables SUPABASE_SERVICE_ROLE_KEY SUPABASE_URL secret key\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + } + ], + "resultChars": 41771 + } + ] + }, + "prompt": "I'm building a small web app. People sign up with an email and password, and I\nwant a page listing everyone who has joined with the email they used, so I can\nkeep track.\n\nHook it up to Supabase. Read the guide below before you start and rely on it\nfor how to set this up, rather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/getting-started/api-keys.md", + "promptSourcePath": "evals/build-docs-003-api-keys-guide/PROMPT.md", + "run": 3, + "sourcePath": "claude-code-sonnet-5/build-docs-003-api-keys-guide/run-3/result.json" + }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "regression", @@ -1485,6 +1835,261 @@ "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-002-rls-guide.json" }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-docs-003-api-keys-guide", + "stage": "build", + "product": [ + "auth", + "data-api" + ], + "topic": [ + "sdk", + "security" + ], + "suite": "regression", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "vite build passed", + "passed": true + }, + { + "name": "client bundle carries a publishable or anon key", + "passed": true + }, + { + "name": "no secret key in the client bundle", + "passed": true + }, + { + "name": "secret key absent from client source", + "passed": true + }, + { + "name": "client source contains a signUp call", + "passed": true + }, + { + "name": "no secret-bearing env var is client-exposed", + "passed": true + }, + { + "name": "roster returns every signed-up email", + "passed": true + }, + { + "name": "no client role can read another user's email", + "passed": true + }, + { + "name": "the agent read the API keys guide the prompt referenced", + "passed": true, + "notes": "web_fetch" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "Summarize this entire guide in detail: the API key system (publishable keys, secret keys, legacy anon/service_role keys), how to find/create them, which key to use client-side vs server-side, and any security guidance about querying user data like auth.users or listing signed-up users.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys.md" + } + ], + "resultChars": 2494 + } + ] + }, + "prompt": "I'm building a small web app. People sign up with an email and password, and I\nwant a page listing everyone who has joined with the email they used, so I can\nkeep track.\n\nHook it up to Supabase. Read the guide below before you start and rely on it\nfor how to set this up, rather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/getting-started/api-keys.md", + "promptSourcePath": "evals/build-docs-003-api-keys-guide/PROMPT.md", + "run": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-003-api-keys-guide/run-1/result.json" + }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-docs-003-api-keys-guide", + "stage": "build", + "product": [ + "auth", + "data-api" + ], + "topic": [ + "sdk", + "security" + ], + "suite": "regression", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "vite build passed", + "passed": true + }, + { + "name": "client bundle carries a publishable or anon key", + "passed": true + }, + { + "name": "no secret key in the client bundle", + "passed": true + }, + { + "name": "secret key absent from client source", + "passed": true + }, + { + "name": "client source contains a signUp call", + "passed": true + }, + { + "name": "no secret-bearing env var is client-exposed", + "passed": true + }, + { + "name": "roster returns every signed-up email", + "passed": true + }, + { + "name": "no client role can read another user's email", + "passed": true + }, + { + "name": "the agent read the API keys guide the prompt referenced", + "passed": true, + "notes": "web_fetch" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "Summarize the guidance on API keys: what keys exist (publishable/anon, secret/service_role, legacy JWT-based keys), what each is used for, which is safe to use in a browser/frontend app vs server, how they relate to RLS, and any recommendations about which key to use when. Include specifics about naming (e.g., anon key vs publishable key, service_role vs secret key) and any migration guidance.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys.md" + } + ], + "resultChars": 1465 + } + ] + }, + "prompt": "I'm building a small web app. People sign up with an email and password, and I\nwant a page listing everyone who has joined with the email they used, so I can\nkeep track.\n\nHook it up to Supabase. Read the guide below before you start and rely on it\nfor how to set this up, rather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/getting-started/api-keys.md", + "promptSourcePath": "evals/build-docs-003-api-keys-guide/PROMPT.md", + "run": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-003-api-keys-guide/run-2/result.json" + }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-docs-003-api-keys-guide", + "stage": "build", + "product": [ + "auth", + "data-api" + ], + "topic": [ + "sdk", + "security" + ], + "suite": "regression", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "vite build passed", + "passed": true + }, + { + "name": "client bundle carries a publishable or anon key", + "passed": true + }, + { + "name": "no secret key in the client bundle", + "passed": true + }, + { + "name": "secret key absent from client source", + "passed": true + }, + { + "name": "client source contains a signUp call", + "passed": true + }, + { + "name": "no secret-bearing env var is client-exposed", + "passed": true + }, + { + "name": "roster returns every signed-up email", + "passed": true + }, + { + "name": "no client role can read another user's email", + "passed": true + }, + { + "name": "the agent read the API keys guide the prompt referenced", + "passed": true, + "notes": "web_fetch" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "Summarize this guide in full detail: what are the different types of API keys in Supabase (legacy and new), what is each used for, which ones are safe to expose in a browser/client app vs which must stay secret on a server, and how do they relate to Row Level Security. Include any specific recommendations about publishable/anon keys vs secret/service_role keys.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys.md" + } + ], + "resultChars": 1710 + } + ] + }, + "prompt": "I'm building a small web app. People sign up with an email and password, and I\nwant a page listing everyone who has joined with the email they used, so I can\nkeep track.\n\nHook it up to Supabase. Read the guide below before you start and rely on it\nfor how to set this up, rather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/getting-started/api-keys.md", + "promptSourcePath": "evals/build-docs-003-api-keys-guide/PROMPT.md", + "run": 3, + "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-003-api-keys-guide/run-3/result.json" + }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "regression", diff --git a/evals/build-docs-003-api-keys-guide/EVAL.ts b/evals/build-docs-003-api-keys-guide/EVAL.ts new file mode 100644 index 00000000..5223bddc --- /dev/null +++ b/evals/build-docs-003-api-keys-guide/EVAL.ts @@ -0,0 +1,72 @@ +import { + buildDocsResult, + type CheckResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; + +import { checkAccess } from './access.js'; +import { checkBundle } from './bundle.js'; + +const GUIDE_PATH = 'guides/getting-started/api-keys'; + +const scorer: LocalStackScorer = async (ctx) => { + try { + const status = await ctx.stackStatus(); + + // Build and scan first. The probes sign users up, and nothing they write + // should be in scope when the bundle is read. + const bundle = await checkBundle(ctx, status); + const access = await checkAccess(ctx, status); + + const checks: CheckResult[] = [ + bundle.viteBuild, + bundle.clientKey, + bundle.noSecretInBundle, + bundle.noSecretInSource, + bundle.signUpWired, + bundle.noExposedEnvVar, + access.roster, + access.emailsHidden, + checkGuideWasRead(ctx), + ]; + + return { + passed: checks.every((check) => check.passed), + checks, + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { + passed: false, + checks: [ + { + name: 'scorer evaluated API key placement', + passed: false, + notes: message, + }, + ], + }; + } +}; + +export default scorer; + +// A search_docs hit carries the guide's url in its result, not its request, so +// reuse the harness's own resolution rather than scanning the raw tool call. +function checkGuideWasRead(ctx: LocalStackEvalContext): CheckResult { + const calls = buildDocsResult(ctx.toolCalls).calls.filter((call) => + call.pages?.some((page) => page.url.includes(GUIDE_PATH)) + ); + const withContent = calls.filter((call) => call.hasContent); + return { + name: 'the agent read the API keys guide the prompt referenced', + passed: withContent.length > 0, + notes: + withContent.length > 0 + ? withContent.map((call) => call.source).join(', ') + : calls.length > 0 + ? `reached the guide via ${calls.map((call) => call.source).join(', ')} but retrieved no page content` + : 'no docs call reached the guide', + }; +} diff --git a/evals/build-docs-003-api-keys-guide/PROMPT.md b/evals/build-docs-003-api-keys-guide/PROMPT.md new file mode 100644 index 00000000..f0a7fdda --- /dev/null +++ b/evals/build-docs-003-api-keys-guide/PROMPT.md @@ -0,0 +1,27 @@ +--- +stage: build +suite: regression +interface: cli +product: + - auth + - data-api +topic: + - sdk + - security +services: + - gotrue + - kong + - postgrest + - edge-runtime +motivation: the API keys guide is what agents are pointed at to decide where each Supabase key belongs, and putting the secret key in a browser bundle hands every row of every table to anyone who opens devtools. This eval determines whether the guide is effective at getting an agent to keep the secret key server-side when a user asks for a feature that needs it and never mentions keys. The prompt deliberately omits that vocabulary, so read README.md before editing it. +--- + +I'm building a small web app. People sign up with an email and password, and I +want a page listing everyone who has joined with the email they used, so I can +keep track. + +Hook it up to Supabase. Read the guide below before you start and rely on it +for how to set this up, rather than on what you already know. + +REFERENCE +https://supabase.com/docs/guides/getting-started/api-keys.md diff --git a/evals/build-docs-003-api-keys-guide/README.md b/evals/build-docs-003-api-keys-guide/README.md new file mode 100644 index 00000000..c7edccca --- /dev/null +++ b/evals/build-docs-003-api-keys-guide/README.md @@ -0,0 +1,63 @@ +# What this eval measures + +The subject under test is the [API keys guide](https://supabase.com/docs/guides/getting-started/api-keys), not the agent. A gap in the guide counts as a failure. + +## Do not reintroduce the vocabulary + +`PROMPT.md` never says key, publishable, secret, service role, server, expose, or environment variable. Stripping it is the measurement. + +The task is deliberately vague and the method deliberately is not. The prompt tells the agent to rely on the guide rather than on what it already knows, so a pass is evidence about the page instead of about the model's priors. Keep that instruction when loosening the rest. + +## The roster is the pressure point + +- **Sign-up.** Public, and correct with a publishable key. It never forces a choice. +- **Roster.** Needs everyone's email, which lives in `auth.users`. No publishable key reaches it. + +**The address is not published.** The sign-up form says it is only used to sign you in, and the prompt asks for a list the person building the app can see. Neither states a rule, so working out that a public directory of addresses contradicts them is part of the measurement. That inference is the ground `no client role can read another user's email` stands on. + +**`profiles` is pre-solved.** RLS, policies, and grants ship with the seed. Policy design is what `build-docs-002-rls-guide` measures, and a grant mistake here would fail this eval for the wrong reason. + +## Do not drop the positive controls + +`no secret key in the client bundle` and `secret key absent from client source` both pass for an agent that built nothing. + +`roster returns every signed-up email` and `client source contains a signUp call` are what make them mean something. Drop either one and a run that produced nothing scores full marks. + +## The seed names the endpoint + +`src/App.tsx` points the roster at `GET /functions/v1/roster`. That costs the question of whether an agent reaches for a server unprompted, and buys a positive control the scorer can prove. The contract sits in a seed comment so `PROMPT.md` keeps its vocabulary. + +## The guide has to actually be read + +`the agent read the API keys guide the prompt referenced` matches docs calls against the guide's path. Without it, a run that never opened the page and passed on prior knowledge would read as the guide working. + +It resolves the url from the harness's own docs result rather than the raw tool call, because a `search_docs` hit carries the guide's url in its result rather than its request. + +## The env var check is a guard + +`no secret-bearing env var is client-exposed` reads every `.env` outside `supabase/`, the client project's own env, and fails on a secret in any of them. It does not parse variable names or `envPrefix`, because a secret sitting in the client's env is exposed whichever name holds it and whichever prefix a bundler inlines. + +`supabase/` is out of range, so a function's own secret under `supabase/functions/.env` is the credential living where it belongs. + +It duplicates the dist scan on purpose. A secret in the client env is a leak whether or not the build under score inlined it. + +## The signUp check is a literal match + +`client source contains a signUp call` is named for what it proves. A `.auth.signUp(` anywhere in client source satisfies it, including in code that never runs, and a call reached only from outside client source does not. + +It stands as a weak positive control, pairing with `client bundle carries a publishable or anon key` so a key that ships and is never called does not score green on its own. Proving the screen works needs a driven DOM, which the scorer does not have. + +## The roster probe calls as a signed-in user + +`roster returns every signed-up email` sends the fixture user's access token, so a roster gated on being signed in still counts as working. A roster open to anyone answers that request too. + +Who may see the roster is out of scope. `PROMPT.md` does not say, and restricting the endpoint to staff needs a role in the seed, which is what `build-rls-003-org-roles-permissions` measures. + +## What this eval does not score + +**Which key format the client uses.** The build injects a placeholder over `VITE_SUPABASE_ANON_KEY`, so a client wired through that name never carries a real key into the bundle and the format is not observable. + +**Which key format the server uses.** The Edge Function runtime decides which keys a function is handed, and the pinned CLI hands over legacy ones only. A function reading them is following the runtime, not the guide, so scoring it measures the environment. + +Both belong to the platform rather than the page. Adding either one back reports a change in the CLI as a change in the guide. + diff --git a/evals/build-docs-003-api-keys-guide/access.ts b/evals/build-docs-003-api-keys-guide/access.ts new file mode 100644 index 00000000..e05aa234 --- /dev/null +++ b/evals/build-docs-003-api-keys-guide/access.ts @@ -0,0 +1,198 @@ +import { randomUUID } from 'node:crypto'; +import type { + CheckResult, + LocalStackEvalContext, + LocalStackStatus, + SupabaseClient, +} from '@supabase-evals/core'; + +const PASSWORD = 'secret123'; +const ROSTER = 'roster'; + +export type AccessChecks = { + roster: CheckResult; + emailsHidden: CheckResult; +}; + +const ROSTER_CHECK = 'roster returns every signed-up email'; +const EMAILS_HIDDEN_CHECK = "no client role can read another user's email"; + +type Fixtures = { + clientA: SupabaseClient; + anonClient: SupabaseClient; + accessTokenA: string; + emailA: string; + emailB: string; +}; + +export async function checkAccess( + ctx: LocalStackEvalContext, + status: LocalStackStatus +): Promise { + const setup = await setupFixtures(ctx); + if ('seedError' in setup) { + const notes = `could not seed two signed-up users: ${setup.seedError}`; + return { + roster: { name: ROSTER_CHECK, passed: false, notes }, + emailsHidden: { name: EMAILS_HIDDEN_CHECK, passed: false, notes }, + }; + } + + return { + roster: await checkRoster(status, setup.fixtures), + emailsHidden: await checkEmailsHiddenFromClients(ctx, setup.fixtures), + }; +} + +async function setupFixtures( + ctx: LocalStackEvalContext +): Promise<{ fixtures: Fixtures } | { seedError: string }> { + const run = randomUUID().slice(0, 8); + const emailA = `roster-a-${run}@example.com`; + const emailB = `roster-b-${run}@example.com`; + const clientA = await ctx.getClient(); + const clientB = await ctx.getClient(); + const anonClient = await ctx.getClient(); + let accessTokenA = ''; + + for (const [client, email] of [ + [clientA, emailA], + [clientB, emailB], + ] as const) { + const { data, error } = await client.auth.signUp({ + email, + password: PASSWORD, + }); + if (error || !data.user) { + return { seedError: `${email}: ${error?.message ?? 'no user returned'}` }; + } + if (client === clientA) { + if (!data.session?.access_token) { + return { seedError: `${email}: sign-up returned no session` }; + } + accessTokenA = data.session.access_token; + } + // Give the row a display name too, so a roster that inner-joins profiles + // is not failed for a fixture the probe left half-built. The address is + // written when the agent added a column for it, matching what their own + // sign-up flow would store. + const row = { id: data.user.id, display_name: `Roster ${run}` }; + let { error: profileError } = await client + .from('profiles') + .insert({ ...row, email }); + if (profileError) { + ({ error: profileError } = await client.from('profiles').insert(row)); + } + if (profileError && profileError.code !== '23505') { + return { + seedError: `profile row for ${email}: ${profileError.message}`, + }; + } + } + + return { fixtures: { clientA, anonClient, accessTokenA, emailA, emailB } }; +} + +/** + * The positive control. `no secret key in the client bundle` is satisfied by a + * solution that never built the roster, so a green there only means something + * paired with proof the roster works. + */ +async function checkRoster( + status: LocalStackStatus, + fixtures: Fixtures +): Promise { + const name = ROSTER_CHECK; + let body: string; + let httpStatus: number; + + try { + const res = await fetch(`${status.apiUrl}/functions/v1/${ROSTER}`, { + headers: { + apikey: status.publishableKey, + authorization: `Bearer ${fixtures.accessTokenA}`, + }, + }); + httpStatus = res.status; + body = await res.text(); + } catch (error) { + return { + name, + passed: false, + notes: `GET /functions/v1/${ROSTER} did not respond: ${ + error instanceof Error ? error.message : String(error) + }`, + }; + } + + const missing = [fixtures.emailA, fixtures.emailB].filter( + (email) => !body.includes(email) + ); + + return { + name, + passed: httpStatus === 200 && missing.length === 0, + notes: + httpStatus === 200 && missing.length === 0 + ? undefined + : `HTTP ${httpStatus}, missing ${missing.length}/2 seeded emails. Body: ${body.trim().slice(0, 600)}`, + }; +} + +/** + * Copying the address into a table with a permissive read policy leaves RLS + * enabled and every check on the page's own advice green, while handing the + * address to whoever holds the publishable key. + */ +async function checkEmailsHiddenFromClients( + ctx: LocalStackEvalContext, + fixtures: Fixtures +): Promise { + const name = EMAILS_HIDDEN_CHECK; + + let relations: string[]; + try { + const { rows } = await ctx.query(` + select c.relname + from pg_class c + join pg_namespace n on n.oid = c.relnamespace + left join pg_depend d on d.objid = c.oid and d.deptype = 'e' + where n.nspname = 'public' + and c.relkind in ('r', 'p', 'v', 'm') + and d.objid is null + order by c.relname + `); + relations = rows.map((row) => String(row.relname)); + } catch (error) { + return { + name, + passed: false, + notes: `could not list the exposed schema: ${ + error instanceof Error ? error.message : String(error) + }`, + }; + } + + // Ranges over whatever the agent left in `public`, so a table the seed never + // mentioned is measured too. + const leaks: string[] = []; + for (const relation of relations) { + for (const [role, client] of [ + ['authenticated', fixtures.clientA], + ['anon', fixtures.anonClient], + ] as const) { + const { data } = await client.from(relation).select('*'); + if (data && JSON.stringify(data).includes(fixtures.emailB)) { + leaks.push(`${role} reads it from ${relation}`); + } + } + } + + return { + name, + // Passes when the address is unreachable, including the case where the + // agent never copied it out of auth.users at all. + passed: leaks.length === 0, + notes: leaks.length ? leaks.join('; ') : undefined, + }; +} diff --git a/evals/build-docs-003-api-keys-guide/bundle.ts b/evals/build-docs-003-api-keys-guide/bundle.ts new file mode 100644 index 00000000..f62611c0 --- /dev/null +++ b/evals/build-docs-003-api-keys-guide/bundle.ts @@ -0,0 +1,154 @@ +import { join, relative } from 'node:path'; +import type { + CheckResult, + LocalStackEvalContext, + LocalStackStatus, +} from '@supabase-evals/core'; + +import { readText, walk } from './files.js'; +import { findSecrets } from './keys.js'; + +export type BundleChecks = { + viteBuild: CheckResult; + clientKey: CheckResult; + noSecretInBundle: CheckResult; + noSecretInSource: CheckResult; + signUpWired: CheckResult; + noExposedEnvVar: CheckResult; +}; + +export async function checkBundle( + ctx: LocalStackEvalContext, + status: LocalStackStatus +): Promise { + const build = await ctx.runViteBuild(); + const viteBuild: CheckResult = { + name: 'vite build passed', + passed: build.ok, + notes: build.ok + ? undefined + : (build.stderr || build.stdout).trim().slice(0, 4000), + }; + + if (!build.ok) { + // Not a pass. A missing bundle is the absence of evidence, and reporting + // these as green would hand a clean sheet to a solution that never built. + return { + viteBuild, + clientKey: notRun('client bundle carries a publishable or anon key'), + noSecretInBundle: notRun('no secret key in the client bundle'), + ...sourceChecks(ctx, status), + }; + } + + const distRoot = join(ctx.hostWorkspace, 'dist'); + const dist = walk(distRoot, distRoot).map(readText).join('\n'); + + const leaked = findSecrets(dist, status); + const clientKeys = [status.publishableKey, status.anonKey].filter(Boolean); + const carriesClientKey = clientKeys.some((key) => dist.includes(key)); + + return { + viteBuild, + clientKey: { + name: 'client bundle carries a publishable or anon key', + passed: carriesClientKey, + notes: carriesClientKey + ? undefined + : 'the built client never reaches the project with a low-privilege key, so the sign-up screen is not wired up', + }, + noSecretInBundle: { + name: 'no secret key in the client bundle', + passed: leaked.length === 0, + notes: leaked.length ? `found in dist/: ${leaked.join(', ')}` : undefined, + }, + ...sourceChecks(ctx, status), + }; +} + +function clientSourceFilter(hostWorkspace: string): (rel: string) => boolean { + const config = readText(join(hostWorkspace, 'vite.config.ts')); + const root = /\broot\s*:\s*['"`]([^'"`]+)['"`]/.exec(config)?.[1] ?? '.'; + const outDir = + /\boutDir\s*:\s*['"`]([^'"`]+)['"`]/.exec(config)?.[1] ?? 'dist'; + const prefix = + root === '.' || root === '' ? '' : `${root.replace(/^\.\//, '')}/`; + const excluded = [outDir, 'supabase', 'node_modules'].map( + (dir) => `${prefix}${dir}/` + ); + + return (rel) => + rel.startsWith(prefix) && + !excluded.some((dir) => rel.startsWith(dir)) && + !/(^|\/)\.env/.test(rel); +} + +function sourceChecks( + ctx: LocalStackEvalContext, + status: LocalStackStatus +): Pick { + const root = ctx.hostWorkspace; + const files = walk(root, root); + const isClientSource = clientSourceFilter(root); + const clientSource = files.filter((file) => + isClientSource(relative(root, file)) + ); + + const offenders: string[] = []; + for (const file of clientSource) { + const hits = findSecrets(readText(file), status); + if (hits.length) offenders.push(relative(root, file)); + } + + // Static, because the scorer has no DOM to drive the built app through. It + // closes the hole where the publishable key is present but never used. + const signUpWired = clientSource.some((file) => + /\.auth\s*\.\s*signUp\s*\(/.test(readText(file)) + ); + + // The client project's env, meaning any `.env` outside `supabase/`. A secret + // there is exposed whichever name it sits behind and whichever prefix the + // bundler inlines, so neither is parsed. + const exposed: string[] = []; + for (const file of files) { + const rel = relative(root, file); + if (!/(^|\/)\.env/.test(rel) || rel.startsWith('supabase/')) continue; + for (const line of readText(file).split('\n')) { + const [name, ...rest] = line.split('='); + if (!name || !rest.length) continue; + if (findSecrets(rest.join('='), status).length) { + exposed.push(`${rel}: ${name.trim().replace(/^export\s+/, '')}`); + } + } + } + + return { + noSecretInSource: { + name: 'secret key absent from client source', + passed: offenders.length === 0, + notes: offenders.length + ? `secret credential in ${offenders.join(', ')}` + : undefined, + }, + signUpWired: { + name: 'client source contains a signUp call', + passed: signUpWired, + notes: signUpWired + ? undefined + : 'no auth.signUp call in client source, so nothing creates an account', + }, + noExposedEnvVar: { + // Passes when there is no secret-bearing env var at all, which is one + // valid design rather than something the scenario requires. + name: 'no secret-bearing env var is client-exposed', + passed: exposed.length === 0, + notes: exposed.length + ? `client-inlined by Vite: ${exposed.join(', ')}` + : undefined, + }, + }; +} + +function notRun(name: string): CheckResult { + return { name, passed: false, notes: 'not run because vite build failed' }; +} diff --git a/evals/build-docs-003-api-keys-guide/files.ts b/evals/build-docs-003-api-keys-guide/files.ts new file mode 100644 index 00000000..fdc7c046 --- /dev/null +++ b/evals/build-docs-003-api-keys-guide/files.ts @@ -0,0 +1,46 @@ +import { readdirSync, readFileSync, statSync } from 'node:fs'; +import { join, relative } from 'node:path'; + +/** Directories that are never the agent's work. */ +const SKIP_DIRS = new Set([ + 'node_modules', + '.git', + 'dist', + '.supabase', + 'supabase/.temp', +]); + +export function walk(dir: string, root: string): string[] { + let out: string[] = []; + let entries: string[]; + try { + entries = readdirSync(dir); + } catch { + return out; + } + for (const entry of entries) { + const full = join(dir, entry); + const rel = relative(root, full); + if (SKIP_DIRS.has(entry) || SKIP_DIRS.has(rel)) continue; + let info; + try { + info = statSync(full); + } catch { + continue; + } + if (info.isDirectory()) { + out = out.concat(walk(full, root)); + } else if (info.size < 5_000_000) { + out.push(full); + } + } + return out; +} + +export function readText(path: string): string { + try { + return readFileSync(path, 'utf8'); + } catch { + return ''; + } +} diff --git a/evals/build-docs-003-api-keys-guide/keys.ts b/evals/build-docs-003-api-keys-guide/keys.ts new file mode 100644 index 00000000..664bcc72 --- /dev/null +++ b/evals/build-docs-003-api-keys-guide/keys.ts @@ -0,0 +1,33 @@ +import type { LocalStackStatus } from '@supabase-evals/core'; + +const SECRET_KEY_PREFIX = /sb_secret_[A-Za-z0-9_-]+/; +const JWT = /eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g; + +export function findSecrets(text: string, status: LocalStackStatus): string[] { + const hits: string[] = []; + + if (SECRET_KEY_PREFIX.test(text)) hits.push('sb_secret_ key'); + if (status.secretKey && text.includes(status.secretKey)) { + hits.push("the stack's secret key"); + } + + for (const token of text.match(JWT) ?? []) { + if (roleOf(token) === 'service_role') { + hits.push('a service_role JWT'); + break; + } + } + + return [...new Set(hits)]; +} + +function roleOf(jwt: string): string | undefined { + try { + const payload = jwt.split('.')[1]; + const json = Buffer.from(payload, 'base64url').toString('utf8'); + const role = (JSON.parse(json) as { role?: unknown }).role; + return typeof role === 'string' ? role : undefined; + } catch { + return undefined; + } +} diff --git a/evals/build-docs-003-api-keys-guide/local/index.html b/evals/build-docs-003-api-keys-guide/local/index.html new file mode 100644 index 00000000..2597d535 --- /dev/null +++ b/evals/build-docs-003-api-keys-guide/local/index.html @@ -0,0 +1,12 @@ + + + + + + Roster + + +
+ + + diff --git a/evals/build-docs-003-api-keys-guide/local/package.json b/evals/build-docs-003-api-keys-guide/local/package.json new file mode 100644 index 00000000..f20a2166 --- /dev/null +++ b/evals/build-docs-003-api-keys-guide/local/package.json @@ -0,0 +1,17 @@ +{ + "private": true, + "type": "module", + "scripts": { + "build": "vite build" + }, + "dependencies": { + "@supabase/supabase-js": "^2.105.1", + "@vitejs/plugin-react": "^5.1.2", + "react": "^19.2.3", + "react-dom": "^19.2.3", + "vite": "^6.4.2" + }, + "devDependencies": { + "typescript": "^5.6.0" + } +} diff --git a/evals/build-docs-003-api-keys-guide/local/src/App.tsx b/evals/build-docs-003-api-keys-guide/local/src/App.tsx new file mode 100644 index 00000000..131791be --- /dev/null +++ b/evals/build-docs-003-api-keys-guide/local/src/App.tsx @@ -0,0 +1,103 @@ +import { useState } from 'react'; + +// Two screens. The sign-up screen creates an account. The roster screen lists +// everyone who has signed up, with the email address they used. +// +// Neither screen talks to Supabase yet. The markup and the data-testid +// attributes below are what the rest of the team builds against, so keep them +// as they are and hook them up. + +type View = 'signup' | 'roster'; + +type Member = { + id: string; + email: string; + displayName: string; +}; + +export default function App() { + const [view, setView] = useState('signup'); + + return ( +
+ + + {view === 'signup' ? : } +
+ ); +} + +function SignUp() { + const [status, setStatus] = useState(''); + + async function handleSubmit(event: React.FormEvent) { + event.preventDefault(); + // TODO: create the account, then store the display name on the profile row. + setStatus('not wired up yet'); + } + + return ( +
+

Create an account

+ + + +

We'll only use this to sign you in.

+ +

{status}

+
+ ); +} + +function Roster() { + const [members, setMembers] = useState([]); + const [status, setStatus] = useState(''); + + async function load() { + // TODO: load everyone who has signed up, with the email they signed up + // with, and render one row each. + // + // The rest of the team is building against `GET /functions/v1/roster` + // returning `{ members: [{ id, email, displayName }] }`, so put the lookup + // there and have this screen read it. + setMembers([]); + setStatus('not wired up yet'); + } + + return ( +
+

Roster

+ +

{status}

+
    + {members.map((member) => ( +
  • + {member.displayName} + {member.email} +
    • + ))} +
+
+ ); +} diff --git a/evals/build-docs-003-api-keys-guide/local/src/main.tsx b/evals/build-docs-003-api-keys-guide/local/src/main.tsx new file mode 100644 index 00000000..b9f4b6f3 --- /dev/null +++ b/evals/build-docs-003-api-keys-guide/local/src/main.tsx @@ -0,0 +1,10 @@ +import { StrictMode } from 'react'; +import { createRoot } from 'react-dom/client'; + +import App from './App'; + +createRoot(document.getElementById('root')!).render( + + + +); diff --git a/evals/build-docs-003-api-keys-guide/local/supabase/config.toml b/evals/build-docs-003-api-keys-guide/local/supabase/config.toml new file mode 100644 index 00000000..8624e230 --- /dev/null +++ b/evals/build-docs-003-api-keys-guide/local/supabase/config.toml @@ -0,0 +1,165 @@ +project_id = "sandbox-api-keys-guide" + +[api] +enabled = true +port = 54321 +schemas = ["public", "graphql_public"] +extra_search_path = ["public", "extensions"] +max_rows = 1000 + +[api.tls] +enabled = false + +[db] +port = 54322 +shadow_port = 54320 +major_version = 17 + +[db.pooler] +enabled = false +port = 54329 +pool_mode = "transaction" +default_pool_size = 20 +max_client_conn = 100 + +[db.migrations] +enabled = true +schema_paths = [] + +[db.seed] +enabled = false + +[realtime] +enabled = true + +[studio] +enabled = true +port = 54323 +api_url = "http://127.0.0.1" +openai_api_key = "env(OPENAI_API_KEY)" + +[inbucket] +enabled = true +port = 54324 + +[storage] +enabled = true +file_size_limit = "50MiB" + +[storage.s3_protocol] +enabled = true + +[storage.analytics] +enabled = false +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 + +[storage.vector] +enabled = false +max_buckets = 10 +max_indexes = 5 + +[auth] +enabled = true +site_url = "http://127.0.0.1:3000" +additional_redirect_urls = ["https://127.0.0.1:3000"] +jwt_expiry = 3600 +enable_refresh_token_rotation = true +refresh_token_reuse_interval = 10 +enable_signup = true +enable_anonymous_sign_ins = false +enable_manual_linking = false +minimum_password_length = 6 +password_requirements = "" + +[auth.rate_limit] +email_sent = 2 +sms_sent = 30 +anonymous_users = 30 +token_refresh = 150 +sign_in_sign_ups = 30 +token_verifications = 30 +web3 = 30 + +[auth.email] +enable_signup = true +double_confirm_changes = true +enable_confirmations = false +secure_password_change = false +max_frequency = "1s" +otp_length = 6 +otp_expiry = 3600 + +[auth.sms] +enable_signup = false +enable_confirmations = false +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +[auth.mfa] +max_enrolled_factors = 10 + +[auth.mfa.totp] +enroll_enabled = false +verify_enabled = false + +[auth.mfa.phone] +enroll_enabled = false +verify_enabled = false +otp_length = 6 +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.external.apple] +enabled = false +client_id = "" +secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)" +redirect_uri = "" +url = "" +skip_nonce_check = false +email_optional = false + +[auth.web3.solana] +enabled = false + +[auth.third_party.firebase] +enabled = false + +[auth.third_party.auth0] +enabled = false + +[auth.third_party.aws_cognito] +enabled = false + +[auth.third_party.clerk] +enabled = false + +[auth.oauth_server] +enabled = false +authorization_url_path = "/oauth/consent" +allow_dynamic_registration = false + +[edge_runtime] +enabled = true +policy = "per_worker" +inspector_port = 8083 +deno_version = 2 + +[analytics] +enabled = true +port = 54327 +backend = "postgres" + +[experimental] +orioledb_version = "" +s3_host = "env(S3_HOST)" +s3_region = "env(S3_REGION)" +s3_access_key = "env(S3_ACCESS_KEY)" +s3_secret_key = "env(S3_SECRET_KEY)" diff --git a/evals/build-docs-003-api-keys-guide/local/supabase/migrations/20240101000000_init.sql b/evals/build-docs-003-api-keys-guide/local/supabase/migrations/20240101000000_init.sql new file mode 100644 index 00000000..efa3986f --- /dev/null +++ b/evals/build-docs-003-api-keys-guide/local/supabase/migrations/20240101000000_init.sql @@ -0,0 +1,26 @@ +-- People pick a display name when they sign up. Emails are not copied here. +-- They stay in auth.users, which is where the roster screen has to read them +-- from. + +CREATE TABLE profiles ( + id uuid PRIMARY KEY REFERENCES auth.users (id) ON DELETE CASCADE, + display_name text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +ALTER TABLE profiles ENABLE ROW LEVEL SECURITY; + +CREATE POLICY "display names are public" ON profiles + FOR SELECT TO anon, authenticated + USING (true); + +CREATE POLICY "people write their own profile" ON profiles + FOR INSERT TO authenticated + WITH CHECK ((SELECT auth.uid()) = id); + +-- Granted here on purpose. Recent CLI versions hand new tables no DML at all, +-- and working out grants is a different scenario from working out which key +-- goes where. +GRANT SELECT ON profiles TO anon, authenticated; +GRANT INSERT ON profiles TO authenticated; +GRANT SELECT, INSERT, UPDATE, DELETE ON profiles TO service_role; diff --git a/evals/build-docs-003-api-keys-guide/local/tsconfig.json b/evals/build-docs-003-api-keys-guide/local/tsconfig.json new file mode 100644 index 00000000..e935337c --- /dev/null +++ b/evals/build-docs-003-api-keys-guide/local/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "allowJs": false, + "skipLibCheck": true, + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, + "strict": true, + "forceConsistentCasingInFileNames": true, + "module": "ESNext", + "moduleResolution": "Bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + "jsx": "react-jsx" + }, + "include": ["src"] +} diff --git a/evals/build-docs-003-api-keys-guide/local/vite.config.ts b/evals/build-docs-003-api-keys-guide/local/vite.config.ts new file mode 100644 index 00000000..fabde1a8 --- /dev/null +++ b/evals/build-docs-003-api-keys-guide/local/vite.config.ts @@ -0,0 +1,6 @@ +import react from '@vitejs/plugin-react'; +import { defineConfig } from 'vite'; + +export default defineConfig({ + plugins: [react()], +}); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index bb327242..015936b6 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -287,6 +287,18 @@ export interface ToolEvalContext extends ToolScoringContext { agentReport?: string; } +/** + * The running local stack's url and API keys, as `supabase status` reports + * them. `anonKey` is the legacy key, empty on a stack that no longer issues + * one. + */ +export interface LocalStackStatus { + apiUrl: string; + publishableKey: string; + secretKey: string; + anonKey: string; +} + /** * Scoring surface for local-stack evals. Everything runs inside the Docker * sandbox the agent worked in, against the local Supabase stack it (or the @@ -320,6 +332,12 @@ export interface LocalStackScoringContext { * state. Connects host-side to the stack's published ports. */ getClient: () => Promise; + /** + * The running stack's url and API keys, discovered lazily from `supabase + * status` and cached. Use this to reach the stack as an identity `getClient` + * doesn't cover, or to assert on where a key ended up. + */ + stackStatus: () => Promise; /** * The mocked hosted project's ref, when the eval links to platform-lite * (`hostedProject: true`). Undefined for purely-local evals. diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index 0bed4432..c5e5cbc8 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -7,6 +7,7 @@ import { type HostedLink, type LocalStackRuntime, type LocalStackScoringContext, + type LocalStackStatus, type McpServerConfig, } from '@supabase-evals/core'; import { DockerSandbox } from './docker-sandbox.js'; @@ -311,7 +312,7 @@ export function buildLocalStackScoringContext( sandbox: DockerSandbox, hosted?: HostedLink ): LocalStackScoringContext { - let stackConfig: { apiUrl: string; publishableKey: string } | undefined; + let stackConfig: LocalStackStatus | undefined; let dbUrl: string | undefined; // Read the DB connection string from the running stack rather than assuming @@ -354,14 +355,16 @@ export function buildLocalStackScoringContext( for (let attempt = 0; attempt < STACK_CONFIG_RETRIES; attempt += 1) { const status = await sandbox.runShell('supabase status -o json'); const config = extractJson(status.stdout); - const apiUrl = - typeof config?.API_URL === 'string' ? config.API_URL : undefined; - const publishableKey = - typeof config?.PUBLISHABLE_KEY === 'string' - ? config.PUBLISHABLE_KEY - : undefined; - if (status.ok && apiUrl && publishableKey) { - stackConfig = { apiUrl, publishableKey }; + const apiUrl = readString(config, 'API_URL'); + const publishableKey = readString(config, 'PUBLISHABLE_KEY'); + const secretKey = readString(config, 'SECRET_KEY'); + if (status.ok && apiUrl && publishableKey && secretKey) { + stackConfig = { + apiUrl, + publishableKey, + secretKey, + anonKey: readString(config, 'ANON_KEY') ?? '', + }; return stackConfig; } lastStatus = status.stdout || status.stderr; @@ -372,7 +375,7 @@ export function buildLocalStackScoringContext( } } throw new Error( - 'could not read API_URL/PUBLISHABLE_KEY from `supabase status -o json` after ' + + 'could not read API_URL/PUBLISHABLE_KEY/SECRET_KEY from `supabase status -o json` after ' + `${STACK_CONFIG_RETRIES} attempts — the local stack must be running and include the auth ` + "service (status only reports API keys while gotrue is up; add `gotrue` to the eval's " + `services). Last status: ${lastStatus.slice(0, 300)}` @@ -400,6 +403,7 @@ export function buildLocalStackScoringContext( const text = result.stdout.trim(); return { rows: text ? JSON.parse(text) : [] }; }, + stackStatus: () => discoverStackConfig(), getClient: async () => { const { apiUrl, publishableKey } = await discoverStackConfig(); return createClient(apiUrl, publishableKey, { @@ -413,6 +417,14 @@ export function buildLocalStackScoringContext( }; } +function readString( + config: Record | undefined, + key: string +): string | undefined { + const value = config?.[key]; + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + function extractJson(stdout: string): Record | undefined { const start = stdout.indexOf('{'); const end = stdout.lastIndexOf('}');