diff --git a/.gitignore b/.gitignore index 5dc75e68..abc3c725 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ dist/ .env*.local # eval seed data may include a .env with well-known local demo keys !evals/resolve-sdk-001-legacy-key-migration/local/app/.env +!evals/build-functions-007-cors-api-key-with-middleware/local/supabase/functions/.env # generated local CLI state, not eval seed data evals/*/local/supabase/.temp/ diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index c4a80fc6..e393df66 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -1707,6 +1707,350 @@ "run": 3, "sourcePath": "codex-gpt-5.6-luna/build-functions-006-dual-auth-with-server/run-3/result.json" }, + { + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-luna", + "reasoningEffort": "medium" + }, + "eval": "build-functions-007-cors-api-key-with-middleware", + "stage": "build", + "product": [ + "edge-functions" + ], + "topic": [ + "sdk", + "security" + ], + "suite": "regression", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "rejects a request with no x-api-key", + "passed": true, + "notes": "status 401: {\"error\":\"Unauthorized\"}" + }, + { + "name": "rejects a request with the wrong x-api-key", + "passed": true, + "notes": "status 401: {\"error\":\"Unauthorized\"}" + }, + { + "name": "accepts the right key and returns the caller key id from ctx", + "passed": true, + "notes": "status 200: {\"ok\":true,\"keyId\":\"7f3a9c\"}" + }, + { + "name": "varies the handler response on Origin for the allowed origin", + "passed": true, + "notes": "vary: Accept-Encoding, Origin" + }, + { + "name": "implementation uses @supabase/middleware", + "passed": true, + "notes": "imports @supabase/middleware" + }, + { + "name": "the API-key check is a defineMiddleware middleware", + "passed": true, + "notes": "calls defineMiddleware" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- files ---' && rg --files -g '\"'!node_modules'\"' -g '\"'!supabase/.branches'\"' | sort && printf '%s\\\\n' '--- cli ---' && supabase --version && supabase functions --help && printf '%s\\\\n' '--- changelog status ---' && curl -fsSL https://supabase.com/changelog.md | head -80\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 6741 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"find . -maxdepth 4 -type f -not -path './.git/*' -print | sort && printf '%s\\\\n' '--- config ---' && sed -n '1,240p' supabase/config.toml && printf '%s\\\\n' '--- env files ---' && find supabase/functions -maxdepth 2 -type f -print -exec sh -c 'echo --- \\\"\"'$1\"; sed -n \"1,160p\" \"$1\"'\"' _ {} \\\\; 2>/dev/null || true && printf '%s\\\\n' '--- middleware docs search ---' && curl -fsSL https://supabase.com/docs/guides/functions/middleware | rg -n -C 3 'pipeline|defineMiddleware|cors' | head -160 || true\"", + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/middleware" + } + ], + "resultChars": 49 + }, + { + "source": "web_search", + "query": "site:supabase.com/docs @supabase/middleware defineMiddleware pipeline Edge Functions CORS ...", + "pages": [] + } + ] + }, + "usage": [ + { + "model": "gpt-5.6-luna", + "inputTokens": 3450781, + "cacheReadInputTokens": 3352184, + "cacheWriteInputTokens": 98438, + "outputTokens": 11227 + } + ], + "stepCount": 53, + "toolCallCount": 46, + "durationMs": 250071, + "prompt": "Build and serve a Supabase Edge Function named `notes-api` for this project,\nreachable over HTTP at `/functions/v1/notes-api`.\n\nPut it together with the **`@supabase/middleware`** package. Import it directly\nin your function:\n\n```ts\nimport { pipeline } from \"npm:@supabase/middleware\";\n```\n\nTwo things need to run in front of the handler:\n\n1. **CORS** for our web app at `https://app.example.com`. Browsers send\n preflight requests first, so those have to work too.\n\n2. **An API key check.** The callers are third-party services, not signed-in\n Supabase users. They send their key in an `x-api-key` header, and it has to\n match the `NOTES_API_KEY` secret that is already in\n `supabase/functions/.env`. Anything else gets a `401` and never reaches the\n handler.\n\nWrite the key check as your own middleware with the package's\n`defineMiddleware`. Our keys look like `nk_live_7f3a9c`; have the middleware put\nthe part after the last underscore on `ctx` as `ctx.caller.keyId`, and have the\nhandler return `{ \"ok\": true, \"keyId\": ctx.caller.keyId }` as JSON.\n\nGet the local stack running so the function is reachable at the path above.", + "promptSourcePath": "evals/build-functions-007-cors-api-key-with-middleware/PROMPT.md", + "run": 1, + "sourcePath": "codex-gpt-5.6-luna/build-functions-007-cors-api-key-with-middleware/run-1/result.json" + }, + { + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-luna", + "reasoningEffort": "medium" + }, + "eval": "build-functions-007-cors-api-key-with-middleware", + "stage": "build", + "product": [ + "edge-functions" + ], + "topic": [ + "sdk", + "security" + ], + "suite": "regression", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "rejects a request with no x-api-key", + "passed": true, + "notes": "status 401: {\"error\":\"Unauthorized\"}" + }, + { + "name": "rejects a request with the wrong x-api-key", + "passed": true, + "notes": "status 401: {\"error\":\"Unauthorized\"}" + }, + { + "name": "accepts the right key and returns the caller key id from ctx", + "passed": true, + "notes": "status 200: {\"ok\":true,\"keyId\":\"7f3a9c\"}" + }, + { + "name": "varies the handler response on Origin for the allowed origin", + "passed": true, + "notes": "vary: Accept-Encoding, Origin" + }, + { + "name": "implementation uses @supabase/middleware", + "passed": true, + "notes": "imports @supabase/middleware" + }, + { + "name": "the API-key check is a defineMiddleware middleware", + "passed": true, + "notes": "calls defineMiddleware" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '---CHANGELOG CHECK---'; curl -fsSL --max-time 15 https://supabase.com/changelog.md | rg -i -m 5 'breaking-change|edge function|middleware' || true; printf '%s\\\\n' '---PROJECT---'; find . -maxdepth 4 -type f -not -path './.git/*' -not -path './.claude/*' | sort; printf '%s\\\\n' '---CONFIG---'; sed -n '1,260p' supabase/config.toml; printf '%s\\\\n' '---CLI---'; supabase --version; supabase functions --help\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 2281 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '---ENV (names only)---'; sed 's/=.*/=/' supabase/functions/.env; printf '%s\\\\n' '---STATUS---'; git status --short; printf '%s\\\\n' '---PACKAGE METADATA---'; curl -fsSL --max-time 20 https://registry.npmjs.org/@supabase%2fmiddleware | node -e 'let s=\\\"\\\";process.stdin.on(\\\"data\\\",d=>s+=d).on(\\\"end\\\",()=>{let j=JSON.parse(s); let v=j[\\\"dist-tags\\\"].latest; console.log(JSON.stringify({latest:v,version:j.versions[v]},null,2))})'; printf '%s\\\\n' '---DOC SEARCH---'; curl -fsSL --max-time 20 https://supabase.com/docs/guides/functions/middleware | rg -n -i -C 2 'pipeline|defineMiddleware|cors' | head -120 || true\"", + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/middleware" + } + ], + "resultChars": 49 + }, + { + "source": "web_search", + "query": "\"failed to determine entrypoint\" Supabase Edge Runtime functions serve", + "pages": [] + } + ] + }, + "usage": [ + { + "model": "gpt-5.6-luna", + "inputTokens": 2893780, + "cacheReadInputTokens": 2780411, + "cacheWriteInputTokens": 113252, + "outputTokens": 10370 + } + ], + "stepCount": 39, + "toolCallCount": 30, + "durationMs": 207756, + "prompt": "Build and serve a Supabase Edge Function named `notes-api` for this project,\nreachable over HTTP at `/functions/v1/notes-api`.\n\nPut it together with the **`@supabase/middleware`** package. Import it directly\nin your function:\n\n```ts\nimport { pipeline } from \"npm:@supabase/middleware\";\n```\n\nTwo things need to run in front of the handler:\n\n1. **CORS** for our web app at `https://app.example.com`. Browsers send\n preflight requests first, so those have to work too.\n\n2. **An API key check.** The callers are third-party services, not signed-in\n Supabase users. They send their key in an `x-api-key` header, and it has to\n match the `NOTES_API_KEY` secret that is already in\n `supabase/functions/.env`. Anything else gets a `401` and never reaches the\n handler.\n\nWrite the key check as your own middleware with the package's\n`defineMiddleware`. Our keys look like `nk_live_7f3a9c`; have the middleware put\nthe part after the last underscore on `ctx` as `ctx.caller.keyId`, and have the\nhandler return `{ \"ok\": true, \"keyId\": ctx.caller.keyId }` as JSON.\n\nGet the local stack running so the function is reachable at the path above.", + "promptSourcePath": "evals/build-functions-007-cors-api-key-with-middleware/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-5.6-luna/build-functions-007-cors-api-key-with-middleware/run-2/result.json" + }, + { + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-luna", + "reasoningEffort": "medium" + }, + "eval": "build-functions-007-cors-api-key-with-middleware", + "stage": "build", + "product": [ + "edge-functions" + ], + "topic": [ + "sdk", + "security" + ], + "suite": "regression", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "rejects a request with no x-api-key", + "passed": true, + "notes": "status 401: {\"error\":\"Unauthorized\"}" + }, + { + "name": "rejects a request with the wrong x-api-key", + "passed": true, + "notes": "status 401: {\"error\":\"Unauthorized\"}" + }, + { + "name": "accepts the right key and returns the caller key id from ctx", + "passed": true, + "notes": "status 200: {\"ok\":true,\"keyId\":\"7f3a9c\"}" + }, + { + "name": "varies the handler response on Origin for the allowed origin", + "passed": true, + "notes": "vary: Accept-Encoding, Origin" + }, + { + "name": "implementation uses @supabase/middleware", + "passed": true, + "notes": "imports @supabase/middleware" + }, + { + "name": "the API-key check is a defineMiddleware middleware", + "passed": true, + "notes": "calls defineMiddleware" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -120'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 6742 + }, + { + "source": "web_search", + "query": "site:supabase.com/docs @supabase/middleware defineMiddleware pipeline Edge Functions", + "pages": [] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/reference/middleware withCors allowedOrigins @supabase/middleware/cors", + "pages": [] + }, + { + "source": "web_search", + "query": "https://supabase.com/docs/reference/middleware/cors-withcors", + "pages": [ + { + "url": "https://supabase.com/docs/reference/middleware/cors-withcors" + } + ] + }, + { + "source": "web_search", + "query": "\"withCors\" \"allowedOrigins\" \"@supabase/middleware\"", + "pages": [] + }, + { + "source": "web_search", + "query": "Supabase CLI \"failed to determine entrypoint\" edge runtime functions serve", + "pages": [] + } + ] + }, + "usage": [ + { + "model": "gpt-5.6-luna", + "inputTokens": 3446294, + "cacheReadInputTokens": 3344238, + "cacheWriteInputTokens": 101894, + "outputTokens": 13458 + } + ], + "stepCount": 54, + "toolCallCount": 49, + "durationMs": 240010, + "prompt": "Build and serve a Supabase Edge Function named `notes-api` for this project,\nreachable over HTTP at `/functions/v1/notes-api`.\n\nPut it together with the **`@supabase/middleware`** package. Import it directly\nin your function:\n\n```ts\nimport { pipeline } from \"npm:@supabase/middleware\";\n```\n\nTwo things need to run in front of the handler:\n\n1. **CORS** for our web app at `https://app.example.com`. Browsers send\n preflight requests first, so those have to work too.\n\n2. **An API key check.** The callers are third-party services, not signed-in\n Supabase users. They send their key in an `x-api-key` header, and it has to\n match the `NOTES_API_KEY` secret that is already in\n `supabase/functions/.env`. Anything else gets a `401` and never reaches the\n handler.\n\nWrite the key check as your own middleware with the package's\n`defineMiddleware`. Our keys look like `nk_live_7f3a9c`; have the middleware put\nthe part after the last underscore on `ctx` as `ctx.caller.keyId`, and have the\nhandler return `{ \"ok\": true, \"keyId\": ctx.caller.keyId }` as JSON.\n\nGet the local stack running so the function is reachable at the path above.", + "promptSourcePath": "evals/build-functions-007-cors-api-key-with-middleware/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-5.6-luna/build-functions-007-cors-api-key-with-middleware/run-3/result.json" + }, { "experiment": "codex-gpt-5.6-luna", "experimentSuite": "regression", @@ -5989,6 +6333,268 @@ "run": 3, "sourcePath": "codex-gpt-5.6-luna-no-skills/build-functions-006-dual-auth-with-server/run-3/result.json" }, + { + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-luna", + "reasoningEffort": "medium" + }, + "eval": "build-functions-007-cors-api-key-with-middleware", + "stage": "build", + "product": [ + "edge-functions" + ], + "topic": [ + "sdk", + "security" + ], + "suite": "regression", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "rejects a request with no x-api-key", + "passed": true, + "notes": "status 401: {\"error\":\"Unauthorized\"}" + }, + { + "name": "rejects a request with the wrong x-api-key", + "passed": true, + "notes": "status 401: {\"error\":\"Unauthorized\"}" + }, + { + "name": "accepts the right key and returns the caller key id from ctx", + "passed": true, + "notes": "status 200: {\"ok\":true,\"keyId\":\"7f3a9c\"}" + }, + { + "name": "varies the handler response on Origin for the allowed origin", + "passed": true, + "notes": "vary: Accept-Encoding, Origin" + }, + { + "name": "implementation uses @supabase/middleware", + "passed": true, + "notes": "imports @supabase/middleware" + }, + { + "name": "the API-key check is a defineMiddleware middleware", + "passed": true, + "notes": "calls defineMiddleware" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "site:supabase.com/docs @supabase/middleware defineMiddleware pipeline Edge Functions", + "pages": [] + }, + { + "source": "web_search", + "query": "\"failed to determine entrypoint\" \"supabase functions serve\"", + "pages": [] + } + ] + }, + "usage": [ + { + "model": "gpt-5.6-luna", + "inputTokens": 4337813, + "cacheReadInputTokens": 4245099, + "cacheWriteInputTokens": 92495, + "outputTokens": 12470 + } + ], + "stepCount": 73, + "toolCallCount": 48, + "durationMs": 304160, + "prompt": "Build and serve a Supabase Edge Function named `notes-api` for this project,\nreachable over HTTP at `/functions/v1/notes-api`.\n\nPut it together with the **`@supabase/middleware`** package. Import it directly\nin your function:\n\n```ts\nimport { pipeline } from \"npm:@supabase/middleware\";\n```\n\nTwo things need to run in front of the handler:\n\n1. **CORS** for our web app at `https://app.example.com`. Browsers send\n preflight requests first, so those have to work too.\n\n2. **An API key check.** The callers are third-party services, not signed-in\n Supabase users. They send their key in an `x-api-key` header, and it has to\n match the `NOTES_API_KEY` secret that is already in\n `supabase/functions/.env`. Anything else gets a `401` and never reaches the\n handler.\n\nWrite the key check as your own middleware with the package's\n`defineMiddleware`. Our keys look like `nk_live_7f3a9c`; have the middleware put\nthe part after the last underscore on `ctx` as `ctx.caller.keyId`, and have the\nhandler return `{ \"ok\": true, \"keyId\": ctx.caller.keyId }` as JSON.\n\nGet the local stack running so the function is reachable at the path above.", + "promptSourcePath": "evals/build-functions-007-cors-api-key-with-middleware/PROMPT.md", + "run": 1, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-functions-007-cors-api-key-with-middleware/run-1/result.json" + }, + { + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-luna", + "reasoningEffort": "medium" + }, + "eval": "build-functions-007-cors-api-key-with-middleware", + "stage": "build", + "product": [ + "edge-functions" + ], + "topic": [ + "sdk", + "security" + ], + "suite": "regression", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "read stack config from `supabase status`", + "passed": false, + "notes": "missing API_URL; got keys: ANON_KEY, DB_URL, INBUCKET_URL, JWT_SECRET, MAILPIT_URL, PUBLISHABLE_KEY, S3_PROTOCOL_ACCESS_KEY_ID, S3_PROTOCOL_ACCESS_KEY_SECRET, S3_PROTOCOL_REGION, SECRET_KEY, SERVICE_ROLE_KEY, STORAGE_S3_URL, STUDIO_URL" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "site:supabase.com/docs @supabase/middleware defineMiddleware pipeline cors Edge Functions", + "pages": [] + }, + { + "source": "web_search", + "query": "\"failed to determine entrypoint\" supabase functions serve", + "pages": [] + }, + { + "source": "web_search", + "query": "site:github.com/supabase/cli \"functions/main/index.ts\" \"functions serve\"", + "pages": [] + }, + { + "source": "web_search", + "query": "github supabase edge-runtime examples main index.ts userWorkers.create servicePath", + "pages": [] + } + ] + }, + "usage": [ + { + "model": "gpt-5.6-luna", + "inputTokens": 3260356, + "cacheReadInputTokens": 3166105, + "cacheWriteInputTokens": 94086, + "outputTokens": 11680 + } + ], + "stepCount": 55, + "toolCallCount": 40, + "durationMs": 236891, + "prompt": "Build and serve a Supabase Edge Function named `notes-api` for this project,\nreachable over HTTP at `/functions/v1/notes-api`.\n\nPut it together with the **`@supabase/middleware`** package. Import it directly\nin your function:\n\n```ts\nimport { pipeline } from \"npm:@supabase/middleware\";\n```\n\nTwo things need to run in front of the handler:\n\n1. **CORS** for our web app at `https://app.example.com`. Browsers send\n preflight requests first, so those have to work too.\n\n2. **An API key check.** The callers are third-party services, not signed-in\n Supabase users. They send their key in an `x-api-key` header, and it has to\n match the `NOTES_API_KEY` secret that is already in\n `supabase/functions/.env`. Anything else gets a `401` and never reaches the\n handler.\n\nWrite the key check as your own middleware with the package's\n`defineMiddleware`. Our keys look like `nk_live_7f3a9c`; have the middleware put\nthe part after the last underscore on `ctx` as `ctx.caller.keyId`, and have the\nhandler return `{ \"ok\": true, \"keyId\": ctx.caller.keyId }` as JSON.\n\nGet the local stack running so the function is reachable at the path above.", + "promptSourcePath": "evals/build-functions-007-cors-api-key-with-middleware/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-functions-007-cors-api-key-with-middleware/run-2/result.json" + }, + { + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-luna", + "reasoningEffort": "medium" + }, + "eval": "build-functions-007-cors-api-key-with-middleware", + "stage": "build", + "product": [ + "edge-functions" + ], + "topic": [ + "sdk", + "security" + ], + "suite": "regression", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "rejects a request with no x-api-key", + "passed": true, + "notes": "status 401: {\"error\":\"Unauthorized\"}" + }, + { + "name": "rejects a request with the wrong x-api-key", + "passed": true, + "notes": "status 401: {\"error\":\"Unauthorized\"}" + }, + { + "name": "accepts the right key and returns the caller key id from ctx", + "passed": true, + "notes": "status 200: {\"ok\":true,\"keyId\":\"7f3a9c\"}" + }, + { + "name": "varies the handler response on Origin for the allowed origin", + "passed": true, + "notes": "vary: Accept-Encoding, Origin" + }, + { + "name": "implementation uses @supabase/middleware", + "passed": true, + "notes": "imports @supabase/middleware" + }, + { + "name": "the API-key check is a defineMiddleware middleware", + "passed": true, + "notes": "calls defineMiddleware" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "site:supabase.com/docs @supabase/middleware defineMiddleware pipeline Edge Functions", + "pages": [] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/reference/middleware withCors @supabase/middleware/cors", + "pages": [] + }, + { + "source": "web_search", + "query": "\"failed to determine entrypoint\" supabase functions serve", + "pages": [] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/guides/functions \"export default {\" \"fetch\" Edge Function", + "pages": [] + } + ] + }, + "usage": [ + { + "model": "gpt-5.6-luna", + "inputTokens": 2751553, + "cacheReadInputTokens": 2652093, + "cacheWriteInputTokens": 99337, + "outputTokens": 7942 + } + ], + "stepCount": 41, + "toolCallCount": 29, + "durationMs": 183790, + "prompt": "Build and serve a Supabase Edge Function named `notes-api` for this project,\nreachable over HTTP at `/functions/v1/notes-api`.\n\nPut it together with the **`@supabase/middleware`** package. Import it directly\nin your function:\n\n```ts\nimport { pipeline } from \"npm:@supabase/middleware\";\n```\n\nTwo things need to run in front of the handler:\n\n1. **CORS** for our web app at `https://app.example.com`. Browsers send\n preflight requests first, so those have to work too.\n\n2. **An API key check.** The callers are third-party services, not signed-in\n Supabase users. They send their key in an `x-api-key` header, and it has to\n match the `NOTES_API_KEY` secret that is already in\n `supabase/functions/.env`. Anything else gets a `401` and never reaches the\n handler.\n\nWrite the key check as your own middleware with the package's\n`defineMiddleware`. Our keys look like `nk_live_7f3a9c`; have the middleware put\nthe part after the last underscore on `ctx` as `ctx.caller.keyId`, and have the\nhandler return `{ \"ok\": true, \"keyId\": ctx.caller.keyId }` as JSON.\n\nGet the local stack running so the function is reachable at the path above.", + "promptSourcePath": "evals/build-functions-007-cors-api-key-with-middleware/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-functions-007-cors-api-key-with-middleware/run-3/result.json" + }, { "experiment": "codex-gpt-5.6-luna-no-skills", "experimentSuite": "regression", diff --git a/evals/build-functions-007-cors-api-key-with-middleware/EVAL.ts b/evals/build-functions-007-cors-api-key-with-middleware/EVAL.ts new file mode 100644 index 00000000..d972ed82 --- /dev/null +++ b/evals/build-functions-007-cors-api-key-with-middleware/EVAL.ts @@ -0,0 +1,260 @@ +import { posix } from 'node:path'; +import { + type CheckResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; + +// Regression eval for @supabase/middleware: the prompt names the package and +// asks for one built-in (CORS) plus one hand-written middleware (an API-key +// gate) composed in front of a handler. It guards the usability property — +// "when pointed at the package, agents compose with it correctly" — as a +// canary for package or docs changes that break agent usage. The "uses +// @supabase/middleware" and "uses defineMiddleware" checks are GATING. +// +// Requests go through the local Kong gateway, which owns CORS on +// /functions/v1/*; see README.md for what that leaves observable. +const FUNCTION = 'notes-api'; +const ORIGIN = 'https://app.example.com'; +// Mirrors local/supabase/functions/.env, which the agent is told about. +const API_KEY = 'nk_live_7f3a9c'; +const KEY_ID = '7f3a9c'; + +interface InvokeResult { + status: number; + ok: boolean; + body: string; + headers: Headers; +} + +const scorer: LocalStackScorer = async (ctx) => { + const checks: CheckResult[] = []; + try { + const status = await readStatus(ctx); + const apiUrl = str(status.API_URL); + if (!apiUrl) { + return fail( + 'read stack config from `supabase status`', + `missing API_URL; got keys: ${Object.keys(status).join(', ')}` + ); + } + const url = `${apiUrl}/functions/v1/${FUNCTION}`; + + // 1. No key → 401 from the gate; the handler's body never appears. Kong + // routes /functions/v1/* without an apikey, so the request arrives the + // way a third-party caller's would. + const noKey = await invoke(url, 'GET', {}); + checks.push({ + name: 'rejects a request with no x-api-key', + passed: noKey.status === 401 && !noKey.body.includes(KEY_ID), + notes: `status ${noKey.status}: ${preview(noKey.body)}`, + }); + + // 2. Wrong key → 401 as well. + const wrongKey = await invoke(url, 'GET', { + 'x-api-key': 'nk_live_000000', + }); + checks.push({ + name: 'rejects a request with the wrong x-api-key', + passed: wrongKey.status === 401 && !wrongKey.body.includes(KEY_ID), + notes: `status ${wrongKey.status}: ${preview(wrongKey.body)}`, + }); + + // 3. Right key → 200, and the handler reads what the gate contributed. + const rightKey = await invoke(url, 'GET', { + origin: ORIGIN, + 'x-api-key': API_KEY, + }); + const parsed = parseJson(rightKey.body); + checks.push({ + name: 'accepts the right key and returns the caller key id from ctx', + passed: + rightKey.status === 200 && + parsed?.ok === true && + parsed?.keyId === KEY_ID, + notes: `status ${rightKey.status}: ${preview(rightKey.body)}`, + }); + + // 4. The function's CORS ran for the allowed origin. Kong's own CORS + // plugin answers preflights and rewrites Access-Control-Allow-Origin to + // `*` on every response, so neither is observable here. A CORS middleware + // that resolved a specific origin appends `Vary: Origin`, and Kong leaves + // that alone. + const vary = rightKey.headers.get('vary') ?? ''; + checks.push({ + name: 'varies the handler response on Origin for the allowed origin', + passed: /\bOrigin\b/i.test(vary), + notes: `vary: ${vary || '(none)'}`, + }); + + // 5 + 6. GATING: built with @supabase/middleware, and the key check is a + // real middleware rather than an `if` inside the handler. + checks.push(...(await sourceChecks(ctx))); + + return { passed: checks.every((c) => c.passed), checks }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + checks.push({ + name: 'scorer completed without errors', + passed: false, + notes: msg, + }); + return { passed: false, checks }; + } +}; + +export default scorer; + +function str(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function preview(body: string): string { + return body.replace(/\s+/g, ' ').slice(0, 160); +} + +function parseJson(body: string): Record | undefined { + try { + const value: unknown = JSON.parse(body); + return value && typeof value === 'object' + ? (value as Record) + : undefined; + } catch { + return undefined; + } +} + +function fail( + name: string, + notes: string +): { passed: false; checks: CheckResult[] } { + return { passed: false, checks: [{ name, passed: false, notes }] }; +} + +async function invoke( + url: string, + method: string, + headers: Record +): Promise { + const res = await fetch(url, { method, headers }); + return { + status: res.status, + ok: res.ok, + body: await res.text(), + headers: res.headers, + }; +} + +/** Parse `supabase status -o json` for the stack's URL and keys. */ +async function readStatus( + ctx: LocalStackEvalContext +): Promise> { + const res = await ctx.exec('supabase status -o json'); + const start = res.stdout.indexOf('{'); + const end = res.stdout.lastIndexOf('}'); + if (start === -1 || end <= start) { + throw new Error( + `could not read \`supabase status\`: ${res.stderr || res.stdout}` + ); + } + return JSON.parse(res.stdout.slice(start, end + 1)); +} + +/** + * GATING: the function must import @supabase/middleware and declare its own + * middleware with `defineMiddleware`. A hand-rolled CORS + `if (key !== …)` + * handler fails this eval even if it behaves correctly. The prompt doesn't + * dictate a file layout, so this follows relative imports from the + * entrypoint (e.g. a `_shared` helper) — the required calls can live in + * either file. + */ +async function sourceChecks( + ctx: LocalStackEvalContext +): Promise { + const candidates = [ + `supabase/functions/${FUNCTION}/index.ts`, + `supabase/functions/${FUNCTION}/index.tsx`, + ]; + for (const path of candidates) { + if (await ctx.fileExists(path)) { + const src = await readModuleTree(ctx, path); + const importsPackage = + /(?:from|import)\s*\(?\s*['"](?:npm:|jsr:)?@supabase\/middleware(?:@[^'"/]+)?(?:\/[^'"]*)?['"]/.test( + src + ); + const definesMiddleware = /\bdefineMiddleware\s*[<(]/.test(src); + return [ + { + name: 'implementation uses @supabase/middleware', + passed: importsPackage, + notes: importsPackage + ? 'imports @supabase/middleware' + : 'hand-rolled — this eval requires @supabase/middleware', + }, + { + name: 'the API-key check is a defineMiddleware middleware', + passed: definesMiddleware, + notes: definesMiddleware + ? 'calls defineMiddleware' + : 'no defineMiddleware call in the function source or its local imports', + }, + ]; + } + } + return [ + { + name: 'implementation uses @supabase/middleware', + passed: false, + notes: 'could not locate function source to inspect', + }, + ]; +} + +const LOCAL_IMPORT = /(?:from|import)\s*\(?\s*['"](\.\.?\/[^'"]*)['"]/g; + +/** + * Concatenates `entryPath`'s source with every module it locally imports + * (transitively, up to a generous cap), so the source checks see across a + * `_shared` helper boundary rather than just the entrypoint file. + */ +async function readModuleTree( + ctx: LocalStackEvalContext, + entryPath: string +): Promise { + const seen = new Set(); + const queue = [entryPath]; + const chunks: string[] = []; + + while (queue.length > 0 && seen.size < 20) { + const next = queue.shift(); + if (!next) continue; + const resolved = await resolveModulePath(ctx, next); + if (!resolved || seen.has(resolved)) continue; + seen.add(resolved); + + const src = await ctx.readFile(resolved).catch(() => ''); + chunks.push(src); + + const dir = posix.dirname(resolved); + for (const match of src.matchAll(LOCAL_IMPORT)) { + queue.push(posix.join(dir, match[1])); + } + } + + return chunks.join('\n'); +} + +/** Tries `path` as given, then with common extensions, then as a directory index. */ +async function resolveModulePath( + ctx: LocalStackEvalContext, + path: string +): Promise { + const withExtensions = ['.ts', '.tsx', '.js', '.mjs'].flatMap((ext) => [ + `${path}${ext}`, + `${path}/index${ext}`, + ]); + for (const candidate of [path, ...withExtensions]) { + if (await ctx.fileExists(candidate)) return candidate; + } + return undefined; +} diff --git a/evals/build-functions-007-cors-api-key-with-middleware/PROMPT.md b/evals/build-functions-007-cors-api-key-with-middleware/PROMPT.md new file mode 100644 index 00000000..2864b923 --- /dev/null +++ b/evals/build-functions-007-cors-api-key-with-middleware/PROMPT.md @@ -0,0 +1,44 @@ +--- +stage: build +suite: regression +interface: cli +cliVersion: 2.109.1 +product: + - edge-functions +topic: + - sdk + - security +services: + - kong + - edge-runtime +projectRunning: false +motivation: SDK-1420 +--- + +Build and serve a Supabase Edge Function named `notes-api` for this project, +reachable over HTTP at `/functions/v1/notes-api`. + +Put it together with the **`@supabase/middleware`** package. Import it directly +in your function: + +```ts +import { pipeline } from "npm:@supabase/middleware"; +``` + +Two things need to run in front of the handler: + +1. **CORS** for our web app at `https://app.example.com`. Browsers send + preflight requests first, so those have to work too. + +2. **An API key check.** The callers are third-party services, not signed-in + Supabase users. They send their key in an `x-api-key` header, and it has to + match the `NOTES_API_KEY` secret that is already in + `supabase/functions/.env`. Anything else gets a `401` and never reaches the + handler. + +Write the key check as your own middleware with the package's +`defineMiddleware`. Our keys look like `nk_live_7f3a9c`; have the middleware put +the part after the last underscore on `ctx` as `ctx.caller.keyId`, and have the +handler return `{ "ok": true, "keyId": ctx.caller.keyId }` as JSON. + +Get the local stack running so the function is reachable at the path above. diff --git a/evals/build-functions-007-cors-api-key-with-middleware/README.md b/evals/build-functions-007-cors-api-key-with-middleware/README.md new file mode 100644 index 00000000..f54f6bba --- /dev/null +++ b/evals/build-functions-007-cors-api-key-with-middleware/README.md @@ -0,0 +1,32 @@ +# build-functions-007-cors-api-key-with-middleware + +Regression sibling of the `@supabase/server` pair (005/006), for +`@supabase/middleware`. The prompt names the package and asks for an Edge +Function that composes the built-in CORS middleware with a hand-written API-key +gate, so the eval measures "agents compose with the package correctly when +pointed at it", not discovery. + +## Seed + +- `local/supabase/config.toml`: a stock project with the edge runtime on Deno 2. + `verify_jwt` is left at its default. The callers in the prompt send no + Supabase JWT, so the agent has to turn platform JWT verification off for + this function or every request dies at the gateway before the middleware + runs. +- `local/supabase/functions/.env`: `NOTES_API_KEY=nk_live_7f3a9c`. The prompt + says the secret is already there; the scorer sends the same value. + +## Scorer + +Behavioral checks hit the served function through Kong: requests with no key +and with the wrong key (both `401`), and one with the right key (`200`, +`{ ok: true, keyId: "7f3a9c" }`). Two gating source checks require an +`@supabase/middleware` import and a `defineMiddleware` call, so a hand-rolled +solution fails even if it behaves. + +The local Kong (2.8.x) runs its own CORS plugin on `/functions/v1/*`: it +answers `OPTIONS` preflights before they reach the function and rewrites +`Access-Control-Allow-Origin` to `*` on every response. Neither can be +asserted through the gateway. What does survive is `Vary: Origin`, which a +CORS middleware appends when it resolves a specific allowed origin, so the +scorer checks that header on the successful request instead. diff --git a/evals/build-functions-007-cors-api-key-with-middleware/local/supabase/config.toml b/evals/build-functions-007-cors-api-key-with-middleware/local/supabase/config.toml new file mode 100644 index 00000000..33346a31 --- /dev/null +++ b/evals/build-functions-007-cors-api-key-with-middleware/local/supabase/config.toml @@ -0,0 +1,165 @@ +project_id = "sandbox-cors-api-key-middleware" + +[api] +enabled = true +port = 54321 +schemas = ["public", "graphql_public"] +extra_search_path = ["public", "extensions"] +max_rows = 1000 + +[api.tls] +enabled = false + +[db] +port = 54322 +shadow_port = 54320 +major_version = 17 + +[db.pooler] +enabled = false +port = 54329 +pool_mode = "transaction" +default_pool_size = 20 +max_client_conn = 100 + +[db.migrations] +enabled = true +schema_paths = [] + +[db.seed] +enabled = false + +[realtime] +enabled = true + +[studio] +enabled = true +port = 54323 +api_url = "http://127.0.0.1" +openai_api_key = "env(OPENAI_API_KEY)" + +[inbucket] +enabled = true +port = 54324 + +[storage] +enabled = true +file_size_limit = "50MiB" + +[storage.s3_protocol] +enabled = true + +[storage.analytics] +enabled = false +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 + +[storage.vector] +enabled = false +max_buckets = 10 +max_indexes = 5 + +[auth] +enabled = true +site_url = "http://127.0.0.1:3000" +additional_redirect_urls = ["https://127.0.0.1:3000"] +jwt_expiry = 3600 +enable_refresh_token_rotation = true +refresh_token_reuse_interval = 10 +enable_signup = true +enable_anonymous_sign_ins = false +enable_manual_linking = false +minimum_password_length = 6 +password_requirements = "" + +[auth.rate_limit] +email_sent = 2 +sms_sent = 30 +anonymous_users = 30 +token_refresh = 150 +sign_in_sign_ups = 30 +token_verifications = 30 +web3 = 30 + +[auth.email] +enable_signup = true +double_confirm_changes = true +enable_confirmations = false +secure_password_change = false +max_frequency = "1s" +otp_length = 6 +otp_expiry = 3600 + +[auth.sms] +enable_signup = false +enable_confirmations = false +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +[auth.mfa] +max_enrolled_factors = 10 + +[auth.mfa.totp] +enroll_enabled = false +verify_enabled = false + +[auth.mfa.phone] +enroll_enabled = false +verify_enabled = false +otp_length = 6 +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.external.apple] +enabled = false +client_id = "" +secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)" +redirect_uri = "" +url = "" +skip_nonce_check = false +email_optional = false + +[auth.web3.solana] +enabled = false + +[auth.third_party.firebase] +enabled = false + +[auth.third_party.auth0] +enabled = false + +[auth.third_party.aws_cognito] +enabled = false + +[auth.third_party.clerk] +enabled = false + +[auth.oauth_server] +enabled = false +authorization_url_path = "/oauth/consent" +allow_dynamic_registration = false + +[edge_runtime] +enabled = true +policy = "per_worker" +inspector_port = 8083 +deno_version = 2 + +[analytics] +enabled = true +port = 54327 +backend = "postgres" + +[experimental] +orioledb_version = "" +s3_host = "env(S3_HOST)" +s3_region = "env(S3_REGION)" +s3_access_key = "env(S3_ACCESS_KEY)" +s3_secret_key = "env(S3_SECRET_KEY)" diff --git a/evals/build-functions-007-cors-api-key-with-middleware/local/supabase/functions/.env b/evals/build-functions-007-cors-api-key-with-middleware/local/supabase/functions/.env new file mode 100644 index 00000000..5a30f968 --- /dev/null +++ b/evals/build-functions-007-cors-api-key-with-middleware/local/supabase/functions/.env @@ -0,0 +1 @@ +NOTES_API_KEY=nk_live_7f3a9c