diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 759b30d0..16db1625 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -31,7 +31,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 6b7c5078-be1d-46c0-9534-c55b5068c5ff, signUp returned {\"userId\":\"6b7c5078-be1d-46c0-9534-c55b5068c5ff\"}" + "notes": "db user 6cb72215-6e1f-427d-b917-16258a534d6a, signUp returned {\"userId\":\"6cb72215-6e1f-427d-b917-16258a534d6a\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -46,7 +46,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"6b7c5078-be1d-46c0-9534-c55b5068c5ff\"}" + "notes": "{\"userId\":\"6cb72215-6e1f-427d-b917-16258a534d6a\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -75,65 +75,43 @@ }, "docs": { "calls": [ - { - "source": "web_fetch", - "query": "List any entries tagged breaking-change, and anything relating to supabase-js client library, auth signUp / signInWithPassword / getUser / getClaims, or publishable API keys. Include dates and links.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3460 - }, { "source": "search_docs", - "query": "{ searchDocs(query: \"signUp email password user metadata display name javascript\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"signUp email password options data user metadata display name\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" }, { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" + "url": "https://supabase.com/docs/guides/platform/sso/testing-best-practices", + "title": "SSO Testing and Best Practices" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" + "url": "https://supabase.com/docs/guides/auth/auth-mfa", + "title": "Multi-Factor Authentication" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/reference/csharp/auth-signinwithpassword", - "title": "SignIn(email, password)" + "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", + "title": "Passwordless email sign-in" } ], - "resultChars": 22849 + "resultChars": 67576 }, { - "source": "search_docs", - "query": "{ searchDocs(query: \"getClaims getUser getSession which to use javascript current user\", limit: 4) { nodes { title href content } } }", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | grep -i -n \"breaking\" | head -20", "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", - "title": "Build a User Management App with React" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getsession" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", - "title": "Build a User Management App with Angular" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 100489 + "resultChars": 3727 } ] }, @@ -174,7 +152,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 185e19e1-b705-470f-9021-213eb7236821, signUp returned {\"userId\":\"185e19e1-b705-470f-9021-213eb7236821\"}" + "notes": "db user 477f8ee5-3f8e-4fca-bff5-9ef4d7b32cf6, signUp returned {\"userId\":\"477f8ee5-3f8e-4fca-bff5-9ef4d7b32cf6\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -189,7 +167,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"185e19e1-b705-470f-9021-213eb7236821\"}" + "notes": "{\"userId\":\"477f8ee5-3f8e-4fca-bff5-9ef4d7b32cf6\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -217,7 +195,75 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | head -120", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 6691 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"signUp email password user metadata display_name options data\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" + }, + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", + "title": "Passwordless email sign-in" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signup" + }, + { + "url": "https://supabase.com/docs/reference/csharp/auth-signinwithpassword", + "title": "SignIn(email, password)" + } + ], + "resultChars": 38360 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"createClient publishable key javascript initializing\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/realtime/broadcast", + "title": "Broadcast" + }, + { + "url": "https://supabase.com/docs/reference/javascript/initializing" + }, + { + "url": "https://supabase.com/docs/guides/api/creating-routes", + "title": "Creating API Routes" + }, + { + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" + } + ], + "resultChars": 89718 + } + ] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", @@ -256,7 +302,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 12e8796d-45de-4b71-97bd-411feef5aa01, signUp returned {\"userId\":\"12e8796d-45de-4b71-97bd-411feef5aa01\"}" + "notes": "db user 4058aaa5-1010-4e8c-879c-b9d6b4679134, signUp returned {\"userId\":\"4058aaa5-1010-4e8c-879c-b9d6b4679134\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -271,7 +317,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"12e8796d-45de-4b71-97bd-411feef5aa01\"}" + "notes": "{\"userId\":\"4058aaa5-1010-4e8c-879c-b9d6b4679134\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -300,41 +346,15 @@ }, "docs": { "calls": [ - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | head -60", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3012 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | grep -n -i -B2 -A4 \"breaking\" | grep -i -E \"auth|supabase-js|client|key|session|breaking change\" | head -60", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 10664 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | grep -n -i -E \"supabase-js|publishable|api key|signUp|password\" | head -40", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 7609 - }, { "source": "search_docs", - "query": "{ searchDocs(query: \"signUp email password user metadata display name javascript\", limit: 4) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"signUp email password user metadata options data javascript\", limit: 4) { nodes { title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" + }, { "url": "https://supabase.com/docs/reference/javascript/auth-signup" }, @@ -342,68 +362,11 @@ "url": "https://supabase.com/docs/guides/auth/auth-mfa", "title": "Multi-Factor Authentication" }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" - }, { "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" } ], "resultChars": 22309 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"getClaims getUser getSession which to use client\", limit: 4) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", - "title": "Build a User Management App with React" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", - "title": "Build a User Management App with Angular" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" - } - ], - "resultChars": 145025 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/reference/javascript/auth-getclaims.md | head -60; echo \"=== GETUSER ===\"; curl -s https://supabase.com/docs/reference/javascript/auth-getuser.md | head -40", - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-getclaims.md" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser.md" - } - ], - "resultChars": 15 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"auth getClaims\", limit: 2) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", - "title": "Build a User Management App with React" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" - } - ], - "resultChars": 70527 } ] }, @@ -446,7 +409,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 2 rows" + "notes": "found 4 rows" }, { "name": "row level security is enabled on todos", @@ -464,7 +427,7 @@ { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "2 rows" + "notes": "4 rows" } ], "skills": { @@ -480,34 +443,15 @@ "docs": { "calls": [ { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md 2>/dev/null | head -60 || echo \"CHANGELOG FETCH FAILED (likely no network)\"", + "source": "web_fetch", + "query": "List any entries tagged breaking-change, and any entries related to: RLS policies, row level security defaults, the local CLI dev stack (supabase start/init/config.toml), migrations, seed.sql / db seeding behavior, or default grants for anon/authenticated roles on new tables. Include dates and short descriptions.", + "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 3012 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/api/securing-your-api.md | head -120", - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/securing-your-api.md" - } - ], - "resultChars": 6999 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/api/securing-your-api.md | sed -n '/^## Configure Data API security/,/^### Configure a pre-request/p' | head -140", - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/securing-your-api.md" - } - ], - "resultChars": 4028 + "resultChars": 3775 } ] }, @@ -584,52 +528,39 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "{ searchDocs(query: \"seed data local development seed.sql config\", limit: 5) { nodes { title href content } } }", + "source": "web_fetch", + "query": "List any entries tagged breaking-change, and any entries relating to: Row Level Security / RLS policy behaviour, default grants to anon/authenticated roles, local CLI `supabase start`/`db reset`/seed.sql behaviour, config.toml seed settings, or PostgREST default table exposure. Include dates and links.", "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/github-integration", - "title": "GitHub integration" - }, - { - "url": "https://supabase.com/docs/guides/local-development/seeding-your-database", - "title": "Seeding your database" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - } - ], - "resultChars": 64416 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | head -c 4000", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 3967 + "resultChars": 1480 }, { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog/45329-breaking-change-tables-not-exposed-to-data-and-graphql-api-automatically.md 2>/dev/null | head -c 3000 || sed -n '208,220p' /tmp/cl.md", + "source": "search_docs", + "query": "query { searchDocs(query: \"expose table to Data API grant select anon authenticated securing your API\", limit: 4) { nodes { title href content } } }", + "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog/45329-breaking-change-tables-not-exposed-to-data-and-graphql-api-automatically.md" + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + }, + { + "url": "https://supabase.com/docs/guides/observability/advisors?queryGroups=lint&lint=0017_foreign_table_in_api", + "title": "Database Advisor: Lint 0017_foreign_table_in_api" + }, + { + "url": "https://supabase.com/docs/guides/observability/advisors?queryGroups=lint&lint=0027_pg_graphql_authenticated_table_exposed", + "title": "Database Advisor: Lint 0027_pg_graphql_authenticated_table_exposed" + }, + { + "url": "https://supabase.com/docs/guides/observability/advisors?queryGroups=lint&lint=0026_pg_graphql_anon_table_exposed", + "title": "Database Advisor: Lint 0026_pg_graphql_anon_table_exposed" } ], - "resultChars": 2994 + "resultChars": 41479 } ] }, @@ -685,7 +616,7 @@ { "name": "REST API returns no todos to anonymous requests", "passed": true, - "notes": "error 42501: permission denied for table todos" + "notes": "0 rows" }, { "name": "REST API returns the todos to authenticated requests", @@ -707,23 +638,13 @@ "calls": [ { "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md 2>&1 | grep -iE 'breaking|rls|polic|migration' | head -30", + "query": "curl -s https://supabase.com/changelog.md 2>&1 | head -c 3000", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 5297 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog/45329-breaking-change-tables-not-exposed-to-data-and-graphql-api-automatically.md 2>&1 | head -60", - "pages": [ - { - "url": "https://supabase.com/changelog/45329-breaking-change-tables-not-exposed-to-data-and-graphql-api-automatically.md" - } - ], - "resultChars": 3976 + "resultChars": 2973 } ] }, @@ -782,7 +703,18 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "curl -s --max-time 25 https://supabase.com/changelog.md 2>&1 | grep -i -n \"breaking\\|declarative\\|db diff\" | head -20 || echo \"(changelog fetch failed)\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 3694 + } + ] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", @@ -896,32 +828,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"declarative database schemas generate migration db diff local\", limit: 4) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", - "title": "Declarative database schemas" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments", - "title": "Managing Environments" - } - ], - "resultChars": 66910 - } - ] + "calls": [] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", @@ -966,7 +873,7 @@ { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 12) from the queue" + "notes": "function removed the seeded message (id 5) from the queue" } ], "skills": { @@ -983,87 +890,91 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron schedule job queues pgmq send read delete\", limit: 8) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pg_cron schedule job every minute queues pgmq send\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, { "url": "https://supabase.com/docs/guides/database/extensions/pg_net", "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/platform/upgrading", - "title": "Upgrading" + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" }, { "url": "https://supabase.com/docs/guides/cron", "title": "Cron" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" }, { "url": "https://supabase.com/docs/guides/cron/install", "title": "Install" - }, + } + ], + "resultChars": 69432 + }, + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | grep -iE 'cron|queue|pgmq|edge function|breaking' | head -40", + "pages": [ { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 96246 + "resultChars": 7390 }, { - "source": "web_fetch", - "query": "Show the exact SQL to create a queue, and the exact function signatures for sending, reading, and deleting messages (both pgmq.* and pgmq_public.* wrappers). Include argument names and order.", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog/39378-potential-breaking-change-in-pgmq-from-1-4-4-to-1-5-1-and-temporary-halt-on-upgrade-for-existing-projects.md 2>/dev/null | head -60", "pages": [ { - "url": "https://supabase.com/docs/guides/queues/quickstart.md" + "url": "https://supabase.com/changelog/39378-potential-breaking-change-in-pgmq-from-1-4-4-to-1-5-1-and-temporary-halt-on-upgrade-for-existing-projects.md" } ], - "resultChars": 1021 + "resultChars": 1370 }, { - "source": "web_fetch", - "query": "Show the exact SQL syntax for cron.schedule including job name, every-minute schedule, and how to unschedule. Note any guidance about idempotency or naming.", - "hasContent": true, + "source": "search_docs", + "query": "{ searchDocs(query: \"Supabase Queues API read delete messages edge function\", limit: 2) { nodes { title href } } }", + "hasContent": false, "pages": [ { - "url": "https://supabase.com/docs/guides/cron/quickstart.md" + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" } ], - "resultChars": 1011 + "resultChars": 339 }, { - "source": "web_fetch", - "query": "List the exact pgmq_public function signatures with argument names and order for send, read, pop, delete, archive. Also how to create a queue in SQL.", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions.md | head -120", "pages": [ { - "url": "https://supabase.com/docs/guides/queues/api.md" + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions.md" } ], - "resultChars": 760 + "resultChars": 3873 }, { - "source": "web_fetch", - "query": "Show any JavaScript/supabase-js example of reading from and deleting messages in a queue, including the .schema('pgmq_public').rpc(...) calls and the shape of the returned message objects (field names like msg_id, message).", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s https://supabase.com/docs/guides/cron/quickstart.md | head -100", "pages": [ { - "url": "https://supabase.com/docs/guides/queues/api.md" + "url": "https://supabase.com/docs/guides/cron/quickstart.md" } ], - "resultChars": 862 + "resultChars": 2878 } ] }, @@ -1105,12 +1016,12 @@ { "name": "cron command enqueues to the 'tasks' queue", "passed": true, - "notes": "queue depth 1 -> 2" + "notes": "queue depth 0 -> 1" }, { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 5) from the queue" + "notes": "function removed the seeded message (id 7) from the queue" } ], "skills": { @@ -1127,27 +1038,59 @@ "calls": [ { "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | head -60", + "query": "curl -s https://supabase.com/changelog.md | head -c 4000", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 3012 + "resultChars": 3967 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Queues quickstart pgmq send read delete edge function\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + }, + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, + { + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" + } + ], + "resultChars": 54557 }, { "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | grep -iE 'cron|queue|pgmq|edge function|functions' -B4 | head -80", + "query": "curl -s https://supabase.com/changelog.md | grep -iE \"breaking|cron|queue|pgmq|edge function\" | head -40", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 4958 + "resultChars": 7390 }, { "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog/39378-potential-breaking-change-in-pgmq-from-1-4-4-to-1-5-1-and-temporary-halt-on-upgrade-for-existing-projects.md 2>/dev/null | head -60", + "query": "curl -s \"https://supabase.com/changelog/39378-potential-breaking-change-in-pgmq-from-1-4-4-to-1-5-1-and-temporary-halt-on-upgrade-for-existing-projects.md\" | head -c 3000", "pages": [ { "url": "https://supabase.com/changelog/39378-potential-breaking-change-in-pgmq-from-1-4-4-to-1-5-1-and-temporary-halt-on-upgrade-for-existing-projects.md" @@ -1157,7 +1100,7 @@ }, { "source": "shell_fetch", - "query": "curl -sL https://supabase.com/docs/guides/queues/api.md | head -150", + "query": "curl -s \"https://supabase.com/docs/guides/queues/api.md\" | head -c 6000", "pages": [ { "url": "https://supabase.com/docs/guides/queues/api.md" @@ -1167,16 +1110,42 @@ }, { "source": "shell_fetch", - "query": "curl -sL https://supabase.com/docs/guides/functions/quickstart.md | head -80; echo \"########## config\"; curl -sL https://supabase.com/docs/guides/local-development/cli/config.md | grep -iE 'verify_jwt|functions\\.|\\[functions' -A3 | head -60", + "query": "curl -s \"https://supabase.com/docs/guides/queues/api.md\" | tail -c 2000; echo \"=== EDGE FN CONSUMER DOC ===\"; curl -s \"https://supabase.com/docs/guides/functions/examples/queues.md\" | head -c 5000", "pages": [ { - "url": "https://supabase.com/docs/guides/functions/quickstart.md" + "url": "https://supabase.com/docs/guides/queues/api.md" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/config.md" + "url": "https://supabase.com/docs/guides/functions/examples/queues.md" + } + ], + "resultChars": 7028 + }, + { + "source": "shell_fetch", + "query": "curl -s \"https://supabase.com/docs/guides/cron/quickstart.md\" | head -c 5000", + "pages": [ + { + "url": "https://supabase.com/docs/guides/cron/quickstart.md" } ], - "resultChars": 3577 + "resultChars": 4790 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Consuming Supabase Queue Messages with Edge Functions\", limit: 2) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/architecture", + "title": "Edge Functions Architecture" + } + ], + "resultChars": 10960 } ] }, @@ -1218,12 +1187,12 @@ { "name": "cron command enqueues to the 'tasks' queue", "passed": true, - "notes": "queue depth 1 -> 2" + "notes": "queue depth 0 -> 1" }, { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 51) from the queue" + "notes": "function removed the seeded message (id 37) from the queue" } ], "skills": { @@ -1240,75 +1209,93 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"cron job schedule pgmq queue send message\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"cron job schedule pg_cron create job\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" + "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", + "title": "pg_cron debugging guide" }, { "url": "https://supabase.com/docs/guides/database/extensions/pg_net", "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" + }, + { + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" }, { "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", "title": "Automatic embeddings" }, { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" + "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-sms-hook", + "title": "Send SMS Hook" } ], - "resultChars": 72816 + "resultChars": 77278 }, { - "source": "web_fetch", - "query": "Summarize the current recommended way to create a queue, send messages, read messages, and delete/archive them. Include exact SQL function signatures and any notes about queue types (basic/unlogged/partitioned) and pgmq_public wrappers.", + "source": "search_docs", + "query": "{ searchDocs(query: \"queues pgmq read delete messages edge function\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/queues/quickstart.md" + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", + "title": "Backup and Restore using the CLI" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" } ], - "resultChars": 1463 + "resultChars": 103837 }, { - "source": "web_fetch", - "query": "Show exactly how to create, name, schedule (every minute), and manage a cron job with cron.schedule / cron.alter_job / cron.unschedule. Include how to name a job, the SQL syntax, and any notes about editing existing jobs or the 'seconds' syntax.", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -sL https://supabase.com/changelog.md | grep -inE \"cron|queue|pgmq|edge function|breaking\" | head -40", "pages": [ { - "url": "https://supabase.com/docs/guides/cron/quickstart.md" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 1282 + "resultChars": 7545 }, { - "source": "web_fetch", - "query": "List any entries tagged breaking-change, and any entries related to Edge Functions runtime/deno version, Queues/pgmq, pg_cron, or API keys (publishable/secret vs anon/service_role). Include dates and links.", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -sL \"https://supabase.com/changelog/39378-potential-breaking-change-in-pgmq-from-1-4-4-to-1-5-1-and-temporary-halt-on-upgrade-for-existing-projects.md\" | head -c 3000", "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/changelog/39378-potential-breaking-change-in-pgmq-from-1-4-4-to-1-5-1-and-temporary-halt-on-upgrade-for-existing-projects.md" } ], - "resultChars": 3060 + "resultChars": 1370 }, { - "source": "web_fetch", - "query": "List the exact pgmq_public API functions exposed over PostgREST with their parameter names, especially read (queue_name, sleep_seconds, n) and delete (queue_name, message_id). Show a supabase-js example of reading and deleting messages including which schema and which API key is required.", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -sL \"https://supabase.com/docs/guides/functions/secrets.md\" | head -c 3000", "pages": [ { - "url": "https://supabase.com/docs/guides/queues/api.md" + "url": "https://supabase.com/docs/guides/functions/secrets.md" } ], - "resultChars": 1048 + "resultChars": 2998 } ] }, @@ -1511,13 +1498,45 @@ "calls": [ { "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | head -60", + "query": "curl -s https://supabase.com/changelog.md 2>&1 | head -80", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 3012 + "resultChars": 4384 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"secret key server-side supabase-js service role backend\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/quickstarts/refine", + "title": "Use Supabase with Refine" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", + "title": "Performing administration tasks on the server side with a secret key" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", + "title": "Build a Social Auth App with Expo React Native" + } + ], + "resultChars": 121868 } ] }, @@ -1581,57 +1600,50 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | head -60", + "source": "web_fetch", + "query": "List any recent breaking changes or notable entries related to: API keys (publishable/secret keys, sb_secret_ format), supabase-js client versions, PostgREST max-rows / row limits, or service_role behavior. Include dates and links.", + "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 3012 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog/45702-developer-update-may-2026.md 2>/dev/null | head -80", - "pages": [ - { - "url": "https://supabase.com/changelog/45702-developer-update-may-2026.md" - } - ], - "resultChars": 4515 + "resultChars": 1370 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"secret key service_role server-side createClient API keys\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"secret key server-side supabase-js service_role new API keys\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", + "title": "Build a User Management App with SvelteKit" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-vue", + "title": "Build a User Management App with Ionic Vue" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" } ], - "resultChars": 99945 + "resultChars": 135506 } ] }, @@ -1699,7 +1711,50 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_fetch", + "query": "List any entries tagged breaking-change, and any entries about supabase-js v2, API keys (publishable/secret keys), or PostgREST filters/embedded resources. Include dates and links.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 6313 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"filter comparing two columns PostgREST, embedded resource filter, pagination range\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + }, + { + "url": "https://supabase.com/docs/guides/ai/langchain", + "title": "LangChain" + }, + { + "url": "https://supabase.com/docs/guides/observability/advanced-log-filtering", + "title": "Query and filter logs" + }, + { + "url": "https://supabase.com/docs/reference/swift/using-filters", + "title": "Using Filters" + }, + { + "url": "https://supabase.com/docs/reference/javascript/using-filters-filter" + }, + { + "url": "https://supabase.com/docs/reference/kotlin/using-filters", + "title": "Using Filters" + } + ], + "resultChars": 51102 + } + ] }, "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", @@ -1765,47 +1820,7 @@ ] }, "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "List any entries tagged breaking-change, and any entries about API keys (publishable/secret keys, sb_secret_), supabase-js v2 client behaviour, or PostgREST filtering/ordering changes. Include dates and links.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3940 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"PostgREST filter compare two columns in same row\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/postgrest-error-400-column-example_tableexample_column-does-not-exist-when-using-or-operators-46ff23", - "title": "PostgREST error: 400 'column example_table.example_column does not exist' when using OR operators" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/rls-performance-and-best-practices-Z5Jjwv", - "title": "RLS Performance and Best Practices" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/advanced-log-filtering", - "title": "Advanced Log Querying and Filtering" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-columns", - "title": "Vector columns" - } - ], - "resultChars": 45579 - } - ] + "calls": [] }, "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", @@ -1865,7 +1880,40 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"migrate existing postgres database to Supabase restore pg_dump pg_restore\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", + "title": "Migrate from Vercel Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/neon", + "title": "Migrate from Neon to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/heroku", + "title": "Migrate from Heroku to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/mysql", + "title": "Migrate from MySQL to Supabase" + } + ], + "resultChars": 49442 + } + ] }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", @@ -1891,12 +1939,27 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { - "name": "scorer completed without errors", - "passed": false, - "notes": "query failed: ERROR: relation \"teams\" does not exist\nLINE 1: ..., '[]'::json) from (select count(*)::int as n from teams) t;\n ^\n" + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true } ], "skills": { @@ -1910,7 +1973,40 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"migrate existing Postgres database to Supabase pg_restore custom dump\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", + "title": "Migrate from Vercel Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/neon", + "title": "Migrate from Neon to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/heroku", + "title": "Migrate from Heroku to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/mysql", + "title": "Migrate from MySQL to Supabase" + } + ], + "resultChars": 49442 + } + ] }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", @@ -2020,12 +2116,12 @@ { "name": "user A cannot force-read user B note", "passed": true, - "notes": "status=200" + "notes": "status=403" }, { "name": "user B cannot force-read user A note", "passed": true, - "notes": "status=200" + "notes": "status=403" } ], "skills": { @@ -2041,31 +2137,31 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Function auth get user from JWT Authorization header RLS\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Edge Function auth user Authorization header createClient RLS\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" }, { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" } ], - "resultChars": 40314 + "resultChars": 67005 } ] }, @@ -2117,12 +2213,12 @@ { "name": "user A cannot force-read user B note", "passed": true, - "notes": "status=403" + "notes": "status=200" }, { "name": "user B cannot force-read user A note", "passed": true, - "notes": "status=403" + "notes": "status=200" } ], "skills": { @@ -2138,27 +2234,27 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Function auth get user Authorization header RLS\", limit: 4) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"edge function auth getUser Authorization header user context RLS\", limit: 4) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, { "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { "url": "https://supabase.com/docs/guides/functions/auth", "title": "Securing Edge Functions" } ], - "resultChars": 29217 + "resultChars": 19048 } ] }, @@ -2190,7 +2286,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "rejects missing auth", @@ -2199,23 +2295,23 @@ }, { "name": "user A reads own note", - "passed": true, - "notes": "status=200" + "passed": false, + "notes": "edge function dependency \"npm:@supabase/server@1.5.3\" is not available in the eval runtime (MODULE_NOT_FOUND). Add it to the runtime's dependencies if an eval needs it." }, { "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" + "passed": false, + "notes": "edge function dependency \"npm:@supabase/server@1.5.3\" is not available in the eval runtime (MODULE_NOT_FOUND). Add it to the runtime's dependencies if an eval needs it." }, { "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=403" + "passed": false, + "notes": "edge function dependency \"npm:@supabase/server@1.5.3\" is not available in the eval runtime (MODULE_NOT_FOUND). Add it to the runtime's dependencies if an eval needs it." }, { "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=403" + "passed": false, + "notes": "edge function dependency \"npm:@supabase/server@1.5.3\" is not available in the eval runtime (MODULE_NOT_FOUND). Add it to the runtime's dependencies if an eval needs it." } ], "skills": { @@ -2229,39 +2325,15 @@ }, "docs": { "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Function auth user JWT Authorization header createClient RLS\", limit: 4) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - } - ], - "resultChars": 56091 - }, { "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | grep -iE \"edge function|anon key|publishable|breaking\" | head -30", + "query": "curl -s https://supabase.com/changelog.md | grep -i -E \"breaking|edge function|anon key|publishable\" | head -40", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 5536 + "resultChars": 7371 }, { "source": "shell_fetch", @@ -2271,7 +2343,7 @@ "url": "https://supabase.com/docs/guides/functions/auth.md" } ], - "resultChars": 5299 + "resultChars": 5158 }, { "source": "shell_fetch", @@ -2281,7 +2353,7 @@ "url": "https://supabase.com/docs/guides/functions/auth-headers.md" } ], - "resultChars": 3330 + "resultChars": 3747 } ] }, @@ -2324,37 +2396,37 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"MISSING_CREDENTIALS\",\"message\":\"[@supabase/server] No credentials found on the request. This endpoint accepts auth mode(s):" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"7569ff0a-d265-41c0-abc2-33ae75228fc8\",\"metric\":\"steps_a_mta4gorm\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"7f28496c-2b44-4be7-99ab-0b81ac3b52c4\",\"metric\":\"steps_a_mtsfy5cu\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"7569ff0a-d265-41c0-abc2-33ae75228fc8\",\"metric\":\"steps_a_mta4gorm\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"7f28496c-2b44-4be7-99ab-0b81ac3b52c4\",\"metric\":\"steps_a_mtsfy5cu\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"beb23f4e-9f15-4c26-959f-b4f632a3cf5b\",\"metric\":\"steps_b_mta4gorm\",\"value\":222}]" + "notes": "status 200: [{\"user_id\":\"430dcf08-afdd-4545-ae34-696659f5c832\",\"metric\":\"steps_b_mtsfy5cu\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_JWT\",\"message\":\"[@supabase/server] The JWT in the Authorization header failed verification: its header is missing \\" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" }, { "name": "implementation uses @supabase/server", @@ -2374,19 +2446,31 @@ "docs": { "calls": [ { - "source": "web_fetch", - "query": "How do Edge Functions authenticate callers? Explain verify_jwt, the Authorization header, the apikey header, how to get the signed-in user (auth.getUser), and how to detect/handle a service-role key caller. Include code snippets verbatim.", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | head -100", "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth.md" + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5440 + }, + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog/45702-developer-update-may-2026.md 2>/dev/null | head -60; echo \"=== ALT ===\"; curl -s https://supabase.com/changelog/45702-developer-update-may-2026 2>/dev/null | grep -o -i -E \".{300}supabase/server.{600}\" | head -5", + "pages": [ + { + "url": "https://supabase.com/changelog/45702-developer-update-may-2026.md" + }, + { + "url": "https://supabase.com/changelog/45702-developer-update-may-2026" } ], - "resultChars": 1916 + "resultChars": 4369 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"edge function service role key apikey header verify JWT new API keys publishable secret\", limit: 12) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"@supabase/server SDK createClient edge functions auth\", limit: 12) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -2394,54 +2478,58 @@ "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", + "title": "Resumable WebSockets with Edge Functions" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/auth/server-side/migrating-to-ssr-from-auth-helpers", + "title": "Migrating to the SSR package from Auth Helpers" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-updateclient" }, { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-getclient" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" + }, + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-deleteclient" } ], - "resultChars": 174799 + "resultChars": 76009 }, { "source": "web_fetch", - "query": "Full reference for @supabase/server withSupabase: auth modes, ctx fields (supabase, supabaseAdmin, userClaims, authMode), error responses, required env vars. Quote all code examples verbatim.", + "query": "Describe the @supabase/server SDK: installation, exports, API surface, how it handles auth (user JWT vs secret/service-role key), client creation, CORS, context injection. Include full code examples verbatim.", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/supabase-server.md" + "url": "https://supabase.com/blog/introducing-supabase-server" } ], - "resultChars": 215 + "resultChars": 4497 } ] }, @@ -2474,7 +2562,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "seed rows present", @@ -2484,42 +2572,42 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"[@supabase/server] No credentials found on the request. This endpoint accepts auth mode(s): \\\"user\\\", \\\"secret\\\".\",\"code\":\"MISSING_CREDENTIALS\"}" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"7b9113e1-029d-46b5-9745-41f904405b11\",\"metric\":\"steps_a_mta4geue\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"21ee8fe0-99ab-4a09-9911-da6a05d0ebb2\",\"metric\":\"steps_a_mtsg0v46\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"7b9113e1-029d-46b5-9745-41f904405b11\",\"metric\":\"steps_a_mta4geue\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"21ee8fe0-99ab-4a09-9911-da6a05d0ebb2\",\"metric\":\"steps_a_mtsg0v46\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"02e17dab-608f-4c44-9afd-9e14d569be9b\",\"metric\":\"steps_b_mta4geue\",\"value\":222}]" + "notes": "status 200: [{\"user_id\":\"17854e55-866a-475d-9d65-7c5c9fa72b9f\",\"metric\":\"steps_b_mtsg0v46\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"secret\\\".\",\"code\":\"INVALID_API_KEY\"}" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"[@supabase/server] The JWT in the Authorization header failed verification: its header is missing \\\"kid\\\".\",\"code\":\"INVALID_JWT\"}" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"secret\\\".\",\"code\":\"INVALID_API_KEY\"}" }, { "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" + "passed": false, + "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" } ], "skills": { @@ -2535,46 +2623,65 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"edge function service role key apikey header authenticate user JWT\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Edge Function service role key apikey header verify_jwt auth\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, { "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "title": "API keys" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { "url": "https://supabase.com/docs/guides/functions/auth", "title": "Securing Edge Functions" }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + }, { "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" } ], - "resultChars": 87424 + "resultChars": 75084 }, { - "source": "web_fetch", - "query": "Describe the new publishable/secret API key formats (sb_publishable_, sb_secret_), how they differ from legacy anon/service_role JWT keys, and any guidance about verifying or comparing them in Edge Functions. Also list any env vars exposed to edge functions.", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s https://supabase.com/docs/guides/functions/auth.md | head -200", + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth.md" + } + ], + "resultChars": 7819 + }, + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/docs/guides/api/api-keys.md | head -150", "pages": [ { "url": "https://supabase.com/docs/guides/api/api-keys.md" } ], - "resultChars": 1347 + "resultChars": 14 + }, + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/docs/guides/functions/auth-headers.md", + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth-headers.md" + } + ], + "resultChars": 3747 } ] }, @@ -2607,7 +2714,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "seed rows present", @@ -2617,37 +2724,37 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" }, { "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"f9227b29-d5a1-40f9-8fbf-b0d5794218ff\",\"metric\":\"steps_a_mta4b366\",\"value\":111}]" + "passed": false, + "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" }, { "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: [{\"user_id\":\"f9227b29-d5a1-40f9-8fbf-b0d5794218ff\",\"metric\":\"steps_a_mta4b366\",\"value\":111}]" + "passed": false, + "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" }, { "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"50d3beb7-a66e-4fe9-adc2-107d3b97be03\",\"metric\":\"steps_b_mta4b366\",\"value\":222}]" + "passed": false, + "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" }, { "name": "implementation uses @supabase/server", @@ -2666,152 +2773,95 @@ }, "docs": { "calls": [ - { - "source": "web_fetch", - "query": "List any breaking-change entries related to Edge Functions, API keys (publishable/secret keys), JWT verification, or the supabase-js / @supabase/server libraries. Include dates and links.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3204 - }, { "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server package edge functions server client\", limit: 8) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Edge Function service role key apikey header verify_jwt auth getUser\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" }, { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/reference/javascript/auth-admin-listusers" }, { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/functions/architecture", - "title": "Edge Functions Architecture" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" } ], - "resultChars": 60233 + "resultChars": 69705 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"edge function authentication verify_jwt service role key apikey header user token\", limit: 8) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"@supabase/server package edge functions publishable secret API keys\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-email-hook", - "title": "Send Email Hook" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - } - ], - "resultChars": 153936 - }, - { - "source": "web_fetch", - "query": "Full details on @supabase/server: what it is, install name/version, exact API (createClient / client factory signature), how it reads Authorization headers, how it handles service/secret keys vs publishable keys, and code examples. Reproduce code examples verbatim.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package.md" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" } ], - "resultChars": 1300 + "resultChars": 76850 }, { - "source": "web_fetch", - "query": "Reproduce this page in full, especially: how Edge Functions handle auth, verify_jwt config, the apikey header, service role key usage, getUser vs getClaims, and all code examples verbatim.", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s https://supabase.com/docs/guides/functions/auth.md 2>&1 | head -200", "pages": [ { "url": "https://supabase.com/docs/guides/functions/auth.md" } ], - "resultChars": 3602 - }, - { - "source": "web_fetch", - "query": "Reproduce the @supabase/server API reference: withSupabase signature and options, createSupabaseContext, the full list of ctx fields (supabase, supabaseAdmin, userClaims, authMode, etc.), auth mode strings, and all code examples verbatim.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/server/introduction" - } - ], - "resultChars": 1518 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"withSupabase authMode ctx supabaseAdmin userClaims\", limit: 3) { nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - } - ], - "resultChars": 171 + "resultChars": 7819 }, { - "source": "web_fetch", - "query": "Reproduce the withSupabase reference verbatim: signature, all options, the SupabaseContext fields, auth modes, and every code example.", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s https://supabase.com/docs/guides/functions/auth-headers.md 2>&1 | head -120", "pages": [ { - "url": "https://supabase.com/docs/reference/server/withsupabase" + "url": "https://supabase.com/docs/guides/functions/auth-headers.md" } ], - "resultChars": 215 + "resultChars": 3747 } ] }, @@ -3015,28 +3065,7 @@ ] }, "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "curl -s --max-time 25 https://supabase.com/changelog.md | head -60", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3012 - }, - { - "source": "shell_fetch", - "query": "curl -s --max-time 25 https://supabase.com/changelog.md | grep -i -n -B4 \"Breaking Change\" | grep -iE \"rls|polic|auth|jwt|role|security\" ", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3638 - } - ] + "calls": [] }, "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", @@ -3182,7 +3211,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 01a03e37-f8c0-72b8-bd4d-52c1f9613f04/receipt-alpha.pdf, 01a03e37-f8c0-72b8-bd4d-52c1f9613f04/receipt-beta.pdf" + "notes": "saw: 01a08041-ec37-704c-979c-6f5b84775f78/receipt-alpha.pdf, 01a08041-ec37-704c-979c-6f5b84775f78/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -3203,7 +3232,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Creates a private user-files bucket, keeps RLS enabled, adds authenticated owner-scoped SELECT and INSERT policies, and uses createSignedUrl with a one-hour expiry." + "judgeNotes": "Creates a private user-files bucket, keeps RLS enabled, adds authenticated owner-scoped SELECT and INSERT policies (plus safe UPDATE/DELETE), and uses createSignedUrl with an expiry. No public access or client-side service-role key." } ], "skills": { @@ -3219,7 +3248,7 @@ "calls": [ { "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | head -100", + "query": "curl -s https://supabase.com/changelog.md 2>/dev/null | head -100", "pages": [ { "url": "https://supabase.com/changelog.md" @@ -3229,49 +3258,62 @@ }, { "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog/43465-developer-update-march-2026.md 2>/dev/null | grep -iA30 -B5 \"storage\" | head -80", + "query": "curl -s https://supabase.com/changelog.md 2>/dev/null | grep -iE \"storage|bucket|signed\" | head -40", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 7080 + }, + { + "source": "shell_fetch", + "query": "echo \"=== 2025 storage schema restriction ===\"; curl -s https://supabase.com/changelog/34270-restricting-access-on-auth-storage-and-realtime-schemas-on-april-21-2025.md 2>/dev/null | head -60; echo; echo \"=== March 2026 dev update ===\"; curl -s https://supabase.com/changelog/43465-developer-update-march-2026.md 2>/dev/null | grep -iA12 \"storage\"| head -50", "pages": [ + { + "url": "https://supabase.com/changelog/34270-restricting-access-on-auth-storage-and-realtime-schemas-on-april-21-2025.md" + }, { "url": "https://supabase.com/changelog/43465-developer-update-march-2026.md" } ], - "resultChars": 5122 + "resultChars": 5268 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"storage RLS policy user id folder ownership private bucket\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"storage bucket RLS policy user folder access control uid\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, { "url": "https://supabase.com/docs/guides/storage/security/access-control", "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + }, + { + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/guides/storage/schema/design", - "title": "The Storage Schema" + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" + "url": "https://supabase.com/docs/guides/storage/schema/design", + "title": "The Storage Schema" } ], - "resultChars": 34710 + "resultChars": 64148 }, { "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/storage/schema/helper-functions.md", + "query": "curl -s https://supabase.com/docs/guides/storage/schema/helper-functions.md 2>/dev/null | head -200", "pages": [ { "url": "https://supabase.com/docs/guides/storage/schema/helper-functions.md" @@ -3281,17 +3323,27 @@ }, { "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/storage/security/ownership.md", + "query": "curl -s https://supabase.com/docs/guides/storage/security/access-control.md 2>/dev/null | head -180", + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/security/access-control.md" + } + ], + "resultChars": 4168 + }, + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/docs/reference/javascript/storage-from-createsignedurl.md 2>/dev/null | head -80", "pages": [ { - "url": "https://supabase.com/docs/guides/storage/security/ownership.md" + "url": "https://supabase.com/docs/reference/javascript/storage-from-createsignedurl.md" } ], - "resultChars": 1435 + "resultChars": 31 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"createSignedUrl expires in signed URL download storage javascript\", limit: 4) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"createSignedUrl temporary expiring link share file\", limit: 4) { nodes { title href content ... on ClientLibraryFunctionReference { methodName language } } } }", "hasContent": true, "pages": [ { @@ -3304,10 +3356,11 @@ "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" + "url": "https://supabase.com/docs/reference/csharp/storage-from-createsignedurl", + "title": "From().CreateSignedUrl()" } ], - "resultChars": 3116 + "resultChars": 3351 } ] }, @@ -3354,7 +3407,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 01a03e38-b20f-77a4-9663-bb1dde74fd5d/receipt-alpha.pdf, 01a03e38-b20f-77a4-9663-bb1dde74fd5d/receipt-beta.pdf" + "notes": "saw: 01a08042-d790-77a9-b435-cfc77724e472/receipt-alpha.pdf, 01a08042-d790-77a9-b435-cfc77724e472/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -3375,7 +3428,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Creates a private user-files bucket, authenticated owner-scoped SELECT and INSERT policies with RLS retained, and supabase-js createSignedUrl code with expiration." + "judgeNotes": "Configures a private user-files bucket, authenticated owner-scoped SELECT and INSERT policies with RLS retained, and supabase-js createSignedUrl code with a short expiry." } ], "skills": { @@ -3389,78 +3442,136 @@ }, "docs": { "calls": [ + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md 2>&1 | head -60", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 3012 + }, + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md 2>&1 | grep -iE \"storage|bucket|signed|rls|policy\" | head -40", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 7230 + }, + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog/43465-developer-update-march-2026.md 2>&1 | head -80", + "pages": [ + { + "url": "https://supabase.com/changelog/43465-developer-update-march-2026.md" + } + ], + "resultChars": 4887 + }, { "source": "search_docs", - "query": "{ searchDocs(query: \"storage RLS policy private bucket user folder owner uid first path segment\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"storage bucket RLS policy user folder owner access control\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + }, { "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", + "title": "Backup and Restore using the CLI" + }, + { + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" + "url": "https://supabase.com/docs/guides/storage/security/ownership", + "title": "Ownership" }, { "url": "https://supabase.com/docs/guides/storage/schema/design", "title": "The Storage Schema" - }, + } + ], + "resultChars": 98621 + }, + { + "source": "shell_fetch", + "query": "cd /tmp && for p in \"guides/storage/security/access-control\" \"guides/storage/serving/downloads\"; do echo \"=====$p=====\"; curl -s \"https://supabase.com/docs/$p.md\" | head -200; done", + "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" + "url": "https://supabase.com/docs/$p.md" } ], - "resultChars": 34710 + "resultChars": 8021 }, { "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/storage/schema/helper-functions.md | head -200", + "query": "cd /tmp && echo \"=====HELPERS=====\"; curl -s \"https://supabase.com/docs/guides/storage/schema/helper-functions.md\"", "pages": [ { "url": "https://supabase.com/docs/guides/storage/schema/helper-functions.md" } ], - "resultChars": 3332 + "resultChars": 3395 }, { "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | grep -iE \"breaking|storage|signed|rls\" | head -40", + "query": "cd /tmp && echo \"=====SCHEMA DESIGN=====\"; curl -s \"https://supabase.com/docs/guides/storage/schema/design.md\"; echo; echo \"=====CREATE BUCKET REF=====\"; curl -s \"https://supabase.com/docs/reference/javascript/storage-createbucket.md\" | head -120", "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/storage/schema/design.md" + }, + { + "url": "https://supabase.com/docs/reference/javascript/storage-createbucket.md" } ], - "resultChars": 7264 + "resultChars": 2682 + }, + { + "source": "shell_fetch", + "query": "cd /tmp && for u in \"reference/javascript/storage-from-createsignedurl\" \"reference/javascript/storage-from-upload\"; do echo \"=====$u=====\"; curl -s \"https://supabase.com/docs/$u.md\" | head -60; echo; done", + "pages": [ + { + "url": "https://supabase.com/docs/$u.md" + } + ], + "resultChars": 156 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"createSignedUrl expiring temporary link share file download transform\", limit: 4) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"createSignedUrl expiresIn share private file\", limit: 3) { nodes { title href ... on ClientLibraryFunctionReference { methodName language content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" + "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", + "title": "Storage Image Transformations" }, { - "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" }, { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", - "title": "Storage Image Transformations" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" } ], - "resultChars": 22012 + "resultChars": 1711 } ] }, @@ -3507,7 +3618,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 01a03e38-0d93-76d9-8166-ab63bf6beb11/receipt-alpha.pdf, 01a03e38-0d93-76d9-8166-ab63bf6beb11/receipt-beta.pdf" + "notes": "saw: 01a0803f-d8fe-76ee-b0cd-bc4182a846c2/receipt-alpha.pdf, 01a0803f-d8fe-76ee-b0cd-bc4182a846c2/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -3528,7 +3639,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Creates a private user-files bucket, keeps storage.objects RLS enabled, adds authenticated owner-scoped SELECT and INSERT policies, and uses createSignedUrl with a short expiry." + "judgeNotes": "Private bucket, RLS remains enabled, authenticated owner-scoped SELECT/INSERT policies are present, and supabase-js uses createSignedUrl with an expiry. No prohibited public or service-role access." } ], "skills": { @@ -3544,7 +3655,7 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"storage RLS policies bucket owner folder user id access control\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"storage RLS policy restrict users to their own folder user id\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -3552,31 +3663,31 @@ "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" }, { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" }, { "url": "https://supabase.com/docs/guides/storage/security/ownership", "title": "Ownership" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", + "title": "Column Level Security" + }, + { + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" } ], - "resultChars": 64054 + "resultChars": 32092 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"createSignedUrl signed URL expiring temporary share private file\", limit: 4) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"createSignedUrl expiring temporary share link storage javascript\", limit: 4) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -3585,37 +3696,15 @@ { "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" }, - { - "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" - }, { "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" - } - ], - "resultChars": 3116 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"storage helper functions allow_only_operation allow_any_operation foldername\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - } - ], - "resultChars": 3819 - }, - { - "source": "web_fetch", - "query": "List any entries tagged breaking-change, or any entries related to Storage buckets, storage RLS policies, signed URLs, or bucket types (analytics/vector). Include dates and links.", - "hasContent": true, - "pages": [ + }, { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" } ], - "resultChars": 2152 + "resultChars": 6480 } ] }, @@ -3654,12 +3743,12 @@ { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "10 passed, 6 failed" + "notes": "11 passed, 11 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies `posts` as having the broken tenant predicate and grounds the conclusion in failed pgTAP tests plus direct verification. It correctly distinguishes `notes` as having the intended predicate." + "judgeNotes": "Correctly identifies the `posts` policy as allowing any authenticated organization member to read all organizations’ posts, and grounds this conclusion in the pgTAP failures. It distinguishes the correctly correlated `notes` policy and treats test results as authoritative." } ], "skills": { @@ -3705,17 +3794,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" + "notes": "4 file(s): supabase/tests/000_rls_enabled.test.sql, supabase/tests/001_notes_tenant_isolation.test.sql, supabase/tests/002_posts_tenant_isolation.test.sql, supabase/tests/003_memberships_tenant_isolation.test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "8 passed, 6 failed" + "notes": "2 passed, 2 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies the posts SELECT policy as missing the organization correlation, causing authenticated members to read other tenants’ posts, and grounds this conclusion in the pgTAP failures. It correctly treats notes as isolated." + "judgeNotes": "Correctly identifies the `posts` tenant-isolation flaw: the policy checks membership in any organization without correlating `memberships.org_id` to `posts.org_id`. It grounds this conclusion in pgTAP failures showing cross-organization reads, while treating `notes` as the passing control." } ], "skills": { @@ -3729,7 +3818,40 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"pgTAP database testing RLS policies supabase test db\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/testing/overview", + "title": "Testing Overview" + }, + { + "url": "https://supabase.com/docs/guides/database/testing", + "title": "Testing Your Database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", + "title": "Testing and linting" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" + } + ], + "resultChars": 61545 + } + ] }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", @@ -3761,17 +3883,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "2 file(s): supabase/tests/001_tenant_isolation_test.sql, supabase/tests/002_membership_integrity_test.sql" + "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "8 passed, 4 failed" + "notes": "17 passed, 5 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies the `posts` RLS policy as missing the organization correlation, cites failing pgTAP cross-tenant tests, and distinguishes it from the correctly written `notes` policy." + "judgeNotes": "Correctly identifies the `posts` RLS policy as allowing cross-organization reads and grounds this in the failing pgTAP assertions. It explicitly distinguishes `notes` as correctly isolated." } ], "skills": { @@ -3780,8 +3902,7 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { @@ -3823,12 +3944,12 @@ { "name": "HNSW index on the embedding column", "passed": true, - "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" + "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" }, { "name": "index operator class matches the search operator", "passed": true, - "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" + "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" }, { "name": "user A search returns only own sections, best match first", @@ -3860,75 +3981,47 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"pgvector semantic search embeddings hnsw index match function RLS\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pgvector semantic search match_document_sections hnsw index gte-small 384 dimensions RLS\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/ai/hybrid-search", "title": "Hybrid search" }, - { - "url": "https://supabase.com/docs/guides/ai/going-to-prod", - "title": "Going to Production" - }, { "url": "https://supabase.com/docs/guides/ai/semantic-search", "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/ai/going-to-prod", + "title": "Going to Production" }, { - "url": "https://supabase.com/docs/guides/ai/vector-indexes/hnsw-indexes", - "title": "HNSW indexes" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" }, { "url": "https://supabase.com/docs/guides/ai/vector-indexes", "title": "Vector indexes" - } - ], - "resultChars": 60951 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/ai/rag-with-permissions.md | head -180", - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions.md" - } - ], - "resultChars": 8878 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/ai/semantic-search.md | sed -n '1,120p'", - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/semantic-search.md" - } - ], - "resultChars": 7767 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/functions/examples/semantic-search.md | grep -inE \"384|vector\\(|gte-small|dimension\" | head -20", - "pages": [ + }, { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search.md" + "url": "https://supabase.com/docs/guides/ai/vector-indexes/ivf-indexes", + "title": "IVFFlat indexes" } ], - "resultChars": 587 + "resultChars": 57448 }, { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | grep -iE \"breaking|vector|pgvector|rls|hnsw\" | head -30", + "source": "search_docs", + "query": "{ searchDocs(query: \"RAG with permissions row level security document_sections embedding match function\", limit: 3) { nodes { title href content } } }", + "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" } ], - "resultChars": 5181 + "resultChars": 12865 } ] }, @@ -3968,12 +4061,12 @@ { "name": "HNSW index on the embedding column", "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" + "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" }, { "name": "index operator class matches the search operator", "passed": true, - "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" + "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" }, { "name": "user A search returns only own sections, best match first", @@ -4005,59 +4098,35 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"RAG with permissions RLS document_sections match_document_sections embedding\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pgvector semantic search RLS match documents hnsw index\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" + "url": "https://supabase.com/docs/guides/ai/hybrid-search", + "title": "Hybrid search" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/ai/vector-indexes", + "title": "Vector indexes" }, { - "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", - "title": "Declarative database schemas" - } - ], - "resultChars": 60812 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"gte-small Supabase.ai Session embedding dimensions vector column edge function\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" }, { "url": "https://supabase.com/docs/guides/ai/semantic-search", "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/ai/concepts", - "title": "Concepts" + "url": "https://supabase.com/docs/guides/ai/going-to-prod", + "title": "Going to Production" }, { - "url": "https://supabase.com/docs/guides/ai", - "title": "AI & Vectors" + "url": "https://supabase.com/docs/guides/ai/vector-indexes/hnsw-indexes", + "title": "HNSW indexes" } ], - "resultChars": 62575 + "resultChars": 60951 } ] }, @@ -4134,63 +4203,27 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"semantic search pgvector match_document_sections embedding RLS\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"gte-small embedding dimensions Supabase.ai Session edge function vector\", limit: 4) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" + "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", + "title": "Semantic Search" }, { "url": "https://supabase.com/docs/guides/ai/semantic-search", "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-columns", - "title": "Vector columns" - } - ], - "resultChars": 82517 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Supabase.ai Session gte-small embedding dimensions edge function\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" + "url": "https://supabase.com/docs/guides/functions/ai-models", + "title": "Running AI Models" }, { "url": "https://supabase.com/docs/guides/ai/concepts", "title": "Concepts" - }, - { - "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", - "title": "Choosing your Compute Add-on" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" } ], - "resultChars": 74041 + "resultChars": 41299 } ] }, @@ -4227,12 +4260,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "HTTPS Supabase scrape uses the required path and Basic Auth password_file; the secret path is volume-mounted, and the existing app job remains intact." + "judgeNotes": "HTTPS Supabase scrape uses the correct metrics path, Basic Auth with password_file, matching mounted secret directory, preserves the app job, and targets a hosted project." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README provides correct Secret API key creation, matching mounted secret-file path, Prometheus reload, and concrete verification via curl and Prometheus targets." + "judgeNotes": "README covers Secret API key creation, matching password_file placement, Compose startup/reload, and concrete endpoint plus Prometheus target verification." } ], "skills": { @@ -4246,9 +4279,19 @@ }, "docs": { "calls": [ + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | head -60; echo \"---CLI---\"; supabase --version", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 3256 + }, { "source": "search_docs", - "query": "{ searchDocs(query: \"Prometheus metrics endpoint scrape project metrics\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Prometheus metrics endpoint scrape project metrics\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -4256,33 +4299,27 @@ "title": "Scrape a project's metrics" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-self-hosted", "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" + "url": "https://supabase.com/docs/guides/observability/metrics", + "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" + "url": "https://supabase.com/docs/guides/observability/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-cloud", "title": "Metrics API with Grafana Cloud" - } - ], - "resultChars": 20571 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/telemetry/metrics.md | head -150", - "pages": [ + }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics.md" + "url": "https://supabase.com/docs/guides/observability/access-data", + "title": "Observe the data" } ], - "resultChars": 14 + "resultChars": 23063 } ] }, @@ -4319,12 +4356,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "HTTPS Supabase scrape uses the correct metrics path, Basic Auth password_file, matching mounted secrets directory, valid project target, and preserves the app job." + "judgeNotes": "HTTPS Supabase scrape is correctly configured with the required path, project target, Basic Auth password_file, preserved app job, and matching Compose secret mount." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README provides Secret API key creation, matching secret-file placement, Compose startup/reload guidance, and concrete verification through curl, Prometheus targets, and Grafana." + "judgeNotes": "README provides correct Secret API key creation, matching secret-file placement, Prometheus recreation, endpoint/auth details, and concrete verification via curl, Prometheus targets, and PromQL/Grafana." } ], "skills": { @@ -4338,19 +4375,9 @@ }, "docs": { "calls": [ - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/telemetry/metrics.md | head -200", - "pages": [ - { - "url": "https://supabase.com/docs/guides/telemetry/metrics.md" - } - ], - "resultChars": 14 - }, { "source": "search_docs", - "query": "{ searchDocs(query: \"Prometheus metrics endpoint scrape project\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Prometheus metrics endpoint scrape project metrics\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -4358,43 +4385,37 @@ "title": "Scrape a project's metrics" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-self-hosted", "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", + "url": "https://supabase.com/docs/guides/observability/metrics", "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", + "url": "https://supabase.com/docs/guides/observability/metrics/vendor-agnostic", "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-cloud", "title": "Metrics API with Grafana Cloud" - } - ], - "resultChars": 20571 - }, - { - "source": "shell_fetch", - "query": "curl -sL https://supabase.com/docs/guides/telemetry/metrics.md | head -250", - "pages": [ + }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics.md" + "url": "https://supabase.com/docs/guides/observability/access-data", + "title": "Observe the data" } ], - "resultChars": 5200 + "resultChars": 23063 }, { "source": "shell_fetch", - "query": "curl -sL https://supabase.com/changelog.md 2>/dev/null | grep -i -E 'metric|prometheus|grafana|api key|breaking' | head -40", + "query": "curl -s https://supabase.com/docs/guides/telemetry/metrics.md | head -150", "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/telemetry/metrics.md" } ], - "resultChars": 7221 + "resultChars": 14 } ] }, @@ -4422,7 +4443,7 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "preserved existing app scrape job", @@ -4431,12 +4452,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "HTTPS Supabase scrape uses the correct metrics path, Basic Auth with password_file, matching volume mount, valid project target, and preserves the app job." + "judgeNotes": "HTTPS Supabase target, required metrics path, Basic Auth password_file, preserved app job, and matching Compose secret mount are all present." }, { "name": "documented live deployment and verification steps", - "passed": false, - "judgeNotes": "The secret setup is mismatched: `chmod 600` makes the host-owned file unreadable to the Prometheus container’s non-root user. Also, the README reloads Prometheus directly rather than restarting/recreating the Compose service with a verified readable secret mount." + "passed": true, + "judgeNotes": "README covers Secret API key creation, matching mounted secret file, Prometheus reload, and concrete verification via targets, PromQL, and direct HTTP 200." } ], "skills": { @@ -4452,7 +4473,7 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"Prometheus metrics endpoint scrape project metrics\", limit: 6) { nodes { title href content } } }", + "query": "{\n searchDocs(query: \"Prometheus metrics endpoint project metrics scrape\", limit: 6) {\n nodes {\n title\n href\n content\n }\n }\n}", "hasContent": true, "pages": [ { @@ -4460,34 +4481,49 @@ "title": "Scrape a project's metrics" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-self-hosted", "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" + "url": "https://supabase.com/docs/guides/observability/metrics", + "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" + "url": "https://supabase.com/docs/guides/observability/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-cloud", "title": "Metrics API with Grafana Cloud" + }, + { + "url": "https://supabase.com/docs/guides/observability/access-data", + "title": "Observe the data" } ], - "resultChars": 20571 + "resultChars": 23063 }, { "source": "web_fetch", - "query": "Extract the exact Prometheus metrics endpoint URL format, the authentication method (username/password), which plans it is available on, recommended scrape interval, and a sample prometheus.yml scrape_config. Quote exactly.", + "query": "Extract the full details of the Supabase project metrics endpoint: the exact URL format, authentication method (username/password, which credentials), the exact prometheus.yml scrape config example including metrics_path, scheme, basic_auth, scrape_interval recommendations, and any notes about which plans support it or rate limits.", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/telemetry/metrics.md" } ], - "resultChars": 935 + "resultChars": 1250 + }, + { + "source": "web_fetch", + "query": "List any entries or breaking changes related to the Metrics API, Prometheus metrics endpoint, /customer/v1/privileged/metrics, observability, or API key formats (sb_secret_, service_role, legacy anon/service_role JWT keys). Include dates.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 1189 } ] }, @@ -4549,9 +4585,13 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions environment variables secrets managing\", limit: 5) { nodes { title href content } } }", + "query": "{searchDocs(query:\"edge function secrets environment variables deploy\", limit:6){nodes{title href content}}}", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", + "title": "Inspecting edge function environment variables" + }, { "url": "https://supabase.com/docs/guides/functions/secrets", "title": "Environment Variables" @@ -4561,19 +4601,49 @@ "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", + "title": "Transcription Telegram Bot" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", + "title": "Streaming Speech with ElevenLabs" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/deployment/branching/configuration", + "title": "Configuration" } ], - "resultChars": 71163 + "resultChars": 51315 + }, + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | head -60", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 3012 + }, + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | grep -n -i -B4 \"Breaking Change\" | grep -i -E \"^##|Breaking|function|secret|env\" | head -60", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 9049 + }, + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | grep -n -i -E \"secret|env var|environment variable\" | head -30", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 1413 } ] }, @@ -4635,7 +4705,7 @@ "calls": [ { "source": "shell_fetch", - "query": "curl -s --max-time 25 https://supabase.com/changelog.md | head -60", + "query": "curl -s https://supabase.com/changelog.md 2>&1 | head -60", "pages": [ { "url": "https://supabase.com/changelog.md" @@ -4645,13 +4715,9 @@ }, { "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions secrets environment variables deploy\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Edge Functions managing secrets environment variables\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" - }, { "url": "https://supabase.com/docs/guides/functions/secrets", "title": "Environment Variables" @@ -4665,15 +4731,39 @@ "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + } + ], + "resultChars": 76808 + }, + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/docs/guides/functions/secrets.md | head -120", + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/secrets.md" + } + ], + "resultChars": 4463 + }, + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/docs/guides/functions/secrets.md | tail -40", + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/secrets.md" } ], - "resultChars": 69137 + "resultChars": 1248 } ] }, @@ -4735,33 +4825,55 @@ "calls": [ { "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | head -100", + "query": "curl -s https://supabase.com/changelog.md 2>&1 | head -60", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 5440 + "resultChars": 3012 }, { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/functions/secrets.md", + "source": "search_docs", + "query": "{ searchDocs(query: \"Edge Functions environment variables secrets manage\", limit: 6) { nodes { title href content } } }", + "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/secrets.md" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", + "title": "Inspecting edge function environment variables" + }, + { + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" } ], - "resultChars": 5105 + "resultChars": 73366 }, { "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/functions/cors.md | head -60", + "query": "curl -s https://supabase.com/docs/guides/functions/secrets.md", "pages": [ { - "url": "https://supabase.com/docs/guides/functions/cors.md" + "url": "https://supabase.com/docs/guides/functions/secrets.md" } ], - "resultChars": 3412 + "resultChars": 5105 } ] }, @@ -4820,7 +4932,19 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_fetch", + "query": "Extract the full current instructions for self-hosting Supabase with Docker: the exact steps, how secrets/API keys (JWT_SECRET, ANON_KEY, SERVICE_ROLE_KEY, publishable/secret keys) are meant to be generated, whether there is a setup script, and any security steps required before exposing to the internet. Quote commands verbatim.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/docker.md" + } + ], + "resultChars": 2311 + } + ] }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", @@ -4877,19 +5001,7 @@ ] }, "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "Extract the full self-hosting via Docker instructions verbatim: the git clone / file copy steps, the complete list of .env variables that must be changed for production (secrets), how ANON_KEY and SERVICE_ROLE_KEY are generated, how the dashboard is secured, and any security warnings about exposing ports/services. Include exact commands.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/docker.md" - } - ], - "resultChars": 2701 - } - ] + "calls": [] }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", @@ -4948,34 +5060,26 @@ "docs": { "calls": [ { - "source": "shell_fetch", - "query": "curl -s --max-time 30 https://supabase.com/docs/guides/self-hosting/docker.md 2>&1 | head -200", - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/docker.md" - } - ], - "resultChars": 9141 - }, - { - "source": "shell_fetch", - "query": "curl -s --max-time 30 https://supabase.com/docs/guides/self-hosting/docker.md 2>&1 | sed -n '200,420p'", + "source": "web_fetch", + "query": "Give the full self-hosting with Docker instructions verbatim: the git clone / download steps, the .env variables that must be set, how to generate API keys (anon/service_role or publishable/secret), securing the services, dashboard auth, and any notes about docker-compose files (s3, pooler, vector). Include all env var names mentioned.", + "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/self-hosting/docker.md" } ], - "resultChars": 12943 + "resultChars": 5242 }, { - "source": "shell_fetch", - "query": "curl -s --max-time 30 https://supabase.com/docs/guides/self-hosting/docker.md 2>&1 | sed -n '420,560p'", + "source": "web_fetch", + "query": "List any entries related to self-hosting, docker-compose, analytics/vector, JWT signing keys, API keys (publishable/secret keys), or breaking changes in the last year.", + "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker.md" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 6786 + "resultChars": 2091 } ] }, @@ -5035,7 +5139,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "Correctly diagnoses the ineffective profile-only soft delete, implements real auth revocation by banning the user and deleting sessions/refresh tokens, closes stale-JWT Data API access via live RLS checks, accurately explains remaining stateless-JWT windows elsewhere and mitigations, and correctly distinguishes frontend publishable keys from server-only RLS-bypassing secret keys." + "judgeNotes": "Correctly diagnoses the soft-delete-only flow, revokes sessions/refresh tokens and disables auth access, closes the demonstrated Data API window through live-state RLS checks, accurately notes stateless JWTs remain locally valid until expiry, and correctly distinguishes frontend publishable keys from server-only RLS-bypassing secret keys." } ], "skills": { @@ -5050,37 +5154,137 @@ "docs": { "calls": [ { - "source": "web_fetch", - "query": "Summarize the new API key system: publishable key vs secret key, what replaces anon and service_role, which one goes in a frontend/browser, how each interacts with RLS and Postgres roles, key prefixes/format, rotation/revocation abilities, and migration notes/timeline for legacy keys.", + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable key secret key API keys migration anon service_role RLS\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/api/api-keys.md" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", + "title": "Build a Social Auth App with Expo React Native" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", + "title": "Build a User Management App with SvelteKit" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-solidjs", + "title": "Build a User Management App with SolidJS" + } + ], + "resultChars": 195336 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"delete user access token still valid revoke sessions sign out user deletion\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", + "title": "OAuth 2.1 Flows" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-admin-signout", + "title": "signOut()" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signout" } ], - "resultChars": 1470 + "resultChars": 87787 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable and secret API keys\", limit: 10) { nodes { title href } } }", + "hasContent": false, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", + "title": "Build a Social Auth App with Expo React Native" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", + "title": "Creating a Supabase client for SSR" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + } + ], + "resultChars": 1319 }, { "source": "web_fetch", - "query": "List entries related to API keys (publishable/secret keys, legacy anon/service_role deprecation), JWT signing keys, user deletion, session revocation, and any breaking-change tags. Include dates.", + "query": "Summarize publishable keys (sb_publishable_...) vs secret keys (sb_secret_...) vs legacy anon/service_role. Which goes in the frontend, which Postgres role each maps to, how each interacts with RLS, rotation, and migration steps/deadlines.", "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/api/api-keys.md" } ], - "resultChars": 2201 + "resultChars": 1695 }, { "source": "web_fetch", - "query": "What is the default access token (JWT) expiry? How are sessions terminated/revoked? Does revoking a session or deleting a user immediately invalidate an already-issued access token? What does signOut with global scope do?", + "query": "What are the concrete migration steps from anon/service_role to publishable/secret keys? Any gotchas about JWT signing keys, verify_jwt, RLS behaviour differences, or things that break?", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/sessions.md" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys.md" } ], - "resultChars": 1298 + "resultChars": 1580 } ] }, @@ -5140,7 +5344,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "Correctly diagnosed the soft-delete-only flow, revoked sessions and refresh tokens while disabling the auth identity, closed the covered Data API path via live-state RLS checks, accurately explained residual stateless JWT validity, and correctly distinguished frontend publishable keys from server-only RLS-bypassing secret keys." + "judgeNotes": "Correctly diagnoses the profile-only soft delete, revokes sessions/refresh tokens and blocks future authentication, closes the Data API window via session-aware RLS, accurately preserves the local-JWT expiry caveat, and correctly distinguishes publishable frontend keys from secret server-only RLS-bypassing keys." } ], "skills": { @@ -5155,67 +5359,15 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "{ searchDocs(query: \"delete user account revoke sessions sign out invalidate JWT access token\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-admin-signout", - "title": "signOut()" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - } - ], - "resultChars": 87787 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key legacy anon service_role API keys migration\", limit: 6) { nodes { title href content } } }", + "source": "web_fetch", + "query": "Summarize: publishable key vs secret key vs legacy anon/service_role. Which goes in the frontend? What Postgres role does each map to and how does each interact with RLS? Does the secret key bypass RLS? Rotation, and migration notes.", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" + "url": "https://supabase.com/docs/guides/api/api-keys.md" } ], - "resultChars": 166856 + "resultChars": 1685 } ] }, @@ -5274,8 +5426,8 @@ }, { "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "Correctly diagnoses the soft-delete-only flow, revokes sessions and refresh tokens while blocking future authentication, closes stale-JWT Data API access through RLS, accurately explains remaining local JWT validation until expiry, and correctly distinguishes publishable and secret keys." + "passed": false, + "judgeNotes": "The flow still does not delete the auth user or remove their identity; it only applies a 100-year ban while deleting sessions/refresh tokens. That is access suspension, not the required real account deletion/identity removal. The JWT-window/RLS explanation and publishable-vs-secret key guidance are otherwise consistent and correct." } ], "skills": { @@ -5289,16 +5441,268 @@ }, "docs": { "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"delete user does not invalidate access token revoke sessions signout\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" + }, + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-admin-signout", + "title": "signOut()" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signout" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-signout" + } + ], + "resultChars": 30108 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable key secret key legacy anon service_role API keys migration RLS\", limit: 8) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", + "title": "Build a Social Auth App with Expo React Native" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", + "title": "Build a User Management App with SvelteKit" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-solidjs", + "title": "Build a User Management App with SolidJS" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-vue-3", + "title": "Build a User Management App with Vue 3" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-svelte", + "title": "Build a User Management App with Svelte" + } + ], + "resultChars": 240626 + }, + { + "source": "web_fetch", + "query": "Extract everything about the new API key system: publishable keys (sb_publishable_...) and secret keys (sb_secret_...). Specifically: which key is for frontend/browser vs server-only; what Postgres role each key maps to (anon? service_role?); whether secret keys bypass RLS and whether publishable keys bypass RLS; how the role changes when a user signs in (anon -> authenticated) and whether that differs between legacy anon key and publishable key; differences vs legacy JWT keys (rotation, revocation, expiry, multiple keys, are they JWTs); migration/compatibility (can both coexist, can legacy keys be disabled, deprecation timeline/deadline); whether publishable keys are safe to expose publicly; and recommended environment variable naming conventions. Quote key sentences verbatim where possible.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/api-keys.md" + } + ], + "resultChars": 2590 + }, + { + "source": "web_fetch", + "query": "Extract everything about securing the API: which keys are safe to expose in a browser, which must be server-side only, Postgres roles (anon, authenticated, service_role), RLS bypass behavior, publishable vs secret keys, and any env var naming guidance. Quote key sentences verbatim.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/securing-your-api.md" + } + ], + "resultChars": 1477 + }, + { + "source": "web_fetch", + "query": "Scan for entries related to API keys: publishable keys, secret keys, sb_publishable_, sb_secret_, legacy anon/service_role key deprecation, JWT signing keys, asymmetric JWTs. List each relevant entry with its date, title, and what it announced, including any breaking changes or deadlines.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 2258 + }, + { + "source": "web_fetch", + "query": "Answer precisely with VERBATIM quotes only (say \"NOT STATED\" if the doc does not say it):\n1. Exact quote about the deprecation timeline/deadline for anon and service_role keys. Is a specific date given?\n2. Exact quote about whether secret keys are blocked in browsers / User-Agent header / 401.\n3. Exact quote about whether publishable/secret keys are JWTs or not.\n4. Exact quotes about rotating, revoking, and having multiple keys.\n5. Exact quotes about disabling legacy keys.\n6. Exact quotes about anything that still requires the legacy JWT secret or JWT signing keys.\n7. Exact env var names recommended in the doc.\n8. Exact quote on role mapping for publishable key when signed in vs signed out, and for secret key.\n9. Does the doc say publishable key behavior differs from legacy anon key in terms of role resolution?", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/api-keys.md" + } + ], + "resultChars": 1459 + }, + { + "source": "web_search", + "query": "Supabase changelog publishable keys secret keys sb_publishable general availability legacy anon service_role deprecation", + "hasContent": false, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys | Supabase Docs" + }, + { + "url": "https://supabase.com/changelog/29260-upcoming-changes-to-supabase-api-keys", + "title": "Upcoming changes to Supabase API Keys · Changelog" + }, + { + "url": "https://supabase.com/changelog?next=Y3Vyc29yOnYyOpK0MjAyNC0wOS0xM1QxNTowMDoyOVrOAG2JYw%3D%3D&restPage=2", + "title": "Changelog" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/rotating-anon-service-and-jwt-secrets-1Jq6yd", + "title": "Supabase Docs | Troubleshooting | Rotating Anon, Service, and JWT Secrets" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/api/creating-routes", + "title": "Creating API Routes | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables | Supabase Docs" + } + ], + "resultChars": 2806 + }, + { + "source": "web_fetch", + "query": "Extract: relationship between JWT signing keys and the new publishable/secret API keys. What still requires the legacy JWT secret (symmetric HS256)? Any gotchas or features that don't work with new keys or asymmetric keys? Any deprecation timeline. Quote verbatim.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/signing-keys.md" + } + ], + "resultChars": 1587 + }, + { + "source": "web_fetch", + "query": "Extract the full announcement: date, what changed, timeline/deadlines for legacy anon and service_role key deprecation, rationale (rotation problems with JWT keys), what publishable and secret keys are, revocation, multiple keys, and any breaking-change dates. Quote verbatim where possible.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog/29260-upcoming-changes-to-supabase-api-keys" + } + ], + "resultChars": 215 + }, { "source": "web_fetch", - "query": "Summarize publishable vs secret API keys vs legacy anon/service_role keys: key name prefixes, which goes in a frontend/browser, which Postgres role each maps to, whether RLS applies to each, revocation/rotation behaviour, and any migration notes or deprecation timeline for the legacy keys.", + "query": "Extract precisely: role mapping for publishable vs secret keys (anon/authenticated/service_role); RLS bypass behavior; how sign-in changes the effective role and whether that differs vs legacy anon key; whether both key systems coexist; how to disable legacy keys; deprecation timeline; number of keys allowed / rotation / revocation; whether keys are JWTs; env var naming recommendations; and any gotchas such as things that still need the legacy JWT secret, Edge Functions verify_jwt, storage, realtime, or self-hosting caveats. Quote verbatim.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys.md" + } + ], + "resultChars": 2845 + }, + { + "source": "web_fetch", + "query": "Extract precisely: definitions of publishable and secret keys, which is browser-safe, Postgres role mapping, RLS behavior, whether they are JWTs, how many secret keys you can create, rotation/revocation/disabling, whether publishable keys can be revoked/rotated, and env var naming. Quote verbatim.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys.md" + } + ], + "resultChars": 1544 + }, + { + "source": "web_search", + "query": "supabase github discussions \"Upcoming changes to Supabase API Keys\" legacy anon service_role deprecation date", + "hasContent": false, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/rotating-anon-service-and-jwt-secrets-1Jq6yd", + "title": "Supabase Docs | Troubleshooting | Rotating Anon, Service, and JWT Secrets" + }, + { + "url": "https://supabase.com/blog/supabase-security-2025-retro", + "title": "Supabase Security Retro: 2025" + } + ], + "resultChars": 3142 + }, + { + "source": "web_fetch", + "query": "List ALL entries that mention any of: \"API key\", \"publishable\", \"secret key\", \"anon key\", \"service_role\", \"JWT signing key\", \"asymmetric\". For each give the exact date, exact title, and a one-line summary. Do not omit any. Include entries from 2025 and 2026.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 2485 + }, + { + "source": "web_fetch", + "query": "Give me the raw text of the entire page verbatim, as close to complete as possible. Do not summarize.", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/api/api-keys.md" } ], - "resultChars": 1282 + "resultChars": 22206 + }, + { + "source": "web_fetch", + "query": "Give me the raw text of the entire page verbatim, as close to complete as possible. Do not summarize.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys.md" + } + ], + "resultChars": 2185 } ] }, @@ -5353,7 +5757,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified missing publication membership, added only public.orders to supabase_realtime, and preserved existing tables, RLS, policies, and courier_locations." + "judgeNotes": "Correctly identified missing publication membership, added public.orders to the existing supabase_realtime publication, and preserved RLS, policies, and courier_locations." } ], "skills": { @@ -5419,7 +5823,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified orders missing from supabase_realtime, added only public.orders to the existing publication, and preserved courier_locations, policies, and RLS." + "judgeNotes": "Correctly identified missing publication membership, added only public.orders to supabase_realtime, and preserved RLS, policies, and courier_locations." } ], "skills": { @@ -5485,7 +5889,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified orders missing from supabase_realtime, added only public.orders to the existing publication, and preserved RLS, policies, and courier_locations." + "judgeNotes": "Correctly identified orders missing from supabase_realtime, added only public.orders to the existing publication, verified both feeds remained published, and left RLS and policies unchanged." } ], "skills": { @@ -5498,32 +5902,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"enable realtime postgres changes publication supabase_realtime add table\", limit: 4) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", - "title": "Subscribing to Database Changes" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/realtime-postgres-changes-troubleshooting", - "title": "Realtime: Postgres Changes Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks", - "title": "Benchmarks" - } - ], - "resultChars": 73159 - } - ] + "calls": [] }, "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", @@ -5554,17 +5933,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform and the recurring 8 HTTP 503 gateway failures across 07:00Z–12:00Z on 2026-04-28." + "judgeNotes": "Identified image-transform and clearly documented all 8 recurring HTTP 503 gateway failures from 07:00Z through 12:00Z on 2026-04-28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Correctly attributes the 503s to the gateway/platform layer, supported by missing function invocation/runtime logs, unchanged deployment version across successful requests, and contrast with avatar-upload’s function-level 500." + "judgeNotes": "Correctly attributes the recurring 503s to the gateway/platform layer, supported by their presence only in gateway logs, absence of runtime invocations, nearby successful calls, and unchanged deployment/version. It also distinguishes the avatar-upload function-level 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Recommended concrete actions: open a Supabase support ticket with gateway request IDs and time window, check concurrency/resource limits, add 503 retry backoff, and separately investigate the application-level 500." + "judgeNotes": "Recommended opening a Supabase support ticket with specific gateway request IDs, project, function, and time window, plus checking current status and adding retry backoff." } ], "skills": { @@ -5608,17 +5987,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform as affected and clearly described all 8 recurring HTTP 503 failures from 07:00Z through 12:00Z on 2026-04-28." + "judgeNotes": "Identified image-transform and detailed all 8 recurring HTTP 503 gateway failures from 07:00Z through 12:00Z on 2026-04-28, while correctly separating the older billing-webhook errors." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Attributes the 503s to the edge gateway before handler execution, supported by gateway-only failure rows with no runtime/invocation metadata, nearby successful invocations on unchanged version 42, and contrast with avatar-upload’s genuine function-level 500." + "judgeNotes": "Attributes the 503s to the gateway/platform layer and supports this with missing runtime invocation rows for failures, contrasted with successful nearby invocations and a distinct function-level 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Recommended multiple concrete actions: inspect Edge Function boot logs at failure timestamps, pin dependencies, check runtime resource ceilings, add retry/backoff, and investigate changes around 07:00." + "judgeNotes": "Recommended opening a Supabase support ticket with specific gateway request IDs and missing runtime-log evidence, plus checking function resource limits and adding explicit 503 retry backoff." } ], "skills": { @@ -5657,22 +6036,22 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform and the recurring eight HTTP 503 gateway failures across 07:00Z–12:00Z on 2026-04-28, while correctly distinguishing older billing-webhook errors." + "judgeNotes": "Identified image-transform and the recurring eight HTTP 503 gateway failures across 07:00Z–12:00Z on 2026-04-28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "Correctly identifies gateway-layer 503s using missing invocation/runtime rows, but explicitly recommends changing and redeploying image-transform, which the rubric defines as a failure." + "passed": true, + "judgeNotes": "Correctly attributes the 503s to the gateway/platform layer, supported by gateway-only 503 rows with no matching invocation records, unchanged deployment/version across successful calls, and distinction from avatar-upload’s function-level 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Recommended concrete actions: verify a current time window, pin the dependency, redeploy, and open a Supabase support ticket with the eight gateway request IDs." + "judgeNotes": "Recommended concrete actions: open a Supabase support ticket with gateway request IDs and incident context, add retry/backoff mitigation, separately investigate the application-level 500, and confirm log sampling." } ], "skills": { @@ -5743,7 +6122,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly identified deny-all RLS with no policies, kept RLS enabled, and created authenticated-only SELECT and INSERT policies scoped by user_id = auth.uid(), using USING and WITH CHECK respectively." + "judgeNotes": "Correctly identified deny-all RLS with no policies, kept RLS enabled, and added authenticated owner-scoped SELECT and INSERT policies using auth.uid() with USING/WITH CHECK." } ], "skills": { @@ -5756,28 +6135,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"auth.uid() function definition request.jwt.claims sub\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-getclaims", - "title": "getClaims()" - } - ], - "resultChars": 50036 - } - ] + "calls": [] }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", @@ -5835,7 +6193,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly diagnosed deny-all RLS, retained RLS, and added authenticated owner-scoped SELECT and INSERT policies using auth.uid(), with verification." + "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and added authenticated owner-scoped SELECT and INSERT policies using auth.uid() with USING and WITH CHECK." } ], "skills": { @@ -5906,7 +6264,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly diagnosed deny-all RLS, kept RLS enabled, and created authenticated-only SELECT and INSERT policies scoped by user_id = auth.uid() using USING and WITH CHECK." + "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and created authenticated owner-scoped SELECT and INSERT policies using auth.uid() with USING and WITH CHECK." } ], "skills": { @@ -5919,28 +6277,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"auth.uid() function definition request.jwt.claims sub\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-getclaims", - "title": "getClaims()" - } - ], - "resultChars": 50036 - } - ] + "calls": [] }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", @@ -5991,7 +6328,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "`supabase db push --yes` applied `20240220000000_add_avatar_url.sql` and finished successfully. History was reconciled by adding local `20240115000000_add_profile_bio.sql`, after which the CLI push succeeded. The `psql` commands were read-only; no prohibited workaround was used." + "judgeNotes": "`supabase db push` (#23) applied `20240220000000_add_avatar_url.sql` and finished successfully. The orphan bio history was reconciled by adding `20240115000000_add_profile_bio.sql` locally (#16), after which the CLI push succeeded and migration lists matched (#24). No prohibited workaround was used." } ], "skills": { @@ -6004,36 +6341,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Remote migration versions not found in local migrations directory repair\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", - "title": "Repair the migration history table" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - } - ], - "resultChars": 69700 - } - ] + "calls": [] }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", @@ -6084,7 +6392,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "`supabase db push` applied `20240220000000_add_avatar_url.sql` and finished successfully. The orphan history was reconciled by adding local `20240115000000_add_profile_bio.sql`, after which the push succeeded and migration lists matched. The `psql` commands were read-only; no prohibited workaround was used." + "judgeNotes": "`supabase db push` (#24) applied `20240220000000_add_avatar_url.sql` and finished successfully. History was reconciled by adding local `20240115000000_add_profile_bio.sql` (#18), then letting the CLI migration workflow recognize it and push (#19/#24). No prohibited workaround was used; remote `psql` commands were read-only." } ], "skills": { @@ -6148,7 +6456,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "`supabase db push` (#26) applied `20240220000000_add_avatar_url.sql` and finished successfully. History was reconciled by adding the missing local `20240115000000_add_profile_bio.sql` (#22), after which the push succeeded and migration lists matched (#28). No prohibited direct-SQL or prepared-statement workaround was used." + "judgeNotes": "`supabase db push --yes` applied `20240220000000_add_avatar_url.sql` successfully. The orphan bio migration was reconciled by adding local `20240115000000_add_profile_bio.sql`, after which the push and migration list aligned histories. No prohibited workaround was used; psql commands were read-only." } ], "skills": { @@ -6161,36 +6469,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Remote migration versions not found in local migrations directory repair\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", - "title": "Repair the migration history table" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - } - ], - "resultChars": 69700 - } - ] + "calls": [] }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", @@ -6234,7 +6513,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -6295,7 +6574,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -6308,7 +6587,6 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", "supabase-postgres-best-practices" ] }, @@ -6370,7 +6648,6 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", "supabase-postgres-best-practices" ] }, @@ -6525,8 +6802,7 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { @@ -6646,7 +6922,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user c231a620-b42d-459f-8771-4a83659f8185, signUp returned {\"userId\":\"c231a620-b42d-459f-8771-4a83659f8185\"}" + "notes": "db user c8023535-add9-4930-935f-2e8e61989a21, signUp returned {\"userId\":\"c8023535-add9-4930-935f-2e8e61989a21\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -6661,7 +6937,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"c231a620-b42d-459f-8771-4a83659f8185\"}" + "notes": "{\"userId\":\"c8023535-add9-4930-935f-2e8e61989a21\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -6723,7 +6999,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 22c19ee4-1239-4db7-876b-14d09e5208e4, signUp returned {\"userId\":\"22c19ee4-1239-4db7-876b-14d09e5208e4\"}" + "notes": "db user 80bf8d7f-d167-405d-8fde-3a2e980b147e, signUp returned {\"userId\":\"80bf8d7f-d167-405d-8fde-3a2e980b147e\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -6738,7 +7014,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"22c19ee4-1239-4db7-876b-14d09e5208e4\"}" + "notes": "{\"userId\":\"80bf8d7f-d167-405d-8fde-3a2e980b147e\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -6800,7 +7076,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user f36a02e5-62fd-4a8e-97f5-49cbede5a9b4, signUp returned {\"userId\":\"f36a02e5-62fd-4a8e-97f5-49cbede5a9b4\"}" + "notes": "db user 382891d1-cd9e-4b5f-8ba1-cb550b0364fa, signUp returned {\"userId\":\"382891d1-cd9e-4b5f-8ba1-cb550b0364fa\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -6815,7 +7091,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"f36a02e5-62fd-4a8e-97f5-49cbede5a9b4\"}" + "notes": "{\"userId\":\"382891d1-cd9e-4b5f-8ba1-cb550b0364fa\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -6946,7 +7222,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 2 rows" + "notes": "found 3 rows" }, { "name": "row level security is enabled on todos", @@ -6959,12 +7235,12 @@ { "name": "REST API returns no todos to anonymous requests", "passed": true, - "notes": "0 rows" + "notes": "error 42501: permission denied for table todos" }, { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "2 rows" + "notes": "3 rows" } ], "skills": { @@ -6972,7 +7248,46 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md 2>/dev/null | head -60; echo \"=== EXIT $? ===\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 3028 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"RLS policy read-only table authenticated users select only no write policies\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", + "title": "Column Level Security" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" + }, + { + "url": "https://supabase.com/docs/guides/observability/advisors?queryGroups=lint&lint=0024_permissive_rls_policy", + "title": "Database Advisor: Lint 0024_permissive_rls_policy" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" + } + ], + "resultChars": 56158 + } + ] }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", @@ -7013,7 +7328,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 2 rows" + "notes": "found 3 rows" }, { "name": "row level security is enabled on todos", @@ -7031,7 +7346,7 @@ { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "2 rows" + "notes": "3 rows" } ], "skills": { @@ -7237,7 +7552,7 @@ { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 4) from the queue" + "notes": "function removed the seeded message (id 3) from the queue" } ], "skills": { @@ -7248,20 +7563,24 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"cron job send message to queue pgmq schedule\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pg_cron schedule job queue pgmq send message every minute\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, { "url": "https://supabase.com/docs/guides/queues/pgmq", "title": "PGMQ Extension" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" }, { "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", @@ -7272,27 +7591,39 @@ "title": "Install" } ], - "resultChars": 72816 + "resultChars": 47784 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"edge function read messages from queue and delete pgmq_public\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"queues edge function read delete messages pgmq_public supabase-js\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", "title": "Consuming Supabase Queue Messages with Edge Functions" }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, { "url": "https://supabase.com/docs/guides/queues/api", "title": "API" + }, + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", + "title": "Resumable WebSockets with Edge Functions" } ], - "resultChars": 28293 + "resultChars": 33843 } ] }, @@ -7339,7 +7670,7 @@ { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 29) from the queue" + "notes": "function removed the seeded message (id 10) from the queue" } ], "skills": { @@ -7350,67 +7681,304 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"Queues quickstart pgmq_public read delete edge function\", limit: 6) { nodes { title href content } } }", + "query": "{\n searchDocs(query: \"pg_cron schedule job send message to queue pgmq\", limit: 6) {\n nodes {\n title\n href\n content\n }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" }, { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" + "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", + "title": "pg_cron debugging guide" }, { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" } ], - "resultChars": 54557 + "resultChars": 83282 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron schedule job every minute pgmq send queue\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Queues pgmq create queue\", limit: 3) { nodes { title href subsections { nodes { title content } } } } }", "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"cron.schedule job name every minute\", limit: 3) { nodes { title href } } }", + "hasContent": false, "pages": [ + { + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" + }, { "url": "https://supabase.com/docs/guides/database/extensions/pg_net", "title": "pg_net: Async Networking" }, + { + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" + } + ], + "resultChars": 390 + }, + { + "source": "web_fetch", + "query": "Extract verbatim all SQL and code snippets: how to create a queue (exact function name and signature), enqueue/send, read, pop, delete, archive. Also any mention of pgmq_public schema, supabase-js usage, and which key/role is required. Quote exactly.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/quickstart.md" + } + ], + "resultChars": 1499 + }, + { + "source": "web_fetch", + "query": "Extract verbatim the cron.schedule signatures and examples, including named jobs, every-minute schedules, SQL snippet jobs, and any statement about re-running cron.schedule with the same job name (upsert/replace vs duplicate). Also cron.unschedule and cron.alter_job.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/cron/quickstart.md" + } + ], + "resultChars": 1540 + }, + { + "source": "web_fetch", + "query": "Extract verbatim the exact SQL for creating a queue (pgmq.create vs pgmq.create_queue), and signatures for send, read, pop, delete, archive. Include the pgmq_public wrapper functions if mentioned.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/pgmq.md" + } + ], + "resultChars": 997 + }, + { + "source": "web_fetch", + "query": "List verbatim the environment variables automatically available in Edge Functions (default secrets), exact names.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/secrets.md" + } + ], + "resultChars": 467 + }, + { + "source": "web_fetch", + "query": "Extract the [functions.] config.toml block: keys like verify_jwt, import_map, entrypoint, and the DEFAULT value of verify_jwt. Quote exactly.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/cli/config.md" + } + ], + "resultChars": 215 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"config.toml functions verify_jwt\", limit: 4) { nodes { title href } } }", + "hasContent": false, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" + }, + { + "url": "https://supabase.com/docs/guides/functions/deploy", + "title": "Deploy to Production" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-tips", + "title": "Development tips" + }, + { + "url": "https://supabase.com/docs/guides/ai-tools/byo-mcp", + "title": "Deploy MCP servers" + } + ], + "resultChars": 492 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"cron job send message to queue pgmq.send schedule\", limit: 5) { nodes { title href } } }", + "hasContent": false, + "pages": [ { "url": "https://supabase.com/docs/guides/queues/pgmq", "title": "PGMQ Extension" }, { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/realtime/broadcast", + "title": "Broadcast" + }, + { + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + } + ], + "resultChars": 554 + }, + { + "source": "web_fetch", + "query": "Extract verbatim the [functions.] config.toml block syntax and all keys (verify_jwt, import_map, entrypoint, static_files). State the DEFAULT value of verify_jwt and any notes about --no-verify-jwt CLI flag for serve/deploy.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/function-configuration.md" + } + ], + "resultChars": 1009 + }, + { + "source": "web_fetch", + "query": "Extract verbatim: the pgmq_public schema wrapper function list with exact argument names (send, send_batch, read, pop, archive, delete), how to enable the Queues Data API / exposing pgmq_public, and which roles/keys are required (anon vs authenticated vs service_role) and RLS/grants notes.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/api.md" + } + ], + "resultChars": 1160 + }, + { + "source": "web_fetch", + "query": "Extract verbatim: (a) the cron.schedule call(s) and whether the scheduled SQL calls pgmq.send / util.queue_embeddings directly, (b) any pgmq.create/create queue SQL, (c) the Edge Function code that reads from the queue (imports, createClient options, rpc names), (d) which key it uses (service role vs anon).", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings.md" + } + ], + "resultChars": 1269 + }, + { + "source": "web_search", + "query": "Supabase docs Queues cron pgmq.send cron.schedule example combining cron and queues", + "hasContent": false, + "pages": [ + { + "url": "https://supabase.com/modules/cron", + "title": "Supabase Cron | Schedule Recurring Jobs in Postgres" + }, + { + "url": "https://supabase.com/blog/supabase-cron", + "title": "Supabase Cron" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_cron", + "title": "pg_cron: Schedule Recurring Jobs with Cron Syntax in Postgres | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgmq", + "title": "pgmq: Queues | Supabase Docs" }, { "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" + "title": "Scheduling Edge Functions | Supabase Docs" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings | Supabase Docs" }, { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues | Supabase Docs" + }, + { + "url": "https://supabase.com/blog/supabase-queues", + "title": "Supabase Queues" + }, + { + "url": "https://supabase.com/blog/processing-large-jobs-with-edge-functions", + "title": "Processing large jobs with Edge Functions, Cron, and Queues" + } + ], + "resultChars": 2722 + }, + { + "source": "web_fetch", + "query": "Extract verbatim: cron.schedule SQL that calls pgmq.send / queue functions directly; the queue creation SQL; and the Edge Function code that reads from the queue (imports, createClient with db.schema pgmq_public, rpc('read'/'delete') arg names, which key is used).", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/blog/processing-large-jobs-with-edge-functions" + } + ], + "resultChars": 1058 + }, + { + "source": "web_fetch", + "query": "Extract verbatim the recommended import specifier for supabase-js in a Deno edge function (npm: vs jsr: vs esm.sh) and any example createClient code with Deno.env.get. Also note anything about deno.json / import maps.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/quickstart.md" + } + ], + "resultChars": 632 + }, + { + "source": "web_fetch", + "query": "Extract verbatim guidance on importing npm/JSR packages in edge functions: recommended specifier form (npm:, jsr:, esm.sh, deno.land), examples with @supabase/supabase-js, and deno.json import map examples.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/dependencies.md" + } + ], + "resultChars": 1166 + }, + { + "source": "web_fetch", + "query": "Find the [functions.] section. Report exactly: the list of keys (enabled, verify_jwt, import_map, entrypoint, static_files), and the stated DEFAULT value for verify_jwt. Quote the default line verbatim.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/cli/config" + } + ], + "resultChars": 600 + }, + { + "source": "web_fetch", + "query": "Show the createClient options object, specifically the db: { schema: '...' } option. Quote the example verbatim.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/initializing" } ], - "resultChars": 51570 + "resultChars": 546 } ] }, @@ -7442,22 +8010,12 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" - }, - { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 1 -> 2" - }, - { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 5) from the queue" + "name": "scorer completed without errors", + "passed": false, + "notes": "query failed: ERROR: relation \"cron.job\" does not exist\nLINE 1: ...), '[]'::json) from (select schedule, active from cron.job w...\n ^\n" } ], "skills": { @@ -7468,27 +8026,67 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"queues read delete messages edge function pgmq_public rpc\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pg_cron schedule job queue pgmq send message\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, { "url": "https://supabase.com/docs/guides/queues/pgmq", "title": "PGMQ Extension" }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", + "title": "pg_cron debugging guide" + }, + { + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" + }, + { + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" + }, { "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", "title": "Automatic embeddings" + } + ], + "resultChars": 74440 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Supabase Queues create queue pgmq quickstart\", limit: 2) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + } + ], + "resultChars": 12925 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"cron.schedule job every minute pg_cron quickstart\", limit: 2) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" } ], - "resultChars": 60050 + "resultChars": 15102 } ] }, @@ -8206,274 +8804,42 @@ "available": [], "loaded": [] }, - "docs": { - "calls": [] - }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "run": 3, - "sourcePath": "claude-code-opus-5-no-skills/build-functions-004-service-role-bypass/run-3/result.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "sdk", - "rls", - "security" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"error\":\"Missing credentials\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"a8093f98-061b-40e6-8e63-6157b3515074\",\"metric\":\"steps_a_mta4m48b\",\"value\":111}]" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: [{\"user_id\":\"a8093f98-061b-40e6-8e63-6157b3515074\",\"metric\":\"steps_a_mta4m48b\",\"value\":111}]" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"ee23efe4-c8c8-4bf5-bbf2-e4843f6105c1\",\"metric\":\"steps_b_mta4m48b\",\"value\":222}]" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"error\":\"Missing credentials\"}" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"error\":\"Invalid or expired access token\"}" - }, - { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 401: {\"error\":\"Missing credentials\"}" - }, - { - "name": "implementation uses @supabase/server", - "passed": false, - "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" - } - ], - "skills": { - "available": [], - "loaded": [] - }, "docs": { "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"edge function verify_jwt service role key apikey header authorization\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"edge function auth user RLS Authorization header SUPABASE_ANON_KEY environment variables\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - } - ], - "resultChars": 99771 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key new API keys legacy anon service_role\", limit: 3) { nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", "title": "New API Keys and Asymmetric Authentication" } ], - "resultChars": 456 - }, - { - "source": "web_fetch", - "query": "Extract everything about publishable keys (sb_publishable_), secret keys (sb_secret_), legacy anon/service_role JWT keys: prefixes, how to distinguish them, whether secret keys are JWTs, how they are sent (apikey header vs Authorization), and any mention of environment variables in Edge Functions.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/api-keys" - } - ], - "resultChars": 1593 - }, - { - "source": "web_fetch", - "query": "List the exact default/built-in secrets and environment variables available in Supabase Edge Functions (e.g. SUPABASE_URL, SUPABASE_ANON_KEY, SUPABASE_SERVICE_ROLE_KEY, SUPABASE_PUBLISHABLE_KEY, SUPABASE_SECRET_KEY, SUPABASE_DB_URL). Quote the list verbatim.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets" - } - ], - "resultChars": 831 - }, - { - "source": "web_fetch", - "query": "Quote verbatim the full table/list of default secrets including SUPABASE_PUBLISHABLE_KEYS and SUPABASE_SECRET_KEYS. Show the exact JSON structure/format of those variables and any code example showing how to read them.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets.md" - } - ], - "resultChars": 1155 - }, - { - "source": "web_fetch", - "query": "Explain exactly what the Edge Functions gateway does with the apikey header vs the Authorization header, when verify_jwt is true vs false. Include the exact supabase/config.toml syntax for disabling JWT verification per function, and any code showing how to create a supabase client with the caller's Authorization header for RLS.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth" - } - ], - "resultChars": 1400 - }, - { - "source": "web_fetch", - "query": "Quote the exact config.toml reference entries for functions: functions..verify_jwt, .enabled, .import_map, .entrypoint. Include the default value of verify_jwt and any notes about deprecation or CLI version requirements.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli/config" - } - ], - "resultChars": 1217 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge functions verify_jwt apikey header gateway authorization legacy anon key\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - } - ], - "resultChars": 71357 - }, - { - "source": "web_fetch", - "query": "What does it say specifically about Edge Functions and the new publishable/secret keys? Does the gateway still require an apikey header when verify_jwt is false? What happens if you send sb_secret_ or sb_publishable_ in the Authorization header? Also: can secret keys be distinguished from publishable keys purely by prefix, and are legacy anon/service_role keys distinguishable by decoding the JWT 'role' claim?", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys" - } - ], - "resultChars": 1196 - }, - { - "source": "web_fetch", - "query": "Quote verbatim anything about: verify_jwt = true accepting the anon/publishable key as a valid JWT (i.e. that platform JWT verification does not mean a real user), the difference between gateway-level verification and in-function verification, and any warning about not trusting the Authorization header. Also quote the recommended package/approach for edge function auth.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth.md" - } - ], - "resultChars": 1024 - }, - { - "source": "web_search", - "query": "Supabase edge function verify_jwt true anon key valid JWT accepted gateway pitfall", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions | Supabase Docs" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration | Supabase Docs" - } - ], - "resultChars": 3920 - }, - { - "source": "web_fetch", - "query": "Quote the exact config.toml example for verify_jwt per function and any statement about what JWT verification at the gateway accepts (e.g. anon key counts as a valid JWT), and about the apikey header being required.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/function-configuration" - } - ], - "resultChars": 938 + "resultChars": 45281 } ] }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "run": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-functions-005-dual-auth-user-secret/run-1/result.json" + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "run": 3, + "sourcePath": "claude-code-opus-5-no-skills/build-functions-004-service-role-bypass/run-3/result.json" }, { "experiment": "claude-code-opus-5-no-skills", @@ -8502,9 +8868,49 @@ "passed": false, "checks": [ { - "name": "scorer completed without errors", + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" + }, + { + "name": "rejects request with no credentials", + "passed": true, + "notes": "status 401: {\"error\":\"Missing credentials: send a user access token in Authorization, or the service-role key in apikey\"}" + }, + { + "name": "user with JWT reads only their own rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"71ad008f-81f9-4a16-89fe-c88937112710\",\"metric\":\"steps_a_mtsg6a4x\",\"value\":111}]" + }, + { + "name": "user cannot read another user's rows by passing user_id", + "passed": false, + "notes": "status 403: {\"error\":\"Not allowed to read another user's stats\"}" + }, + { + "name": "service key bypasses RLS to read the target user's rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"ae4a3a22-6e3f-4f50-8bdc-6180caebed4a\",\"metric\":\"steps_b_mtsg6a4x\",\"value\":222}]" + }, + { + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 401: {\"error\":\"Missing credentials: send a user access token in Authorization, or the service-role key in apikey\"}" + }, + { + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 401: {\"error\":\"Invalid or expired access token\"}" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 401: {\"error\":\"Missing credentials: send a user access token in Authorization, or the service-role key in apikey\"}" + }, + { + "name": "implementation uses @supabase/server", "passed": false, - "notes": "could not read `supabase status`: WARN: config section [inbucket] is deprecated. Please use [local_smtp] instead.\nsupabase_db_sandbox-dual-auth container is not ready: starting\nTry rerunning the command with --debug to troubleshoot the error.\n" + "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" } ], "skills": { @@ -8515,43 +8921,329 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server package edge functions API keys secret key publishable key\", limit: 8) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"edge function service role key apikey header dual auth verify_jwt\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" + } + ], + "resultChars": 71623 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge functions default environment variables SUPABASE_URL SUPABASE_SERVICE_ROLE_KEY secrets\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, { "url": "https://supabase.com/docs/guides/functions/auth", "title": "Securing Edge Functions" + } + ], + "resultChars": 41771 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Authorization headers verify_jwt apikey header edge functions gateway check\", limit: 4) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" + }, + { + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" + } + ], + "resultChars": 49168 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge functions import supabase-js npm jsr esm.sh specifier deno managing dependencies\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/dependencies", + "title": "Managing dependencies" + }, + { + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", + "title": "Streaming Speech with ElevenLabs" + } + ], + "resultChars": 29171 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"auth getClaims verify JWT edge function getUser service role client bypass RLS\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-getclaims" + } + ], + "resultChars": 7723 + } + ] + }, + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + "run": 1, + "sourcePath": "claude-code-opus-5-no-skills/build-functions-005-dual-auth-user-secret/run-1/result.json" + }, + { + "experiment": "claude-code-opus-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-5", + "reasoningEffort": "high" + }, + "eval": "build-functions-005-dual-auth-user-secret", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "sdk", + "rls", + "security" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" + }, + { + "name": "rejects request with no credentials", + "passed": true, + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"MISSING_CREDENTIALS\",\"message\":\"[@supabase/server] No credentials found on the request. This endpoint accepts auth mode(s):" + }, + { + "name": "user with JWT reads only their own rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"7751498b-de4b-4979-ba43-1df93f8b6451\",\"metric\":\"steps_a_mtsg74vi\",\"value\":111}]" + }, + { + "name": "user cannot read another user's rows by passing user_id", + "passed": true, + "notes": "status 200: [{\"user_id\":\"7751498b-de4b-4979-ba43-1df93f8b6451\",\"metric\":\"steps_a_mtsg74vi\",\"value\":111}]" + }, + { + "name": "service key bypasses RLS to read the target user's rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"51c9e30a-1c99-4232-a1f1-6c8987816dc4\",\"metric\":\"steps_b_mtsg74vi\",\"value\":222}]" + }, + { + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" + }, + { + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_JWT\",\"message\":\"[@supabase/server] The JWT in the Authorization header failed verification: its header is missing \\" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" + }, + { + "name": "implementation uses @supabase/server", + "passed": true, + "notes": "imports @supabase/server / withSupabase" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function verify_jwt authorization header service role key auth.getUser\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-getclaims" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-listusers" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + } + ], + "resultChars": 20408 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"new API keys publishable secret key sb_secret edge function environment variables\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "title": "API keys" }, { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + } + ], + "resultChars": 80732 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Securing Edge Functions service role secret key apikey header trusted service\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/guides/auth/signing-keys", "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/api/creating-routes", - "title": "Creating API Routes" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + } + ], + "resultChars": 70719 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"@supabase/server createClient edge function dual auth\", limit: 2) { nodes { title href } } }", + "hasContent": false, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" } ], - "resultChars": 91817 + "resultChars": 321 + }, + { + "source": "web_fetch", + "query": "Extract the full guidance on securing edge functions: how to handle a function called BOTH by signed-in users (user JWT in Authorization) and by trusted backend services (secret/service-role key in apikey header). Include all code samples verbatim, the recommended verify_jwt setting, how to compare/verify the secret key safely, and any warnings about what NOT to do.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth.md" + } + ], + "resultChars": 1765 + }, + { + "source": "web_fetch", + "query": "Give the complete reference for the @supabase/server withSupabase API: all auth modes ('user','secret','publishable','none'), the shape of ctx (supabase, supabaseAdmin, authMode, user, claims), what happens on auth failure (status codes), and every code example verbatim. Also any notes about config.toml verify_jwt / auth settings per function.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth" + } + ], + "resultChars": 1564 } ] }, @@ -8594,37 +9286,37 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"error\":\"unauthorized\"}" + "notes": "status 401: {\"error\":\"Missing credentials.\"}" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"7c40287b-c39c-4cb5-b74e-f4d3022e3410\",\"metric\":\"steps_a_mta4kfko\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"db8bcc56-1658-4d93-a3a5-93577559d7e5\",\"metric\":\"steps_a_mtsg2bmd\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": false, - "notes": "status 403: {\"error\":\"forbidden\"}" + "notes": "status 403: {\"error\":\"Not allowed to read another user's stats.\"}" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"87c67823-08aa-4abd-a868-5c595d6b72f3\",\"metric\":\"steps_b_mta4kfko\",\"value\":222}]" + "notes": "status 200: [{\"user_id\":\"5a600e60-bd9e-4e10-81c9-872820be3f8d\",\"metric\":\"steps_b_mtsg2bmd\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"error\":\"unauthorized\"}" + "notes": "status 401: {\"error\":\"Missing credentials.\"}" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"error\":\"unauthorized\"}" + "notes": "status 401: {\"error\":\"Invalid user token.\"}" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"error\":\"unauthorized\"}" + "notes": "status 401: {\"error\":\"Missing credentials.\"}" }, { "name": "implementation uses @supabase/server", @@ -8637,7 +9329,70 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function service role key verify user JWT auth getUser\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-getclaims" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + } + ], + "resultChars": 62706 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"new API keys publishable secret key sb_secret apikey header\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" + } + ], + "resultChars": 149020 + } + ] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", @@ -8970,7 +9725,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 01a03e36-d302-7258-9bcd-557fdaa978a9/receipt-alpha.pdf, 01a03e36-d302-7258-9bcd-557fdaa978a9/receipt-beta.pdf" + "notes": "saw: 01a0803f-a982-7068-8303-6bc5438490f1/receipt-alpha.pdf, 01a0803f-a982-7068-8303-6bc5438490f1/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -8991,7 +9746,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT policies, RLS retained, and short-lived createSignedUrl sharing are all correctly provided." + "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT policies with RLS retained, and expiring createSignedUrl client code are all provided. No prohibited public/service-role pattern." } ], "skills": { @@ -9002,48 +9757,55 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"storage RLS policy user folder ownership private bucket\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"storage RLS policy user can only access own folder user id\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, { "url": "https://supabase.com/docs/guides/storage/security/access-control", "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/storage/security/ownership", + "title": "Ownership" }, { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" } ], - "resultChars": 32975 + "resultChars": 66163 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"createSignedUrl signed url expires temporary share file\", limit: 3) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"createSignedUrl storage signed url expires\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" + }, { "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" }, { "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" + }, { "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" } ], - "resultChars": 2612 + "resultChars": 7638 } ] }, @@ -9090,7 +9852,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 01a03e37-124a-7589-b3f3-b4fd5fe97195/receipt-alpha.pdf, 01a03e37-124a-7589-b3f3-b4fd5fe97195/receipt-beta.pdf" + "notes": "saw: 01a0803e-b3de-7301-9f86-1cb260ba89f0/receipt-alpha.pdf, 01a0803e-b3de-7301-9f86-1cb260ba89f0/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -9111,7 +9873,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT policies with RLS enabled, and expiring createSignedUrl code are all provided. No prohibited public or service-role client access." + "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT RLS policies, RLS retained, and short-lived createSignedUrl sharing code are all provided." } ], "skills": { @@ -9122,66 +9884,35 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"storage RLS policy user can only access own folder user id first path segment\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"storage RLS policy private bucket user folder owner auth.uid signed url\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" - }, - { - "url": "https://supabase.com/docs/guides/storage/debugging/error-codes", - "title": "Error Codes" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" }, { "url": "https://supabase.com/docs/guides/storage/security/access-control", "title": "Storage Access Control" - } - ], - "resultChars": 40293 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"storage helper functions allow_only_operation allow_any_operation foldername\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - } - ], - "resultChars": 3819 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"createSignedUrl expiring temporary link private bucket RLS policy required\", limit: 4) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" }, { - "url": "https://supabase.com/docs/guides/storage/cdn/smart-cdn", - "title": "Smart CDN" + "url": "https://supabase.com/docs/guides/storage/debugging/error-codes", + "title": "Error Codes" + }, + { + "url": "https://supabase.com/docs/guides/storage/schema/design", + "title": "The Storage Schema" } ], - "resultChars": 17698 + "resultChars": 35946 } ] }, @@ -9228,7 +9959,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 01a03e37-6635-779f-a9ce-2a4f75f709a3/receipt-alpha.pdf, 01a03e37-6635-779f-a9ce-2a4f75f709a3/receipt-beta.pdf" + "notes": "saw: 01a08040-2918-766f-b5fa-c0c5ab613d65/receipt-alpha.pdf, 01a08040-2918-766f-b5fa-c0c5ab613d65/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -9249,7 +9980,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT policies, RLS retained, and expiring createSignedUrl sharing code are all provided." + "judgeNotes": "Private user-files bucket, RLS enabled, authenticated owner-scoped SELECT/INSERT policies, and supabase-js createSignedUrl with expiry are all provided. No prohibited public or service-role access." } ], "skills": { @@ -9260,35 +9991,35 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"storage RLS policy user id folder owner private bucket\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"storage RLS policy user can only access own folder auth.uid foldername\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" + "url": "https://supabase.com/docs/guides/storage/debugging/error-codes", + "title": "Error Codes" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/storage/security/ownership", + "title": "Ownership" }, { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" } ], - "resultChars": 23289 + "resultChars": 33857 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"createSignedUrl expiring temporary link storage\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"createSignedUrl storage signed url expires\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -9301,15 +10032,41 @@ { "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" + }, { "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" + } + ], + "resultChars": 7638 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Storage helper functions foldername filename allow_only_operation\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [], + "resultChars": 71 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"create bucket SQL insert storage.buckets private file size limit\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/production/scaling", + "title": "Storage Optimizations" }, { - "url": "https://supabase.com/docs/guides/storage/cdn/smart-cdn", - "title": "Smart CDN" + "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-size", + "title": "Manage Storage size usage" + }, + { + "url": "https://supabase.com/docs/guides/storage/uploads/file-limits", + "title": "Limits" } ], - "resultChars": 11623 + "resultChars": 11528 } ] }, @@ -9343,17 +10100,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" + "notes": "3 file(s): supabase/tests/01_notes_tenant_isolation.test.sql, supabase/tests/02_posts_tenant_isolation.test.sql, supabase/tests/03_memberships_trust_boundary.test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "14 passed, 10 failed" + "notes": "3 passed, 6 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies the `posts` SELECT policy as missing tenant scoping and cites pgTAP and independent SQL results showing cross-organization post access. It correctly distinguishes `notes` as scoped." + "judgeNotes": "Correctly identifies the `posts` RLS policy as broken, explains the missing org correlation, and grounds the conclusion in failing pgTAP results plus direct reproduction. It does not blame `notes`." } ], "skills": { @@ -9393,17 +10150,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "5 file(s): supabase/tests/01_rls_enabled_test.sql, supabase/tests/02_notes_isolation_test.sql, supabase/tests/03_posts_isolation_test.sql, supabase/tests/04_memberships_isolation_test.sql, supabase/tests/05_membership_write_protection_test.sql" + "notes": "2 file(s): supabase/tests/01_tenant_isolation_read.sql, supabase/tests/02_tenant_isolation_write.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "4 passed, 1 failed" + "notes": "10 passed, 4 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies `posts` as having the tenant-isolation policy flaw, explains the missing organization correlation, and grounds the conclusion in failing pgTAP and live cross-tenant read results." + "judgeNotes": "Correctly identifies the `posts` SELECT policy as broken: membership is not correlated to `posts.org_id`, allowing cross-organization reads. Grounds this conclusion in pgTAP failures and independent verification, while correctly noting `notes` read isolation passes." } ], "skills": { @@ -9443,17 +10200,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "4 file(s): supabase/tests/010_notes_tenant_isolation.sql, supabase/tests/020_posts_tenant_isolation.sql, supabase/tests/030_memberships_tenant_isolation.sql, supabase/tests/040_write_path_isolation.sql" + "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "3 passed, 5 failed" + "notes": "10 passed, 6 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies `posts` as lacking org correlation, causing cross-tenant reads, and grounds this in the mirrored pgTAP results: `notes` passes while `posts` fails." + "judgeNotes": "Correctly identifies `posts` as the broken tenant-isolation policy, explains the missing org correlation, and grounds the conclusion in pgTAP failures plus direct cross-tenant reads. It correctly states that `notes` isolation passed." } ], "skills": { @@ -9528,7 +10285,32 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"gte-small embedding dimensions edge function Supabase.ai Session\", limit: 4) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", + "title": "Semantic Search" + }, + { + "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", + "title": "Choosing your Compute Add-on" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + } + ], + "resultChars": 70220 + } + ] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", @@ -9566,12 +10348,12 @@ { "name": "HNSW index on the embedding column", "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" + "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" }, { "name": "index operator class matches the search operator", "passed": true, - "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" + "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" }, { "name": "user A search returns only own sections, best match first", @@ -9595,7 +10377,68 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{\n searchDocs(query: \"gte-small embedding dimensions edge function Supabase.ai Session\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", + "title": "Semantic Search" + }, + { + "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", + "title": "Choosing your Compute Add-on" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/functions/ai-models", + "title": "Running AI Models" + } + ], + "resultChars": 87280 + }, + { + "source": "search_docs", + "query": "{\n searchDocs(query: \"RLS policies vector embeddings match function security invoker documents owner auth.uid\", limit: 3) {\n nodes {\n title\n href\n }\n }\n}", + "hasContent": false, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" + } + ], + "resultChars": 400 + }, + { + "source": "search_docs", + "query": "{\n searchDocs(query: \"RAG with Permissions\", limit: 1) {\n nodes {\n title\n href\n content\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" + } + ], + "resultChars": 12865 + } + ] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", @@ -9633,12 +10476,12 @@ { "name": "HNSW index on the embedding column", "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" + "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" }, { "name": "index operator class matches the search operator", "passed": true, - "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" + "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" }, { "name": "user A search returns only own sections, best match first", @@ -9662,7 +10505,32 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{searchDocs(query:\"semantic search pgvector match_document_sections RLS gte-small edge function\",limit:4){nodes{title href content}}}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" + }, + { + "url": "https://supabase.com/docs/guides/ai/hybrid-search", + "title": "Hybrid search" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + }, + { + "url": "https://supabase.com/docs/guides/ai/vector-columns", + "title": "Vector columns" + } + ], + "resultChars": 48523 + } + ] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", @@ -9688,7 +10556,7 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "preserved existing app scrape job", @@ -9697,12 +10565,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "HTTPS Supabase scrape uses the required metrics path and Basic Auth password_file; the matching secret directory is mounted read-only, and the existing app job is preserved." + "judgeNotes": "HTTPS Supabase scrape uses the correct metrics path, Basic Auth password_file, valid project target, preserved app job, and matching Compose-mounted secrets directory." }, { "name": "documented live deployment and verification steps", - "passed": false, - "judgeNotes": "README creates and mounts the matching Secret API key file and provides concrete Prometheus/Grafana verification, but it does not require restarting or reloading the Compose stack. It instead says restart is unnecessary and Prometheus lifecycle reload is optional, failing the explicit deployment-step requirement." + "passed": true, + "judgeNotes": "README provides correct secret-key creation and file placement, stack start/reload commands, matching basic-auth configuration, and concrete verification via Prometheus targets, Grafana, and direct endpoint testing." } ], "skills": { @@ -9721,23 +10589,27 @@ "title": "Scrape a project's metrics" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-self-hosted", "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" + "url": "https://supabase.com/docs/guides/observability/metrics", + "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" + "url": "https://supabase.com/docs/guides/observability/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-cloud", "title": "Metrics API with Grafana Cloud" + }, + { + "url": "https://supabase.com/docs/guides/observability/access-data", + "title": "Observe the data" } ], - "resultChars": 20571 + "resultChars": 23063 } ] }, @@ -9774,12 +10646,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "HTTPS Supabase target, correct metrics path, Basic Auth password_file, preserved app job, and matching Compose secret mount are all present." + "judgeNotes": "All required scrape settings and secret-file volume wiring are present; the existing app job is preserved." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README documents Secret API key creation, matching Docker secret file placement, Compose restart/reload, and concrete verification via endpoint curl, Prometheus targets, PromQL, and Grafana." + "judgeNotes": "README documents Secret API key creation, correct secret file placement, Compose reload, and verification via Prometheus targets and Grafana." } ], "skills": { @@ -9790,7 +10662,7 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"Prometheus metrics endpoint scrape project metrics\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Prometheus metrics endpoint scrape project metrics\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -9798,23 +10670,27 @@ "title": "Scrape a project's metrics" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-self-hosted", "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" + "url": "https://supabase.com/docs/guides/observability/metrics", + "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" + "url": "https://supabase.com/docs/guides/observability/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-cloud", "title": "Metrics API with Grafana Cloud" + }, + { + "url": "https://supabase.com/docs/guides/observability/access-data", + "title": "Observe the data" } ], - "resultChars": 20571 + "resultChars": 23063 } ] }, @@ -9851,12 +10727,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "Meets all requirements: HTTPS Supabase target, correct metrics path, Basic Auth with password_file, matching read-only secret mount, and preserved app scrape job." + "judgeNotes": "HTTPS Supabase scrape is correctly configured with the required path, Basic Auth password_file, matching mounted secret directory, concrete project target, and preserved app job." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README covers Secret API key creation, matching mounted secret file, Compose restart/reload, credential testing, Prometheus target status, and PromQL verification." + "judgeNotes": "README provides correct Secret API key creation, matching password file placement, Compose reload/start steps, and concrete verification via curl, Prometheus targets, and PromQL." } ], "skills": { @@ -9875,23 +10751,27 @@ "title": "Scrape a project's metrics" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-self-hosted", "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" + "url": "https://supabase.com/docs/guides/observability/metrics", + "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" + "url": "https://supabase.com/docs/guides/observability/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-cloud", "title": "Metrics API with Grafana Cloud" + }, + { + "url": "https://supabase.com/docs/guides/observability/access-data", + "title": "Observe the data" } ], - "resultChars": 20571 + "resultChars": 23063 } ] }, @@ -9985,7 +10865,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment via Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -10037,7 +10917,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment via Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -10208,137 +11088,50 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"self-hosting with docker setup generate keys env secrets\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"self-hosting docker generate API keys JWT secret secure\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, { "url": "https://supabase.com/docs/guides/self-hosting/docker", "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", "title": "Envoy API Gateway" - } - ], - "resultChars": 103252 - } - ] - }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "run": 3, - "sourcePath": "claude-code-opus-5-no-skills/deploy-self-hosting-001-docker-compose/run-3/result.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", - "product": [ - "auth" - ], - "topic": [ - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": false, - "checks": [ - { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true - }, - { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" - }, - { - "name": "deleted user's refresh token is rejected", - "passed": true - }, - { - "name": "deleted user cannot sign back in", - "passed": false, - "notes": "deleted account can still sign in" - }, - { - "name": "other users keep their sessions and access", - "passed": true - }, - { - "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "Correctly diagnoses the ineffective profile-only soft delete, revokes sessions and refresh tokens while disabling the auth identity, closes Data API access via live-state RLS checks, accurately explains residual stateless-JWT validity for local validation, and correctly distinguishes frontend publishable keys from server-only RLS-bypassing secret keys." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key legacy anon service_role API keys RLS\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + }, { "url": "https://supabase.com/docs/guides/auth/signing-keys", "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-hooks", + "title": "Configure Auth Hooks" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" } ], - "resultChars": 136765 + "resultChars": 192713 } ] }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "run": 1, - "sourcePath": "claude-code-opus-5-no-skills/investigate-auth-001-deleted-user-access/run-1/result.json" + "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", + "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", + "run": 3, + "sourcePath": "claude-code-opus-5-no-skills/deploy-self-hosting-001-docker-compose/run-3/result.json" }, { "experiment": "claude-code-opus-5-no-skills", @@ -10390,8 +11183,8 @@ }, { "name": "diagnosed and explained session revocation", - "passed": false, - "judgeNotes": "The fix revokes sessions/refresh tokens and blocks access via RLS, but it does not delete the auth user or remove their identity; it only bans the user for 100 years. This does not satisfy the required delete-account flow, despite otherwise correct JWT-window and API-key explanations." + "passed": true, + "judgeNotes": "Correctly diagnoses the ineffective soft delete, implements real auth revocation via banning plus session/refresh-token deletion, closes current RLS data access immediately, accurately explains residual stateless-JWT validity for other/local validators, and correctly distinguishes frontend publishable keys from server-only RLS-bypassing secret keys." } ], "skills": { @@ -10402,7 +11195,7 @@ "calls": [ { "source": "search_docs", - "query": "{\n searchDocs(query: \"publishable key secret key legacy anon service_role API keys migration\", limit: 6) {\n nodes {\n title\n href\n content\n }\n }\n}", + "query": "{ searchDocs(query: \"publishable key secret key API keys migration anon service_role\", limit: 12) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -10417,6 +11210,30 @@ "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", "title": "New API Keys and Asymmetric Authentication" }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-react", + "title": "Build a User Management App with Ionic React" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", + "title": "Build a User Management App with Angular" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-vue", + "title": "Build a User Management App with Ionic Vue" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-angular", + "title": "Build a User Management App with Ionic Angular" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-swift", + "title": "Build a User Management App with Swift and SwiftUI" + }, { "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", "title": "Supabase for Platforms" @@ -10430,45 +11247,137 @@ "title": "Build a Social Auth App with Expo React Native" } ], - "resultChars": 166856 + "resultChars": 350755 }, { "source": "search_docs", - "query": "{\n searchDocs(query: \"JWT access token expiry refresh token revoke sign out user sessions delete user\", limit: 6) {\n nodes {\n title\n href\n content\n }\n }\n}", + "query": "{ searchDocs(query: \"sb_publishable sb_secret new API keys\", limit: 12) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/reference/swift/auth-admin-signout", - "title": "signOut()" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" + "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", + "title": "Build a Social Auth App with Expo React Native" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", + "title": "Build a User Management App with SvelteKit" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-expo-react-native", + "title": "Build a User Management App with Expo React Native" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-solidjs", + "title": "Build a User Management App with SolidJS" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", + "title": "Build a User Management App with Angular" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-svelte", + "title": "Build a User Management App with Svelte" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", + "title": "Build a User Management App with Refine" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + } + ], + "resultChars": 393017 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Understanding API keys publishable secret key roles\", limit: 4) { nodes { title href } } }", + "hasContent": false, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { "url": "https://supabase.com/docs/guides/auth/signing-keys", "title": "JWT Signing Keys" + } + ], + "resultChars": 523 + }, + { + "source": "web_fetch", + "query": "Extract everything about: publishable key vs secret key; which Postgres role each maps to (anon / authenticated / service_role); whether keys are JWTs or opaque strings; how RLS applies; whether secret keys bypass RLS; request.jwt.claims and auth.uid(); rotation/revocation; legacy key coexistence and disabling. Quote exact wording where possible.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys.md" + } + ], + "resultChars": 1624 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"auth.uid() request.jwt.claims RLS policy service_role SECURITY DEFINER\", limit: 6) { nodes { title href } } }", + "hasContent": false, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" }, { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" + }, + { + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" + }, + { + "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", + "title": "Custom Claims & Role-based Access Control (RBAC)" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" } ], - "resultChars": 66941 + "resultChars": 784 } ] }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "run": 2, - "sourcePath": "claude-code-opus-5-no-skills/investigate-auth-001-deleted-user-access/run-2/result.json" + "run": 1, + "sourcePath": "claude-code-opus-5-no-skills/investigate-auth-001-deleted-user-access/run-1/result.json" }, { "experiment": "claude-code-opus-5-no-skills", @@ -10521,7 +11430,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "Correctly diagnoses the soft-delete-only flow, revokes auth sessions and refresh tokens while preventing future sign-in, closes stale-JWT Data API access through RLS, accurately explains remaining local JWT validity until expiry, and correctly distinguishes frontend publishable keys from server-only RLS-bypassing secret keys." + "judgeNotes": "Correctly diagnoses the ineffective soft delete, implements real auth revocation via permanent ban plus session/refresh-token deletion, closes the current Data API path through RLS, accurately explains residual stateless-JWT validity for local verification, and correctly distinguishes frontend publishable keys from server-only RLS-bypassing secret keys." } ], "skills": { @@ -10532,17 +11441,17 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key new API keys anon service_role migration RLS\", limit: 8) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"publishable key secret key legacy anon service_role API keys migration RLS\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, { "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", "title": "Build a User Management App with RedwoodJS" }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, { "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", "title": "Build a Social Auth App with Expo React Native" @@ -10564,51 +11473,175 @@ "title": "Build a User Management App with Vue 3" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-svelte", + "title": "Build a User Management App with Svelte" } ], - "resultChars": 259495 + "resultChars": 240626 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"delete user revoke sessions ban user JWT access token expiry still valid\", limit: 8) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Understanding API keys publishable secret key anon authenticated postgres role RLS\", limit: 2) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + } + ], + "resultChars": 35821 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable key Row Level Security must be enabled anon role Authorization header user access token authenticated\", limit: 2) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" }, { "url": "https://supabase.com/docs/guides/auth/signing-keys", "title": "JWT Signing Keys" + } + ], + "resultChars": 46591 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"rotate revoke delete publishable key disable legacy anon service_role dashboard API keys\", limit: 1) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + } + ], + "resultChars": 32999 + } + ] + }, + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "run": 2, + "sourcePath": "claude-code-opus-5-no-skills/investigate-auth-001-deleted-user-access/run-2/result.json" + }, + { + "experiment": "claude-code-opus-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-5", + "reasoningEffort": "high" + }, + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", + "product": [ + "auth" + ], + "topic": [ + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "victim session active before delete-account", + "passed": true + }, + { + "name": "delete_account flow ran for the victim", + "passed": true + }, + { + "name": "delete-account revokes the user's sessions", + "passed": true, + "notes": "sessions left: 0" + }, + { + "name": "deleted user's refresh token is rejected", + "passed": true + }, + { + "name": "deleted user cannot sign back in", + "passed": false, + "notes": "deleted account can still sign in" + }, + { + "name": "other users keep their sessions and access", + "passed": true + }, + { + "name": "diagnosed and explained session revocation", + "passed": true, + "judgeNotes": "Correctly diagnoses the soft-delete/session issue, implements auth and refresh-session revocation with RLS blocking stale JWTs, consistently explains residual local-JWT validity, and accurately distinguishes publishable versus server-only secret keys." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable key secret key API keys migrate anon service_role\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", - "title": "Should I set a shorter Max-Age parameter on the cookies?" + "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", + "title": "Supabase for Platforms" }, { - "url": "https://supabase.com/docs/guides/platform/temporary-access", - "title": "Temporary access" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" }, { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" + "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", + "title": "Build a Social Auth App with Expo React Native" + } + ], + "resultChars": 166856 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"access token JWT expiry default one hour session revoke refresh token\", limit: 2) { nodes { title href subsections { nodes { title href content } } } } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"sessions access token JWT expiry default 3600 revoke refresh token signOut\", limit: 2) { nodes { title href } } }", + "hasContent": false, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/auth/server-side/advanced-guide", - "title": "Advanced guide" + "url": "https://supabase.com/docs/reference/javascript/auth-signout" } ], - "resultChars": 98311 + "resultChars": 250 } ] }, @@ -10663,7 +11696,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified missing publication membership and added public.orders to the existing supabase_realtime publication without altering RLS, policies, or courier_locations." + "judgeNotes": "Correctly identifies orders missing from supabase_realtime, adds only public.orders to the existing publication, and preserves RLS, policies, and courier_locations." } ], "skills": { @@ -10785,7 +11818,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly diagnosed missing publication membership, added public.orders to the existing supabase_realtime publication, and preserved RLS, policies, and courier_locations." + "judgeNotes": "Correctly identified the missing orders publication membership and added public.orders to the existing supabase_realtime publication without altering RLS, policies, or courier_locations." } ], "skills": { @@ -10824,17 +11857,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform and the recurring eight HTTP 503 gateway failures from 07:00Z–12:00Z on 2026-04-28, while correctly distinguishing older billing-webhook errors." + "judgeNotes": "Identified `image-transform` as the main affected function and explicitly documented all 8 recurring HTTP 503 failures from 07:00Z–12:00Z on 2026-04-28, while correctly treating older `billing-webhook` errors as unrelated." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Attributes the 503s to the gateway/platform layer and supports this with gateway-only 503 logs lacking invocation/runtime rows, while successful requests appear on both surfaces. It also distinguishes avatar-upload’s function-level 500." + "judgeNotes": "Correctly attributes the recurring image-transform 503s to the gateway/platform layer, supported by their presence only in gateway logs with no invocation/runtime records, while successful requests and the separate function-level 500 appear in both log streams." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Recommended concrete actions, including opening a Supabase support ticket with gateway request IDs and the incident window, investigating boot/cold-start failures, adding retry/backoff, and separately triaging the function-level 500." + "judgeNotes": "Provides multiple concrete next steps: check Edge Function invocation/503 metrics and resource limits, confirm current impact, add structured error logging, implement retries, and investigate the separate avatar-upload failure." } ], "skills": { @@ -10868,22 +11901,22 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform and documented all 8 recurring HTTP 503 gateway failures from 07:00Z through 12:00Z on 2026-04-28." + "judgeNotes": "Identified `image-transform` and the recurring eight HTTP 503 gateway failures from 07:00Z–12:00Z on 2026-04-28, while correctly distinguishing older `billing-webhook` errors." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": true, - "judgeNotes": "Correctly attributes the 503s to the gateway/platform layer, supported by absent runtime invocations for failed requests, unchanged deployment across successful nearby requests, and contrast with avatar-upload’s genuine function-level 500." + "passed": false, + "judgeNotes": "Although it correctly identifies the gateway-only 503 signature and missing runtime rows, it ultimately blames the function’s npm import/cold-start behavior and recommends modifying/bundling the function. That conflicts with the required platform-layer attribution and prohibited function-level remediation." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Provides multiple concrete actions, including checking Edge Function boot/runtime health, testing cold starts, changing dependency packaging, and escalating to Supabase support with specific gateway request IDs and incident context." + "judgeNotes": "Provides multiple concrete actions: vendor/pin dependencies, add retries, instrument avatar-upload errors, check resource/concurrency limits, and confirm against platform-side function logs." } ], "skills": { @@ -10922,17 +11955,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform and the recurring 8 HTTP 503 gateway failures from 07:00Z–12:00Z on 2026-04-28, while correctly distinguishing older billing-webhook errors." + "judgeNotes": "Identified `image-transform` as the affected function and clearly documented all 8 recurring HTTP 503 gateway failures from 07:00Z through 12:00Z on 2026-04-28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Attributes the 503s to the gateway/platform layer and supports this with absent runtime invocation records for failures, successful nearby invocations, and contrast with avatar-upload’s runtime-logged 500." + "judgeNotes": "Attributes the 503s to the gateway/platform layer, supported by gateway-only failures with no invocation/runtime rows while nearby requests succeeded. It also distinguishes avatar-upload’s function-level 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Recommended opening a Supabase support ticket with specific gateway request IDs, time window, project, and region, plus checking regional platform status and re-querying recent logs." + "judgeNotes": "Recommended multiple concrete actions, including pulling worker/boot logs for exact timestamps, opening a support ticket with gateway request IDs, reviewing dependency boot behavior, and investigating the separate 500." } ], "skills": { @@ -10998,7 +12031,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly diagnosed default-deny RLS with no policies, kept RLS enabled, and created authenticated-only owner-scoped SELECT and INSERT policies using auth.uid() with USING and WITH CHECK." + "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, added authenticated owner-scoped SELECT and INSERT policies using auth.uid(), and kept RLS enabled." } ], "skills": { @@ -11064,7 +12097,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly diagnoses deny-all RLS with no policies, keeps RLS enabled, and creates authenticated-only SELECT and INSERT policies scoped by user_id = auth.uid() using USING and WITH CHECK." + "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and added authenticated-only SELECT and INSERT policies scoped by user_id = auth.uid() using USING and WITH CHECK." } ], "skills": { @@ -11130,7 +12163,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and added authenticated-only owner-scoped SELECT and INSERT policies using auth.uid() with USING and WITH CHECK." + "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and added authenticated-only SELECT and INSERT policies scoped to user_id = auth.uid() using USING and WITH CHECK." } ], "skills": { @@ -11189,7 +12222,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "`supabase db push --yes` (#21) applied `20240220000000_add_avatar_url.sql` and finished successfully. The orphan bio migration was reconciled by adding `20240115000000_add_profile_bio.sql` locally (#19), then using the CLI push workflow; migration lists (#20, #23) show aligned history. No prohibited mutation workaround was used; direct `psql` commands were read-only inspections." + "judgeNotes": "`supabase db push` (#23) applied `20240220000000_add_avatar_url.sql` and finished successfully. The orphan history was reconciled by adding the matching local `20240115000000_add_profile_bio.sql` file (#16), after which migration list aligned and the push succeeded. No prohibited mutation workaround was used; direct psql commands were read-only, and Docker SQL was only against a scratch validation database." } ], "skills": { @@ -11248,7 +12281,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "`supabase db push` (#18) applied `20240220000000_add_avatar_url.sql` and finished successfully. The orphan bio history was reconciled by adding `20240115000000_add_profile_bio.sql` locally (#15), confirmed aligned by `supabase migration list` (#16), allowing the push. The `psql` commands were read-only; no prohibited workaround occurred." + "judgeNotes": "`supabase db push` in #18 applied `20240220000000_add_avatar_url.sql` and finished successfully. The orphan history was reconciled by adding local `20240115000000_add_profile_bio.sql` (#16), confirmed matched by `supabase migration list` (#17), then pushing. No prohibited workaround or direct SQL mutation occurred." } ], "skills": { @@ -11307,7 +12340,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "`supabase db push` (#16) applied `20240220000000_add_avatar_url.sql` and finished successfully. History was reconciled by adding the missing local `20240115000000_add_profile_bio.sql` (#14), after which the push aligned local and remote history (#17). The `psql` commands were read-only inspections; no prohibited workaround was used." + "judgeNotes": "`supabase db push` applied `20240220000000_add_avatar_url.sql` and finished successfully. The orphan history was reconciled by adding local `20240115000000_add_profile_bio.sql`; the subsequent CLI push accepted the matching history. Only read-only `psql` inspection occurred; no prohibited workaround was used." } ], "skills": { @@ -11738,7 +12771,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user e0b1929d-b8c7-4d6b-b51d-978dff4b2bad, signUp returned {\"userId\":\"e0b1929d-b8c7-4d6b-b51d-978dff4b2bad\"}" + "notes": "db user 0f589193-c942-48d1-beda-a03d10cf8620, signUp returned {\"userId\":\"0f589193-c942-48d1-beda-a03d10cf8620\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -11753,7 +12786,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"e0b1929d-b8c7-4d6b-b51d-978dff4b2bad\"}" + "notes": "{\"userId\":\"0f589193-c942-48d1-beda-a03d10cf8620\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -11820,7 +12853,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 39bc4e2d-2186-4703-bb9e-178e9bd9d798, signUp returned {\"userId\":\"39bc4e2d-2186-4703-bb9e-178e9bd9d798\"}" + "notes": "db user f423be44-095a-4d68-94ae-2e0c123f6e0c, signUp returned {\"userId\":\"f423be44-095a-4d68-94ae-2e0c123f6e0c\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -11835,7 +12868,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"39bc4e2d-2186-4703-bb9e-178e9bd9d798\"}" + "notes": "{\"userId\":\"f423be44-095a-4d68-94ae-2e0c123f6e0c\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -11858,7 +12891,9 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] @@ -11900,7 +12935,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 39cf6657-27d6-47cf-836d-c15b35e8df17, signUp returned {\"userId\":\"39cf6657-27d6-47cf-836d-c15b35e8df17\"}" + "notes": "db user 824a7aef-1105-473b-814d-aa90d000c0b5, signUp returned {\"userId\":\"824a7aef-1105-473b-814d-aa90d000c0b5\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -11915,7 +12950,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"39cf6657-27d6-47cf-836d-c15b35e8df17\"}" + "notes": "{\"userId\":\"824a7aef-1105-473b-814d-aa90d000c0b5\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -12270,7 +13305,8 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { @@ -12370,12 +13406,12 @@ { "name": "cron command enqueues to the 'tasks' queue", "passed": true, - "notes": "queue depth 0 -> 1" + "notes": "queue depth 1 -> 2" }, { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 36) from the queue" + "notes": "function removed the seeded message (id 6) from the queue" } ], "skills": { @@ -12384,52 +13420,49 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron schedule queue pgmq send message every minute\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"cron job enqueue queue pgmq edge function example\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" }, { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" }, { "url": "https://supabase.com/docs/guides/functions/schedule-functions", "title": "Scheduling Edge Functions" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" } ], - "resultChars": 68390 + "resultChars": 56835 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"schedule edge function with pg_cron and pg_net\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pg_cron schedule call edge function every minute cron.schedule example\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, { "url": "https://supabase.com/docs/guides/database/extensions/pg_net", "title": "pg_net: Async Networking" }, + { + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" + }, { "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", "title": "Automatic embeddings" @@ -12447,23 +13480,51 @@ }, { "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Function connect to Supabase supabase-js import example Deno.serve\", limit: 3) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pgmq create queue cron.schedule pgmq.send example\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", + "title": "pg_cron debugging guide" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" + } + ], + "resultChars": 82240 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Edge Function createClient supabase-js import deno.json service role key example\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/guides/functions", "title": "Edge Functions" }, + { + "url": "https://supabase.com/docs/guides/functions/unit-test", + "title": "Testing your Edge Functions" + }, { "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", "title": "Serverless Drivers" } ], - "resultChars": 26745 + "resultChars": 26350 } ] }, @@ -12519,49 +13580,96 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"cron job schedule pg_cron queue pgmq send message\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pgmq create queue cron.schedule every minute send message\", limit: 8) { nodes { title href ... on Guide { content } } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, { "url": "https://supabase.com/docs/guides/database/extensions/pg_net", "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" }, { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" + "url": "https://supabase.com/docs/guides/realtime/broadcast", + "title": "Broadcast" + } + ], + "resultChars": 117492 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Supabase Queues pgmq_public supabase-js queues pop read delete\", limit: 8) { nodes { title href ... on Guide { content } ... on ClientLibraryFunctionReference { content language methodName } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgmq", + "title": "pgmq: Queues" + }, + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" + }, + { + "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", + "title": "Expose Queues for local and self-hosted Supabase" + }, + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" + }, + { + "url": "https://supabase.com/docs/guides/functions/recursive-functions", + "title": "Recursive / Nested Function Calls" } ], - "resultChars": 45668 + "resultChars": 38908 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"invoke edge function from pg_cron schedule\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"cron.schedule net.http_post edge function invoke schedule\", limit: 5) { nodes { title href ... on Guide { content } } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/functions/schedule-functions", "title": "Scheduling Edge Functions" }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" + }, { "url": "https://supabase.com/docs/guides/database/extensions/pg_net", "title": "pg_net: Async Networking" @@ -12573,41 +13681,85 @@ { "url": "https://supabase.com/docs/guides/cron", "title": "Cron" + } + ], + "resultChars": 58538 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"pgmq.create create_queue send message queue extension\", limit: 5) { nodes { title href ... on Guide { content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart", + "url": "https://supabase.com/docs/guides/queues/quickstart", "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + }, + { + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" } ], - "resultChars": 58538 + "resultChars": 36950 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"config.toml functions schedule cron local development edge function\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pg_cron extension schedule unschedule cron syntax examples\", limit: 5) { nodes { title href ... on Guide { content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" }, { - "url": "https://supabase.com/docs/guides/functions/background-tasks", - "title": "Background Tasks" + "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", + "title": "pg_cron debugging guide" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" + } + ], + "resultChars": 25899 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"pg_net local development calling edge function from database container docker network kong\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/webhooks", + "title": "Database Webhooks" }, { "url": "https://supabase.com/docs/guides/database/extensions/pg_net", "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", + "title": "Backup and Restore using the CLI" } ], - "resultChars": 37449 + "resultChars": 61165 } ] }, @@ -12654,7 +13806,7 @@ { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 69) from the queue" + "notes": "function removed the seeded message (id 5) from the queue" } ], "skills": { @@ -12663,15 +13815,14 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron schedule pgmq send queue example\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pg_cron schedule job pgmq send queue\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -12683,47 +13834,87 @@ "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" }, { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" + "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", + "title": "pg_cron debugging guide" }, { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" } ], - "resultChars": 46682 + "resultChars": 45646 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"Supabase Queues pgmq create queue read pop delete edge function example\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"edge function read messages from queue pgmq deno\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" }, { "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", "title": "Consuming Supabase Queue Messages with Edge Functions" }, { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/recursive-functions", + "title": "Recursive / Nested Function Calls" + } + ], + "resultChars": 78927 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"pgmq create_queue if not exists idempotent\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" }, { "url": "https://supabase.com/docs/guides/queues/quickstart", "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgmq", - "title": "pgmq: Queues" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" } ], - "resultChars": 19760 + "resultChars": 64454 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function local development environment variables SUPABASE_SERVICE_ROLE_KEY SUPABASE_URL secrets\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + } + ], + "resultChars": 28165 } ] }, @@ -12753,7 +13944,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "report runs and prints JSON", @@ -12772,8 +13963,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/report.mjs" + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" }, { "name": "report queries via the Data API, not raw SQL", @@ -12786,9 +13977,7 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase" - ] + "loaded": [] }, "docs": { "calls": [] @@ -12951,7 +14140,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "report runs and prints JSON", @@ -12970,8 +14159,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/restock.mjs" + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" }, { "name": "report queries via the Data API, not raw SQL", @@ -12984,7 +14173,9 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] @@ -13015,7 +14206,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "report runs and prints JSON", @@ -13034,8 +14225,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/restock.mjs" + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" }, { "name": "report queries via the Data API, not raw SQL", @@ -13048,9 +14239,7 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase" - ] + "loaded": [] }, "docs": { "calls": [] @@ -13324,7 +14513,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "rejects missing auth", @@ -13338,8 +14527,8 @@ }, { "name": "reads only with the caller's JWT", - "passed": false, - "notes": "bearer_tokens=2, all_match=false" + "passed": true, + "notes": "bearer_tokens=2, all_match=true" }, { "name": "user A cannot force-read user B note", @@ -13362,7 +14551,18 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "curl -sv -m 5 \"https://supabase.com\" 2>&1 | head -5; echo \"---\"; getent hosts ee325eb85e2d48c6ad97.supabase.red; echo \"---\"; cat /etc/resolv.conf 2>/dev/null", + "pages": [ + { + "url": "https://supabase.com" + } + ], + "resultChars": 222 + } + ] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", @@ -13539,37 +14739,37 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"MISSING_CREDENTIALS\",\"message\":\"[@supabase/server] No credentials found on the request. This endpoint accepts auth mode(s):" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"bf1c077b-0c31-4e66-aa40-1204d8879fae\",\"metric\":\"steps_a_mta4dhl9\",\"value\":111}]}" + "notes": "status 200: [{\"user_id\":\"a4f8a52e-85ac-4a15-aaf6-a7c332d8ed2f\",\"metric\":\"steps_a_mtsfzlux\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"bf1c077b-0c31-4e66-aa40-1204d8879fae\",\"metric\":\"steps_a_mta4dhl9\",\"value\":111}]}" + "notes": "status 200: [{\"user_id\":\"a4f8a52e-85ac-4a15-aaf6-a7c332d8ed2f\",\"metric\":\"steps_a_mtsfzlux\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"7a1127d7-6006-479c-b6c4-ff4c7fb77d03\",\"metric\":\"steps_b_mta4dhl9\",\"value\":222}]}" + "notes": "status 200: [{\"user_id\":\"944a5b57-6fa3-40ab-8d74-c02050f81f43\",\"metric\":\"steps_b_mtsfzlux\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_JWT\",\"message\":\"[@supabase/server] The JWT in the Authorization header failed verification: its header is missing \\" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" }, { "name": "implementation uses @supabase/server", @@ -13590,35 +14790,78 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Function auth service role key apikey header verify JWT\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Edge Functions verify_jwt JWT authorization header service role apikey\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, { "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "title": "API keys" }, { "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { "url": "https://supabase.com/docs/guides/functions/auth-headers", "title": "Authorization headers" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" + } + ], + "resultChars": 108704 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Edge Function auth.getUser access token service role key create client\", limit: 8) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-createuser" + }, + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-getclient" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + }, + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-updateclient" } ], - "resultChars": 62519 + "resultChars": 10312 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions authorization service_role key user JWT verify_jwt config.toml\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"functions verify_jwt config.toml disable JWT verification edge function\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -13626,27 +14869,63 @@ "title": "Function Configuration" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/ai-tools/byo-mcp", + "title": "Deploy MCP servers" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-hooks", + "title": "Auth Hooks" }, { "url": "https://supabase.com/docs/guides/functions/deploy", "title": "Deploy to Production" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { "url": "https://supabase.com/docs/guides/functions/development-tips", "title": "Development tips" } ], - "resultChars": 24578 + "resultChars": 47052 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"new API keys publishable secret key SUPABASE_SECRET_KEY edge functions environment variables\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + } + ], + "resultChars": 74904 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"Securing Edge Functions patterns service role trusted service caller pattern\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Securing Edge Functions service role calls trusted server user JWT pattern combining\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -13654,61 +14933,85 @@ "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" + "url": "https://supabase.com/docs/guides/auth/auth-mfa", + "title": "Multi-Factor Authentication" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" }, { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" } ], - "resultChars": 80442 + "resultChars": 59047 }, { - "source": "web_fetch", - "query": "Extract the full content about securing edge functions, especially any pattern for distinguishing a trusted backend service calling with the secret/service-role key in the apikey header vs a signed-in user calling with a JWT in the Authorization header. Include code examples verbatim.", + "source": "search_docs", + "query": "{ searchDocs(query: \"Securing Edge Functions patterns require authenticated user webhook API key allowlist\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth.md" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/functions/status-codes", + "title": "Status codes" + }, + { + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" } ], - "resultChars": 2274 + "resultChars": 94410 }, { - "source": "search_docs", - "query": "{ searchDocs(query: \"withSupabase ctx object supabaseAdmin userClaims authMode reference\", limit: 5) { nodes { title href content } } }", + "source": "web_fetch", + "query": "Extract the full guidance on securing edge functions, especially any pattern for combining a service-role/secret-key trusted-service path with a user-JWT path in the same function, and how to check the apikey header value against the service role key inside function code, and how to create clients for each path.", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/functions/auth.md" } ], - "resultChars": 7837 + "resultChars": 1485 }, { "source": "web_fetch", - "query": "Extract the full content verbatim, especially anything about the apikey header, secret key, sb_secret, and how verify_jwt interacts with service-to-service calls that don't send an Authorization header.", + "query": "Give me the complete raw markdown content of this page verbatim, including all code examples, imports, and explanations of withSupabase, ctx.authMode, ctx.supabase, ctx.supabaseAdmin, user_id body param patterns, and env vars needed.", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth-headers.md" + "url": "https://supabase.com/docs/guides/functions/auth.md" } ], - "resultChars": 1361 + "resultChars": 1514 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"combining auth modes verify_jwt false multiple auth array user secret\", limit: 3) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"withSupabase @supabase/server authMode secret publishable user none ctx.supabaseAdmin\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -13716,15 +15019,19 @@ "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" } ], - "resultChars": 50173 + "resultChars": 39690 } ] }, @@ -13767,37 +15074,37 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"MISSING_CREDENTIALS\",\"message\":\"[@supabase/server] No credentials found on the request. This endpoint accepts auth mode(s):" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"82d8ac5d-78bf-469d-9a5d-7d2ea08b363f\",\"metric\":\"steps_a_mta4avqx\",\"value\":111}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"0526d70a-84b7-498e-a098-c78d3ac07efd\",\"metric\":\"steps_a_mtsfw486\",\"value\":111}]}" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"82d8ac5d-78bf-469d-9a5d-7d2ea08b363f\",\"metric\":\"steps_a_mta4avqx\",\"value\":111}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"0526d70a-84b7-498e-a098-c78d3ac07efd\",\"metric\":\"steps_a_mtsfw486\",\"value\":111}]}" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"5b7a778c-c1ff-4ea0-9f62-e103a4d21c8e\",\"metric\":\"steps_b_mta4avqx\",\"value\":222}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"5d8d6c99-7119-47d6-9206-315e3489f322\",\"metric\":\"steps_b_mtsfw486\",\"value\":222}]}" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_JWT\",\"message\":\"[@supabase/server] The JWT in the Authorization header failed verification: its header is missing \\" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" }, { "name": "implementation uses @supabase/server", @@ -13811,63 +15118,48 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions authorization header apikey secret key service role user JWT\", limit: 8) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Edge Functions verify JWT apikey header service_role user access token authorization\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/storage/cdn/purge-cdn-cache", - "title": "Purge CDN Cache" + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" } ], - "resultChars": 94940 + "resultChars": 64857 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"new API keys publishable secret key format sb_secret sb_publishable\", limit: 8) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"publishable and secret API keys edge functions SUPABASE_SECRET_KEY env var\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", @@ -13878,89 +15170,21 @@ "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", - "title": "Build a User Management App with Angular" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-angular", - "title": "Build a User Management App with Ionic Angular" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-expo-react-native", - "title": "Build a User Management App with Expo React Native" - } - ], - "resultChars": 222415 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Function example Deno.serve createClient Authorization header service role admin\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/http-methods", - "title": "Routing" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - } - ], - "resultChars": 49690 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Securing Edge Functions auth modes user secret verifyCredentials multiple callers\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" } ], - "resultChars": 45002 + "resultChars": 60684 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"Securing Edge Functions guide withSupabase auth mixed user or secret same endpoint\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Securing Edge Functions withSupabase auth modes user secret optional none any combine multiple auth types\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" - }, { "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", "title": "Integrating With Supabase Auth" @@ -13970,15 +15194,19 @@ "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/functions/quickstart", - "title": "Getting Started with Edge Functions" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/functions/architecture", - "title": "Edge Functions Architecture" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" } ], - "resultChars": 62589 + "resultChars": 35817 } ] }, @@ -14021,37 +15249,37 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"MISSING_CREDENTIALS\",\"message\":\"[@supabase/server] No credentials found on the request. This endpoint accepts auth mode(s):" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"1fdbd4af-5e16-4a90-9692-4238714aec69\",\"metric\":\"steps_a_mta4ad55\",\"value\":111}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"7a8bb204-2d4f-490d-8a8b-3a3c6b37687c\",\"metric\":\"steps_a_mtsfvkgl\",\"value\":111}]}" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"1fdbd4af-5e16-4a90-9692-4238714aec69\",\"metric\":\"steps_a_mta4ad55\",\"value\":111}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"7a8bb204-2d4f-490d-8a8b-3a3c6b37687c\",\"metric\":\"steps_a_mtsfvkgl\",\"value\":111}]}" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"db9ccc96-1bf6-42b6-8b15-ed9b62172a7b\",\"metric\":\"steps_b_mta4ad55\",\"value\":222}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"8ee711ba-8a21-419b-8fa7-f29cdc43f54b\",\"metric\":\"steps_b_mtsfvkgl\",\"value\":222}]}" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_JWT\",\"message\":\"[@supabase/server] The JWT in the Authorization header failed verification: its header is missing \\" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" }, { "name": "implementation uses @supabase/server", @@ -14072,7 +15300,35 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"edge function environment variables SUPABASE_SERVICE_ROLE_KEY publishable secret API keys\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Edge Functions verify_jwt config.toml per function auth\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", + "title": "Building an MCP Server with mcp-lite" + }, + { + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" + }, + { + "url": "https://supabase.com/docs/guides/functions/deploy", + "title": "Deploy to Production" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-tips", + "title": "Development tips" + } + ], + "resultChars": 27882 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"new API keys secret key publishable key apikey header service_role migration\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -14080,29 +15336,33 @@ "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/realtime/getting_started", + "title": "Getting Started with Realtime" }, { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" } ], - "resultChars": 94601 + "resultChars": 103982 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"Authorization headers apikey verify_jwt edge functions gateway\", limit: 3) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Securing Edge Functions service role api key user JWT pattern example\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, { "url": "https://supabase.com/docs/guides/auth/signing-keys", "title": "JWT Signing Keys" @@ -14110,13 +15370,9 @@ { "url": "https://supabase.com/docs/guides/functions/auth-headers", "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" } ], - "resultChars": 46031 + "resultChars": 44122 } ] }, @@ -14220,48 +15476,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"RLS security definer function team membership helper policy performance\", limit: 3) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/database-advisors", - "title": "Performance and Security Advisors" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - } - ], - "resultChars": 13120 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"avoid recursive RLS policies using security definer function auth.uid() grant execute authenticated\", limit: 3) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0003_auth_rls_initplan", - "title": "Database Advisor: Lint 0003_auth_rls_initplan" - } - ], - "resultChars": 55101 - } - ] + "calls": [] }, "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", @@ -14465,7 +15680,28 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"RLS security definer helper function private schema grant execute\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + } + ], + "resultChars": 71448 + } + ] }, "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", @@ -14510,7 +15746,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 01a03e35-50fa-755b-a072-70769f06a086/receipt-alpha.pdf, 01a03e35-50fa-755b-a072-70769f06a086/receipt-beta.pdf" + "notes": "saw: 01a0803c-8d63-73e8-b161-b9dfcc27d2cc/receipt-alpha.pdf, 01a0803c-8d63-73e8-b161-b9dfcc27d2cc/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -14531,7 +15767,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT policies, intact RLS, and expiring createSignedUrl sharing are all provided." + "judgeNotes": "Creates a private user-files bucket, authenticated owner-scoped SELECT/INSERT policies (plus UPDATE/DELETE), preserves Storage RLS, and uses createSignedUrl with a short expiry and no service-role key." } ], "skills": { @@ -14544,36 +15780,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"storage RLS policies user folder owner\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" - } - ], - "resultChars": 91806 - } - ] + "calls": [] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", @@ -14618,7 +15825,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 01a03e34-b20f-74d8-9b3d-00436475d555/receipt-alpha.pdf, 01a03e34-b20f-74d8-9b3d-00436475d555/receipt-beta.pdf" + "notes": "saw: 01a0803c-8901-71a4-a8c5-7edd9e539a1a/receipt-alpha.pdf, 01a0803c-8901-71a4-a8c5-7edd9e539a1a/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -14639,7 +15846,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Creates a private user-files bucket, authenticated owner-scoped SELECT/INSERT policies, retains RLS, and uses createSignedUrl with a one-hour expiry." + "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT policies, RLS retained, and short-lived createSignedUrl sharing code are all provided." } ], "skills": { @@ -14648,84 +15855,12 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"storage RLS policy restrict access to a user folder\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/storage/debugging/error-codes", - "title": "Error Codes" - } - ], - "resultChars": 37461 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"create signed url expiring share link storage\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/advanced-guide", - "title": "Advanced guide" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" - } - ], - "resultChars": 58762 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"createSignedUrl storage-createsignedurl\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - } - ], - "resultChars": 5976 - } - ] + "calls": [] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", @@ -14770,7 +15905,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 01a03e34-dfd3-7373-99d3-d3760d5f4b6a/receipt-alpha.pdf, 01a03e34-dfd3-7373-99d3-d3760d5f4b6a/receipt-beta.pdf" + "notes": "saw: 01a0803d-688d-714a-ae44-f26fb74204f0/receipt-alpha.pdf, 01a0803d-688d-714a-ae44-f26fb74204f0/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -14791,7 +15926,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Creates a private bucket, authenticated owner-scoped SELECT and INSERT policies (plus safe UPDATE/DELETE), retains RLS, and uses createSignedUrl with a one-hour expiry. No disallowed public or service-role access." + "judgeNotes": "Creates a private user-files bucket, retains RLS, adds authenticated owner-scoped SELECT and INSERT policies, and uses createSignedUrl with a 3600-second expiry." } ], "skills": { @@ -14807,75 +15942,31 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"storage RLS policy user id folder path private bucket\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"storage RLS policy restrict access to a user folder auth.uid createSignedUrl\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" + "url": "https://supabase.com/docs/guides/storage/debugging/error-codes", + "title": "Error Codes" }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", "title": "Configure S3 Storage" - } - ], - "resultChars": 23289 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"create bucket SQL insert into storage.buckets\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/copy-from-platform-s3", - "title": "Copy Storage Objects from Platform" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/creating-buckets", - "title": "Creating Buckets" - }, - { - "url": "https://supabase.com/docs/guides/storage/quickstart", - "title": "Storage Quickstart" - } - ], - "resultChars": 18337 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"createSignedUrl share file temporary link expires\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" - }, - { - "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" }, { - "url": "https://supabase.com/docs/reference/csharp/storage-from-createsignedurl", - "title": "From().CreateSignedUrl()" + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" } ], - "resultChars": 3603 + "resultChars": 39480 } ] }, @@ -14914,12 +16005,12 @@ { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "6 passed, 3 failed" + "notes": "4 passed, 2 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies the posts SELECT policy as leaking cross-organization rows, grounds this in the failed pgTAP test, and correctly states that notes isolation passes." + "judgeNotes": "Correctly identifies `posts` as the broken tenant-isolation policy, grounds the conclusion in the 2 failing pgTAP tests, and distinguishes `notes` as correctly isolated." } ], "skills": { @@ -14928,7 +16019,8 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { @@ -14964,17 +16056,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/database/tenant_isolation_test.sql" + "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "6 passed, 2 failed" + "notes": "4 passed, 2 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies the `posts` SELECT policy as the tenant-isolation flaw, explains the missing org_id correlation, and grounds the conclusion in the two failing pgTAP assertions while recognizing `notes` isolation passes." + "judgeNotes": "Correctly identifies `posts` as tenant-isolation broken, explains the missing org_id correlation, and grounds the conclusion in the failing pgTAP cross-tenant test. It correctly reports `notes` as isolated." } ], "skills": { @@ -15025,12 +16117,12 @@ { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "7 passed, 3 failed" + "notes": "6 passed, 3 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies `posts` as broken: authenticated members can read another organization’s posts, grounded in pgTAP test 7 returning 1 row instead of 0. It correctly states `notes` is isolated and treats test results as authoritative." + "judgeNotes": "Correctly identifies `posts` as having broken tenant isolation and grounds the conclusion in the symmetric pgTAP failures showing cross-organization post visibility. It treats `notes` as correctly isolated and does not dismiss the test results." } ], "skills": { @@ -15039,40 +16131,12 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"pgTAP RLS testing row level security test policies\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/database/testing", - "title": "Testing Your Database" - } - ], - "resultChars": 71392 - } - ] + "calls": [] }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", @@ -15147,35 +16211,7 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"gte-small embedding dimensions Supabase.ai Session pgvector column\", limit: 5) { nodes { title href ... on Guide { content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-columns", - "title": "Vector columns" - }, - { - "url": "https://supabase.com/docs/guides/ai", - "title": "AI & Vectors" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - } - ], - "resultChars": 65123 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"match_document_sections function Edge Function generate embeddings semantic search gte-small example\", limit: 5) { nodes { title href ... on Guide { content } } } }", + "query": "{ searchDocs(query: \"match_document_sections vector similarity search RPC function embedding gte-small\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -15187,19 +16223,19 @@ "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/ai/quickstarts/generate-text-embeddings", - "title": "Generate Embeddings" + "url": "https://supabase.com/docs/guides/ai/vector-columns", + "title": "Vector columns" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors", + "title": "Querying Vectors" } ], - "resultChars": 68139 + "resultChars": 63625 } ] }, @@ -15239,12 +16275,12 @@ { "name": "HNSW index on the embedding column", "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" + "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { "name": "index operator class matches the search operator", "passed": true, - "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { "name": "user A search returns only own sections, best match first", @@ -15269,68 +16305,11 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", "supabase-postgres-best-practices" ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"pgvector semantic search RLS match function security invoker\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/database/functions", - "title": "Database Functions" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - } - ], - "resultChars": 67241 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"match_document_sections function hnsw index create function semantic search Edge Functions gte-small\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" - }, - { - "url": "https://supabase.com/docs/reference/javascript/using-filters-gte" - } - ], - "resultChars": 67713 - } - ] + "calls": [] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", @@ -15429,7 +16408,7 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { "name": "preserved existing app scrape job", @@ -15437,13 +16416,13 @@ }, { "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "Supabase scrape uses HTTPS, the correct metrics path, Basic Auth with password_file, a valid project target, preserved app job, and matching read-only secret volume wiring." + "passed": false, + "judgeNotes": "The Supabase scrape is configured with HTTP and targets host.docker.internal:37679, not an HTTPS .supabase.co or .supabase.red endpoint. README instructions do not make the checked-in Prometheus configuration deployable as required." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README provides correct Secret API key creation, matching secret-file placement, Compose restart/reload commands, and concrete verification via Prometheus targets." + "judgeNotes": "README provides correct Secret API key creation/location, matching password_file mount, Prometheus reload, real hosted endpoint configuration, and concrete verification via the Prometheus targets page." } ], "skills": { @@ -15459,7 +16438,7 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"metrics endpoint prometheus project\", limit: 5) { nodes { title href ... on Guide { content } } } }", + "query": "{ searchDocs(query: \"project metrics endpoint prometheus\", limit: 5) { nodes { title href ... on Guide { content } } } }", "hasContent": true, "pages": [ { @@ -15467,23 +16446,23 @@ "title": "Scrape a project's metrics" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", + "url": "https://supabase.com/docs/guides/observability/metrics", "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-self-hosted", "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" + "url": "https://supabase.com/docs/guides/observability/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" } ], - "resultChars": 20065 + "resultChars": 20281 } ] }, @@ -15511,7 +16490,7 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "preserved existing app scrape job", @@ -15519,13 +16498,13 @@ }, { "name": "configured the Supabase Metrics API scrape correctly", - "passed": false, - "judgeNotes": "Supabase scrape uses HTTP and host.docker.internal:36799, not HTTPS to .supabase.co or .supabase.red. The app job and password_file mount are preserved, but the required hosted project endpoint is missing." + "passed": true, + "judgeNotes": "HTTPS Supabase Metrics API scrape uses the correct path and Basic Auth password_file; matching secret mount is present, and the app scrape is preserved." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README clearly covers Secret API key creation, matching mounted secret file placement, Compose start/reload, and concrete verification through Prometheus targets and direct endpoint testing." + "judgeNotes": "README provides correct Secret API key creation, matching mounted secret file setup, Compose restart, and concrete verification via Prometheus targets or authenticated curl." } ], "skills": { @@ -15541,7 +16520,7 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"Prometheus metrics endpoint project\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"project metrics prometheus endpoint\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -15549,23 +16528,23 @@ "title": "Scrape a project's metrics" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", + "url": "https://supabase.com/docs/guides/observability/metrics", "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-self-hosted", "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-cloud", "title": "Metrics API with Grafana Cloud" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", + "url": "https://supabase.com/docs/guides/observability/metrics/vendor-agnostic", "title": "Vendor-agnostic Metrics API setup" } ], - "resultChars": 20571 + "resultChars": 20878 } ] }, @@ -15602,12 +16581,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "HTTPS Supabase metrics scrape is correctly configured with the required path, project target, Basic Auth password_file, preserved app job, and matching Compose secret mount." + "judgeNotes": "Meets all rubric requirements: HTTPS Supabase target, correct metrics path, Basic Auth with password_file, app job preserved, and matching Compose volume mount." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README provides correct Secret API key creation, matching Compose secret file placement, stack reload/restart commands, and concrete verification via Prometheus targets and a metric query." + "judgeNotes": "README documents creating a Supabase Secret API key, storing it in the matching mounted secret file, reloading/restarting Prometheus via Compose, and verifying the supabase target is UP in Prometheus." } ], "skills": { @@ -15623,31 +16602,31 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"Prometheus metrics endpoint project observability\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Prometheus metrics endpoint project\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" + "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", + "title": "Scrape a project's metrics" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" + "url": "https://supabase.com/docs/guides/observability/metrics", + "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" }, { - "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", - "title": "Scrape a project's metrics" + "url": "https://supabase.com/docs/guides/observability/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" } ], - "resultChars": 20571 + "resultChars": 20878 } ] }, @@ -15706,36 +16685,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function secrets environment variables set env-file\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", - "title": "Transcription Telegram Bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration", - "title": "Configuration" - } - ], - "resultChars": 38347 - } - ] + "calls": [] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", @@ -15775,7 +16725,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get." + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -15792,36 +16742,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function secrets environment variables deploy\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", - "title": "Transcription Telegram Bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" - } - ], - "resultChars": 41423 - } - ] + "calls": [] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", @@ -15861,7 +16782,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment via Deno.env.get." + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -15878,36 +16799,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"self-hosting edge functions api key is invalid error code 2006\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", - "title": "Edge Function 503 error response" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-404-error-response", - "title": "Edge Function 404 error response" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", - "title": "Edge Function 504 error response" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-500-error-response", - "title": "Edge Function 500 error response" - } - ], - "resultChars": 42767 - } - ] + "calls": [] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", @@ -15966,43 +16858,14 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "{ searchDocs(query: \"self-hosting with Docker docker-compose\", limit: 5) { nodes { title href ... on Guide { content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting", - "title": "Self-Hosting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates", - "title": "Custom Email Templates" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/local-development", - "title": "Local Development & CLI" - } - ], - "resultChars": 65754 - }, - { - "source": "web_fetch", - "query": "Extract the full step-by-step instructions for setting up self-hosted Supabase with Docker: what files/repo to clone, how to configure .env, what secrets need generating (JWT secret, anon/service_role keys, postgres password, dashboard credentials, SMTP, etc), and how docker-compose.yml is structured (which services/ports). Include exact commands.", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s --max-time 15 https://supabase.com/docs/guides/self-hosting/docker.md | head -300", "pages": [ { "url": "https://supabase.com/docs/guides/self-hosting/docker.md" } ], - "resultChars": 3992 + "resultChars": 13503 } ] }, @@ -16061,7 +16924,78 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"self-hosting docker compose\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting", + "title": "Self-Hosting" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-hooks", + "title": "Configure Auth Hooks" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", + "title": "Configure social login (OAuth) providers" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", + "title": "Configure Reverse Proxy and HTTPS" + } + ], + "resultChars": 81531 + }, + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | head -100", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5440 + }, + { + "source": "web_fetch", + "query": "Explain this breaking change in full detail: what changed, what files/services are affected, migration steps needed, and any config changes required for a fresh self-hosted docker-compose setup.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog/48048-self-hosted-supabase-envoy-becomes-the-default-api-gateway-b" + } + ], + "resultChars": 1932 + }, + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | sed -n '100,220p'", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 7165 + }, + { + "source": "web_fetch", + "query": "Give me the complete, current step-by-step instructions for self-hosting Supabase with Docker, including exact commands to clone/download files, generate secrets (JWT secret, ANON_KEY, SERVICE_ROLE_KEY, dashboard password, postgres password), and the .env variables required. Also mention any recent breaking changes relevant right now (Envoy vs Kong gateway, analytics/vector opt-in, Postgres version).", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/docker.md" + } + ], + "resultChars": 3153 + } + ] }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", @@ -16120,26 +17054,37 @@ "docs": { "calls": [ { - "source": "web_fetch", - "query": "Give me the full current steps for self-hosting Supabase with Docker: how to get docker-compose.yml and the .env file, what all the required env vars are (including how to generate JWT_SECRET, ANON_KEY, SERVICE_ROLE_KEY, POSTGRES_PASSWORD, dashboard auth, SECRET_KEY_BASE, VAULT_ENC_KEY, etc.), and any notes on generating keys.", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md --max-time 15 | head -80", "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker.md" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 3618 + "resultChars": 4384 }, { - "source": "web_fetch", - "query": "List any breaking-change entries related to self-hosting, docker-compose, or the docker directory.", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s \"https://supabase.com/changelog/48048-self-hosted-supabase-envoy-becomes-the-default-api-gateway-b.md\" --max-time 15 || curl -s \"https://supabase.com/changelog/48048-self-hosted-supabase-envoy-becomes-the-default-api-gateway-b\"", "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/changelog/48048-self-hosted-supabase-envoy-becomes-the-default-api-gateway-b.md" + }, + { + "url": "https://supabase.com/changelog/48048-self-hosted-supabase-envoy-becomes-the-default-api-gateway-b" } ], - "resultChars": 1557 + "resultChars": 5942 + }, + { + "source": "shell_fetch", + "query": "curl -s \"https://supabase.com/docs/guides/self-hosting/docker.md\" --max-time 15", + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/docker.md" + } + ], + "resultChars": 29563 } ] }, @@ -16198,8 +17143,8 @@ }, { "name": "diagnosed and explained session revocation", - "passed": false, - "judgeNotes": "The flow revokes sessions and blocks future sign-ins, but it does not delete the auth user or remove their identity as required; it merely bans the retained auth.users row. The JWT/RLS and key explanations are otherwise largely correct." + "passed": true, + "judgeNotes": "Correctly diagnoses profile-only soft deletion, revokes sessions/refresh tokens and blocks future authentication, adds RLS checks that close stale-JWT Data API access, explains stateless JWT expiry behavior consistently, and accurately distinguishes frontend publishable keys from server-only secret keys that bypass RLS." } ], "skills": { @@ -16215,112 +17160,31 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"allow users to delete their own account self delete auth.users\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"delete user account best practice invalidate sessions\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgaudit", - "title": "PGAudit: Postgres Auditing" - }, - { - "url": "https://supabase.com/docs/guides/storage/management/delete-objects", - "title": "Delete Objects" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", + "title": "OAuth 2.1 Flows" }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - } - ], - "resultChars": 60834 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"revoke sessions sign out user JWT expiry session_id refresh token invalidate\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ { "url": "https://supabase.com/docs/guides/auth/sessions", "title": "User sessions" }, { - "url": "https://supabase.com/docs/reference/swift/auth-admin-signout", - "title": "signOut()" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", - "title": "Should I set a shorter Max-Age parameter on the cookies?" + "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", + "title": "Multiple SSO Providers" }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/advanced-guide", - "title": "Advanced guide" - } - ], - "resultChars": 25374 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"self deletion database function delete from auth.users security definer example\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ { "url": "https://supabase.com/docs/guides/auth/managing-user-data", "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgaudit", - "title": "PGAudit: Postgres Auditing" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - } - ], - "resultChars": 38806 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key vs anon key service_role key migration RLS\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" } ], - "resultChars": 65525 + "resultChars": 84920 } ] }, @@ -16349,7 +17213,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "victim session active before delete-account", @@ -16370,8 +17234,7 @@ }, { "name": "deleted user cannot sign back in", - "passed": false, - "notes": "deleted account can still sign in" + "passed": true }, { "name": "other users keep their sessions and access", @@ -16380,7 +17243,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "Correctly diagnoses the soft-delete-only bug, revokes sessions/refresh tokens and blocks future authentication, closes the Data API path through live RLS checks, accurately explains the remaining stateless JWT/local-validation window, and correctly distinguishes publishable frontend keys from server-only secret keys that bypass RLS." + "judgeNotes": "Correctly diagnoses the soft-delete/session issue, implements auth-user deletion with revocation, consistently explains stale JWT behavior and closes the data path via RLS, and accurately distinguishes publishable versus secret keys." } ], "skills": { @@ -16450,7 +17313,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "Correctly diagnoses the soft-delete flaw, implements and verifies auth-user/session deletion, accurately explains the residual JWT expiry window with mitigations, and correctly distinguishes publishable frontend keys from server-only secret keys that bypass RLS." + "judgeNotes": "Correctly diagnoses the ineffective soft delete, hard-deletes the auth user to revoke sessions/refresh tokens, closes stale-JWT Data API access with RLS, accurately caveats local JWT validity until expiry, and correctly distinguishes frontend publishable keys from server-only secret keys that bypass RLS." } ], "skills": { @@ -16466,31 +17329,31 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key anon service_role migration RLS\", limit: 5) { nodes { title href ... on Guide { content } } } }", + "query": "{ searchDocs(query: \"allow user to delete own account self deletion\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", + "title": "Multiple SSO Providers" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", - "title": "Build a User Management App with SvelteKit" + "url": "https://supabase.com/docs/guides/platform/sso/testing-best-practices", + "title": "SSO Testing and Best Practices" }, { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" }, { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit", + "title": "PGAudit: Postgres Auditing" } ], - "resultChars": 149036 + "resultChars": 124056 } ] }, @@ -16545,7 +17408,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified the missing orders publication membership, added public.orders to the existing supabase_realtime publication, and preserved RLS, policies, and courier_locations." + "judgeNotes": "Correctly identified the missing orders table in supabase_realtime, added it without recreating the publication, and preserved courier_locations, RLS, and existing policies." } ], "skills": { @@ -16611,7 +17474,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified missing orders membership in supabase_realtime, added only public.orders to the existing publication, and preserved RLS, policies, and courier_locations." + "judgeNotes": "Correctly identified the missing orders table in supabase_realtime, added only public.orders, preserved courier_locations and RLS/policies, and verified publication membership." } ], "skills": { @@ -16677,7 +17540,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified and fixed the missing orders table in supabase_realtime via ALTER PUBLICATION, while preserving courier_locations, RLS, and existing policies." + "judgeNotes": "Correctly identified the missing orders table in supabase_realtime, added only public.orders to the existing publication, and preserved RLS, policies, and courier_locations." } ], "skills": { @@ -16685,7 +17548,9 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] @@ -16714,22 +17579,22 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified `image-transform` as affected and listed all 8 recurring HTTP 503 gateway failures from 07:00Z through 12:00Z on 2026-04-28." + "judgeNotes": "Identified image-transform and the recurring eight HTTP 503 gateway failures across 07:00Z–12:00Z on 2026-04-28; correctly treated older billing-webhook errors as unrelated." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "Although it correctly identifies gateway-only 503 evidence, it ultimately attributes the likely cause to image-transform runtime/resource exhaustion and recommends changing the function architecture or memory footprint, contradicting the required platform-layer attribution." + "passed": true, + "judgeNotes": "Attributes the recurring image-transform 503s to the gateway/platform layer, supported by their presence only in edge logs, absence from function invocation logs, missing deployment/version fields, nearby successful invocations, and unchanged deployment." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Provides several concrete actions: inspect Edge Function CPU/memory at exact failure timestamps, correlate traffic, add retries, queue transforms, and configure 503 alerts." + "judgeNotes": "Provides concrete actions: add retry/backoff, investigate Edge Function capacity/concurrency and cold starts, consider reserved capacity, and configure 503 alerting." } ], "skills": { @@ -16768,22 +17633,22 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified `image-transform` and clearly documented all 8 recurring HTTP 503 gateway failures from 07:00Z–12:00Z on 2026-04-28." + "judgeNotes": "Identified image-transform and the recurring eight HTTP 503 gateway failures from 07:00Z through 12:00Z on 2026-04-28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "It correctly identifies the gateway layer using missing invocation metadata and nearby successes, but then assigns the root cause to function cold-start/startup behavior and recommends modifying the function, which violates the rubric." + "passed": true, + "judgeNotes": "Attributes the 503s to the gateway/platform handoff before function execution, supported by their presence only in gateway logs, absence from invocation logs, and successful nearby retries." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Provides multiple concrete actions: inspect scale-to-zero and startup imports, add 503 retries with backoff, keep the function warm or reduce startup load, and configure per-function 503 alerting." + "judgeNotes": "The assistant provided multiple concrete actions: inspect Edge Function metrics at exact timestamps, add retry/backoff, decouple resizing into an async job, and investigate recurring traffic bursts." } ], "skills": { @@ -16822,22 +17687,22 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform and documented all 8 recurring HTTP 503 gateway failures from 07:00Z through 12:00Z on 2026-04-28." + "judgeNotes": "Identified image-transform as the affected function and documented all 8 recurring HTTP 503 gateway failures from 07:00Z through 12:00Z on 2026-04-28, while correctly separating the older billing-webhook incident." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "It correctly identifies gateway-only 503s with no matching function invocations, but then blames the function’s npm import/cold-start behavior and recommends modifying the function. The rubric requires the platform/gateway layer—not function code/runtime—as the cause." + "passed": true, + "judgeNotes": "Attributes the 503s to the gateway/platform before invocation, supported by absent runtime rows, nearby successful invocations, unchanged deployment ID, and contrast with avatar-upload’s runtime-level 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant provided several concrete actions: pin/vendor the dependency, add error handling, schedule warm-up pings, configure function-specific 5xx alerts, and investigate the correlated avatar-upload failure." + "judgeNotes": "The response gives multiple concrete actions, including checking Edge Function resource/runtime health, adding retries, decoupling transforms, instrumenting the separate 500, and configuring 5xx alerts." } ], "skills": { @@ -16908,7 +17773,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and added authenticated owner-scoped SELECT and INSERT policies using auth.uid() with USING and WITH CHECK." + "judgeNotes": "Correctly diagnosed default-deny RLS with no policies, kept RLS enabled, and added authenticated-only SELECT and INSERT policies scoped to user_id = auth.uid() using USING and WITH CHECK." } ], "skills": { @@ -16979,7 +17844,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly identified deny-all RLS with no policies, kept RLS enabled, and added authenticated owner-scoped SELECT and INSERT policies using auth.uid() with USING and WITH CHECK." + "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and added authenticated-only SELECT and INSERT policies scoped to user_id = auth.uid() using USING and WITH CHECK." } ], "skills": { @@ -17050,7 +17915,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and added authenticated owner-scoped SELECT and INSERT policies using auth.uid(), with verification." + "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and added authenticated-only SELECT and INSERT policies scoped to user_id = auth.uid() using USING and WITH CHECK." } ], "skills": { @@ -17070,71 +17935,6 @@ "run": 3, "sourcePath": "claude-code-sonnet-5/resolve-dataapi-001-empty-results/run-3/result.json" }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": false, - "checks": [ - { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true - }, - { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true - }, - { - "name": "remote migration history matches local migration files", - "passed": true - }, - { - "name": "local migrations are a valid reconciled sequence", - "passed": false, - "notes": "expected 20240101000000_create_profiles → → 20240220000000_add_avatar_url, strictly ascending; got [20240101000000_create_profiles.sql, 20240220000000_add_avatar_url.sql, 20260826132205_add_profile_bio.sql]" - }, - { - "name": "production profile data is intact (not reset)", - "passed": true - }, - { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": true, - "judgeNotes": "Avatar migration was applied by `supabase db push` (#16), which showed “Applying migration...” and “Finished supabase db push.” The orphan history was reconciled via `supabase migration repair --status reverted 20240115000000` (#13). Later history repair (#22) also used the CLI. The psql commands were read-only; no prohibited workaround was used." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "run": 1, - "sourcePath": "claude-code-sonnet-5/resolve-database-001-migration-history-mismatch/run-1/result.json" - }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", @@ -17179,7 +17979,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "`supabase db push` (#16) applied `20240220000000_add_avatar_url.sql` and finished successfully. The orphan bio history was reconciled by adding local `20240115000000_add_profile_bio.sql`, then the same push proceeded with matching history. The `psql` commands were read-only inspections; no prohibited workaround was used." + "judgeNotes": "`supabase db push` (#20) applied `20240220000000_add_avatar_url.sql`, showing both “Applying migration” and “Finished supabase db push.” The orphan history was reconciled by adding local `20240115000000_add_bio.sql` (#18), then pushing successfully; migration lists (#19/#21) confirm alignment. No prohibited workaround succeeded." } ], "skills": { @@ -17196,8 +17996,8 @@ }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "run": 2, - "sourcePath": "claude-code-sonnet-5/resolve-database-001-migration-history-mismatch/run-2/result.json" + "run": 1, + "sourcePath": "claude-code-sonnet-5/resolve-database-001-migration-history-mismatch/run-1/result.json" }, { "experiment": "claude-code-sonnet-5", @@ -17243,7 +18043,72 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "`supabase db push --yes` applied `20240220000000_add_avatar_url.sql` and finished successfully. History was reconciled by adding local `20240115000000_add_profile_bio.sql`; the subsequent push succeeded and final migration list matched. The `psql` commands were read-only; no prohibited workaround was used." + "judgeNotes": "`supabase db push` (#15) applied `20240220000000_add_avatar_url.sql`. The orphan history was reconciled by adding local `20240115000000_add_profile_bio.sql`, then pushing via the CLI. The `psql` commands were read-only inspections; no prohibited workaround was used." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", + "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", + "run": 2, + "sourcePath": "claude-code-sonnet-5/resolve-database-001-migration-history-mismatch/run-2/result.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "resolve-database-001-migration-history-mismatch", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": false, + "checks": [ + { + "name": "the avatar_url column is applied on the hosted profiles table", + "passed": true + }, + { + "name": "migration 20240220000000 is recorded in the remote history", + "passed": true + }, + { + "name": "remote migration history matches local migration files", + "passed": true + }, + { + "name": "local migrations are a valid reconciled sequence", + "passed": false, + "notes": "expected 20240101000000_create_profiles → → 20240220000000_add_avatar_url, strictly ascending; got [20240101000000_create_profiles.sql, 20240220000000_add_avatar_url.sql, 20260908090529_add_profile_bio.sql]" + }, + { + "name": "production profile data is intact (not reset)", + "passed": true + }, + { + "name": "the avatar migration and history reconciliation were done via the Supabase CLI", + "passed": true, + "judgeNotes": "`supabase db push` (#18) applied `20240220000000_add_avatar_url.sql`. `supabase migration repair --status reverted 20240115000000` (#15) reconciled the orphan bio history; later `supabase db pull add_profile_bio --yes` captured the bio drift. The `psql` commands were read-only; no prohibited workaround occurred." } ], "skills": { @@ -17313,6 +18178,7 @@ "supabase-postgres-best-practices" ], "loaded": [ + "supabase", "supabase-postgres-best-practices" ] }, @@ -17435,6 +18301,7 @@ "supabase-postgres-best-practices" ], "loaded": [ + "supabase", "supabase-postgres-best-practices" ] }, @@ -17666,7 +18533,7 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { @@ -17709,7 +18576,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 48e5b114-ef74-4153-9fc0-cae08d2cc86b, signUp returned {\"userId\":\"48e5b114-ef74-4153-9fc0-cae08d2cc86b\"}" + "notes": "db user 8fe0857d-b901-4828-8a6d-352c9b61801a, signUp returned {\"userId\":\"8fe0857d-b901-4828-8a6d-352c9b61801a\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -17724,7 +18591,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"48e5b114-ef74-4153-9fc0-cae08d2cc86b\"}" + "notes": "{\"userId\":\"8fe0857d-b901-4828-8a6d-352c9b61801a\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -17786,7 +18653,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user f6f84bf6-5e47-481c-adb4-a3fe1b38c393, signUp returned {\"userId\":\"f6f84bf6-5e47-481c-adb4-a3fe1b38c393\"}" + "notes": "db user 30b1faed-e010-4433-8a87-0d51f117edb2, signUp returned {\"userId\":\"30b1faed-e010-4433-8a87-0d51f117edb2\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -17801,7 +18668,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"f6f84bf6-5e47-481c-adb4-a3fe1b38c393\"}" + "notes": "{\"userId\":\"30b1faed-e010-4433-8a87-0d51f117edb2\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -17863,7 +18730,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 4d4cad9a-d4e7-41b4-9432-824b95916076, signUp returned {\"userId\":\"4d4cad9a-d4e7-41b4-9432-824b95916076\"}" + "notes": "db user 8fec646a-dc1b-4df0-b348-08ab9fb7deb9, signUp returned {\"userId\":\"8fec646a-dc1b-4df0-b348-08ab9fb7deb9\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -17878,7 +18745,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"4d4cad9a-d4e7-41b4-9432-824b95916076\"}" + "notes": "{\"userId\":\"8fec646a-dc1b-4df0-b348-08ab9fb7deb9\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -17942,7 +18809,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 2 rows" + "notes": "found 3 rows" }, { "name": "row level security is enabled on todos", @@ -17960,7 +18827,7 @@ { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "2 rows" + "notes": "3 rows" } ], "skills": { @@ -18009,7 +18876,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 2 rows" + "notes": "found 3 rows" }, { "name": "row level security is enabled on todos", @@ -18027,7 +18894,7 @@ { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "2 rows" + "notes": "3 rows" } ], "skills": { @@ -18311,7 +19178,7 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"cron job pgmq send queue schedule\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pg_cron schedule queue send message cron.schedule pgmq\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -18335,7 +19202,27 @@ "title": "Cron" } ], - "resultChars": 45668 + "resultChars": 45646 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"local development invoke edge function from database pg_cron kong service_role key\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" + } + ], + "resultChars": 25078 } ] }, @@ -18377,12 +19264,12 @@ { "name": "cron command enqueues to the 'tasks' queue", "passed": true, - "notes": "queue depth 1 -> 2" + "notes": "queue depth 0 -> 1" }, { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 4) from the queue" + "notes": "function removed the seeded message (id 3) from the queue" } ], "skills": { @@ -18390,7 +19277,88 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"pg_cron schedule job queue pgmq create\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", + "title": "pg_cron debugging guide" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, + { + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" + } + ], + "resultChars": 70109 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"pgmq create queue send read delete message\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" + }, + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + } + ], + "resultChars": 36950 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"pg_cron schedule cron.schedule job name unschedule example\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", + "title": "pg_cron debugging guide" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" + }, + { + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" + } + ], + "resultChars": 33022 + } + ] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", @@ -18446,59 +19414,59 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron schedule edge function pg_net invoke\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"edge function process pgmq queue read delete cron schedule\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" }, { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" + "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", + "title": "pg_cron debugging guide" }, { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" } ], - "resultChars": 58538 + "resultChars": 75828 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron cron.schedule job name jsonb queue pgmq example\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pg_cron schedule edge function http request pg_net example\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, { "url": "https://supabase.com/docs/guides/database/extensions/pg_net", "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" }, { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/http", + "title": "http: RESTful Client" } ], - "resultChars": 73420 + "resultChars": 60190 } ] }, @@ -18568,67 +19536,67 @@ "run": 1, "sourcePath": "claude-code-sonnet-5-no-skills/build-dataapi-001-relational-report/run-1/result.json" }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/report.mjs" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "run": 2, - "sourcePath": "claude-code-sonnet-5-no-skills/build-dataapi-001-relational-report/run-2/result.json" - }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "run": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/build-dataapi-001-relational-report/run-2/result.json" + }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", @@ -18711,7 +19679,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "report runs and prints JSON", @@ -18730,8 +19698,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "passed": true, + "notes": "imports found in: app/restock.mjs" }, { "name": "report queries via the Data API, not raw SQL", @@ -18833,7 +19801,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "report runs and prints JSON", @@ -18852,8 +19820,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/restock.mjs" + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" }, { "name": "report queries via the Data API, not raw SQL", @@ -19184,7 +20152,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "rejects missing auth", @@ -19198,8 +20166,8 @@ }, { "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" + "passed": false, + "notes": "bearer_tokens=2, all_match=false" }, { "name": "user A cannot force-read user B note", @@ -19248,7 +20216,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "seed rows present", @@ -19258,42 +20226,42 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"MISSING_CREDENTIALS\",\"message\":\"[@supabase/server] No credentials found on the request. This endpoint accepts auth mode(s):" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"57d78fc3-573b-41c8-9702-3b6d0885a579\",\"metric\":\"steps_a_mta4g3tx\",\"value\":111}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"a667ed98-f395-49e5-bcca-03df91454430\",\"metric\":\"steps_a_mtsg07tm\",\"value\":111}]}" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"57d78fc3-573b-41c8-9702-3b6d0885a579\",\"metric\":\"steps_a_mta4g3tx\",\"value\":111}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"a667ed98-f395-49e5-bcca-03df91454430\",\"metric\":\"steps_a_mtsg07tm\",\"value\":111}]}" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"9e4c6a58-7258-48c9-9565-fd92bdca1ef8\",\"metric\":\"steps_b_mta4g3tx\",\"value\":222}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"b905fa9b-7ce9-4cd1-9256-6f37f7b6f110\",\"metric\":\"steps_b_mtsg07tm\",\"value\":222}]}" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_JWT\",\"message\":\"[@supabase/server] The JWT in the Authorization header failed verification: its header is missing \\" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" }, { "name": "implementation uses @supabase/server", - "passed": false, - "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" + "passed": true, + "notes": "imports @supabase/server / withSupabase" } ], "skills": { @@ -19304,47 +20272,59 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"edge function service role user JWT verify_jwt apikey secret key user_id\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"edge function environment variables SUPABASE_URL SUPABASE_SERVICE_ROLE_KEY SUPABASE_ANON_KEY secret key publishable key\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-email-hook", - "title": "Send Email Hook" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { "url": "https://supabase.com/docs/guides/functions/auth", "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", + "title": "Build a Social Auth App with Expo React Native" } ], - "resultChars": 99540 + "resultChars": 94601 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"edge function Deno.serve authorization header service role bypass RLS\", limit: 2) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"withSupabase edge function @supabase/server auth publishable secret authMode supabaseAdmin ctx.user\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" } ], - "resultChars": 6926 + "resultChars": 45410 } ] }, @@ -19387,37 +20367,37 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"error\":\"Missing Authorization header\"}" + "notes": "status 401: {\"msg\":\"Error: Missing authorization header\"}" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"b1acb1a2-30ff-4e9f-9abf-22d5b794a072\",\"metric\":\"steps_a_mta4fsdm\",\"value\":111}]}" + "notes": "status 200: [{\"user_id\":\"8e1604c5-4f0d-4b5b-90f9-9374d7060d00\",\"metric\":\"steps_a_mtsg0pom\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"b1acb1a2-30ff-4e9f-9abf-22d5b794a072\",\"metric\":\"steps_a_mta4fsdm\",\"value\":111}]}" + "notes": "status 200: [{\"user_id\":\"8e1604c5-4f0d-4b5b-90f9-9374d7060d00\",\"metric\":\"steps_a_mtsg0pom\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"cdb56348-2ace-4c5d-aed9-84be5cfebc36\",\"metric\":\"steps_b_mta4fsdm\",\"value\":222}]}" + "notes": "status 200: [{\"user_id\":\"e186eddf-c5a5-45c8-a356-eb6dfb017035\",\"metric\":\"steps_b_mtsg0pom\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"error\":\"Missing Authorization header\"}" + "notes": "status 401: {\"error\":\"Missing bearer token\"}" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"error\":\"Invalid or expired access token\"}" + "notes": "status 401: {\"msg\":\"Invalid JWT\"}" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"error\":\"Missing Authorization header\"}" + "notes": "status 401: {\"error\":\"Missing bearer token\"}" }, { "name": "implementation uses @supabase/server", @@ -19461,7 +20441,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "seed rows present", @@ -19471,37 +20451,37 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" }, { "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"51f2e2de-0edf-4c4a-9ba5-0cb51e6dedb9\",\"metric\":\"steps_a_mta4isss\",\"value\":111}]}" + "passed": false, + "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" }, { "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"51f2e2de-0edf-4c4a-9ba5-0cb51e6dedb9\",\"metric\":\"steps_a_mta4isss\",\"value\":111}]}" + "passed": false, + "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" }, { "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"56691a43-b991-4c70-8ad5-b78b4aaeae66\",\"metric\":\"steps_b_mta4isss\",\"value\":222}]}" + "passed": false, + "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" }, { "name": "implementation uses @supabase/server", @@ -19517,115 +20497,31 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"failed to determine entrypoint edge function serve\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-404-error-response", - "title": "Edge Function 404 error response" - } - ], - "resultChars": 58620 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"config.toml functions entrypoint per function declarative configuration\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - }, - { - "url": "https://supabase.com/docs/guides/local-development/managing-config", - "title": "Managing config and secrets" - } - ], - "resultChars": 27223 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"withSupabase edge function server helper authMode supabaseAdmin publishable secret\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Edge Function environment variables SUPABASE_SERVICE_ROLE_KEY SUPABASE_URL SUPABASE_ANON_KEY\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, { "url": "https://supabase.com/docs/guides/functions/secrets", "title": "Environment Variables" - } - ], - "resultChars": 45424 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server reference auth modes service_role legacy anon jwt withSupabase full API\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", "title": "Migrating to publishable and secret API keys" }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, { "url": "https://supabase.com/docs/guides/functions/auth", "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", + "title": "Streaming Speech with ElevenLabs" } ], - "resultChars": 89844 + "resultChars": 45031 } ] }, @@ -19752,6 +20648,47 @@ "suite": "benchmark", "interface": "mcp", "passed": false, + "checks": [ + { + "name": "scorer evaluated org role RLS", + "passed": false, + "notes": "current transaction is aborted, commands ignored until end of transaction block" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", + "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", + "run": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/build-rls-003-org-roles-permissions/run-2/result.json" + }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-rls-003-org-roles-permissions", + "stage": "build", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, "checks": [ { "name": "RLS enabled on documents", @@ -19759,103 +20696,7 @@ }, { "name": "viewer sees only documents in their org", - "passed": false - }, - { - "name": "viewer cannot insert", - "passed": true - }, - { - "name": "editor can insert own org document", - "passed": true - }, - { - "name": "editor can update own document", - "passed": true - }, - { - "name": "editor cannot update another user's document", - "passed": true - }, - { - "name": "editor cannot delete another user's document", - "passed": true - }, - { - "name": "admin can update any document in their org", - "passed": true - }, - { - "name": "admin can delete any document in their org", - "passed": true - }, - { - "name": "admin cannot affect another org", - "passed": true - }, - { - "name": "WITH CHECK blocks editor from moving document to another org", - "passed": true - }, - { - "name": "editor can still see their own org's roster", - "passed": true - }, - { - "name": "cannot see another org's membership roster", - "passed": true - }, - { - "name": "a viewer role in one org doesn't grant admin power in another org", - "passed": true - }, - { - "name": "multi-org user can act as admin in the org where they hold that role", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", - "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", - "run": 2, - "sourcePath": "claude-code-sonnet-5-no-skills/build-rls-003-org-roles-permissions/run-2/result.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-rls-003-org-roles-permissions", - "stage": "build", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on documents", - "passed": true - }, - { - "name": "viewer sees only documents in their org", - "passed": true + "passed": true }, { "name": "viewer cannot insert", @@ -19960,7 +20801,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 01a03e34-9960-75de-8e2e-62ab87a53620/receipt-alpha.pdf, 01a03e34-9960-75de-8e2e-62ab87a53620/receipt-beta.pdf" + "notes": "saw: 01a0803c-6695-7270-947f-f6862d9de00c/receipt-alpha.pdf, 01a0803c-6695-7270-947f-f6862d9de00c/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -19981,7 +20822,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT policies, no RLS disabling, and short-lived createSignedUrl sharing are all provided." + "judgeNotes": "Creates a private user-files bucket, authenticated owner-scoped SELECT/INSERT policies (plus update/delete), retains RLS, and uses createSignedUrl with expiration." } ], "skills": { @@ -20034,7 +20875,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 01a03e34-9b8a-762e-9718-2d008630edbc/receipt-alpha.pdf, 01a03e34-9b8a-762e-9718-2d008630edbc/receipt-beta.pdf" + "notes": "saw: 01a0803c-d85d-7720-aed3-2c4fa56a6b18/receipt-alpha.pdf, 01a0803c-d85d-7720-aed3-2c4fa56a6b18/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -20108,7 +20949,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 01a03e34-aea3-766d-a277-8ae3c4fde5e0/receipt-alpha.pdf, 01a03e34-aea3-766d-a277-8ae3c4fde5e0/receipt-beta.pdf" + "notes": "saw: 01a0803c-71f9-74d9-b704-e612dd6eadf1/receipt-alpha.pdf, 01a0803c-71f9-74d9-b704-e612dd6eadf1/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -20129,7 +20970,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Creates a private bucket, authenticated owner-scoped SELECT/INSERT policies, retains RLS, and uses createSignedUrl with expiration." + "judgeNotes": "Creates a private user-files bucket, applies authenticated owner-scoped SELECT and INSERT policies (plus UPDATE/DELETE), keeps RLS intact, and uses createSignedUrl with a 3600-second expiry." } ], "skills": { @@ -20174,12 +21015,12 @@ { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "4 passed, 2 failed" + "notes": "13 passed, 0 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies the `posts` SELECT policy as leaking cross-organization data, contrasts it with correctly isolated `notes`, and grounds the conclusion in the pgTAP failure (got 1, expected 0)." + "judgeNotes": "Correctly identifies the `posts` cross-tenant read policy flaw, contrasts it with the correctly isolated `notes` table, and grounds the conclusion in the pgTAP failures." } ], "skills": { @@ -20219,17 +21060,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/database/tenant_isolation.sql" + "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "7 passed, 4 failed" + "notes": "7 passed, 5 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies `posts` as leaking cross-tenant reads, citing failed pgTAP tests showing org A can read org B’s post and all posts." + "judgeNotes": "Correctly identifies the `posts` RLS policy as allowing cross-organization reads, grounds this in failing pgTAP results, and correctly distinguishes `notes` as isolated." } ], "skills": { @@ -20269,17 +21110,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" + "notes": "1 file(s): supabase/tests/database/tenant_isolation_test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "9 passed, 6 failed" + "notes": "7 passed, 2 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies the `posts` SELECT policy flaw and grounds it in failing pgTAP test 5, while noting `notes` isolation passed." + "judgeNotes": "Correctly identifies `posts` as having the tenant-isolation flaw, cites the two failing pgTAP assertions, and explains that authenticated members can read other organizations’ posts while `notes` remains isolated." } ], "skills": { @@ -20354,36 +21195,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"match document sections embeddings pgvector function edge function\", limit: 5) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/ai/examples/nextjs-vector-search", - "title": "Vector search with Next.js and OpenAI" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-columns", - "title": "Vector columns" - } - ], - "resultChars": 75797 - } - ] + "calls": [] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", @@ -20450,7 +21262,36 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"gte-small embedding dimensions edge function\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", + "title": "Semantic Search" + }, + { + "url": "https://supabase.com/docs/guides/ai/concepts", + "title": "Concepts" + }, + { + "url": "https://supabase.com/docs/guides/ai/quickstarts/generate-text-embeddings", + "title": "Generate Embeddings" + }, + { + "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", + "title": "Choosing your Compute Add-on" + } + ], + "resultChars": 49782 + } + ] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", @@ -20520,99 +21361,58 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"RAG with permissions row level security document_sections match_document_sections\", limit: 3) { nodes { ... on Guide { title href content } } } }", + "query": "{ searchDocs(query: \"row level security RLS vector search security invoker match function\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" }, { "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/rls-performance-and-best-practices-Z5Jjwv", + "title": "RLS Performance and Best Practices" + }, + { + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" } ], - "resultChars": 39915 - } - ] - }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "run": 3, - "sourcePath": "claude-code-sonnet-5-no-skills/build-vectors-001-rag-with-permissions/run-3/result.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", - "product": [ - "database" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "preserved existing app scrape job", - "passed": true - }, - { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "HTTPS Supabase Metrics API scrape is correctly configured with Basic Auth and password_file; matching secret volume is mounted, and the existing app scrape remains." - }, - { - "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README provides correct Secret API key creation, matching secret-file placement, reload/restart instructions, and concrete verification via Prometheus targets." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ + "resultChars": 79601 + }, { "source": "search_docs", - "query": "{ searchDocs(query: \"self-hosted metrics prometheus endpoint\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"create function match_documents embedding vector similarity search security invoker\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" + "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", + "title": "Semantic Search" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" + "url": "https://supabase.com/docs/guides/ai/vector-columns", + "title": "Vector columns" } ], - "resultChars": 19943 + "resultChars": 27613 } ] }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "run": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/deploy-database-001-prometheus-metrics/run-1/result.json" + "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", + "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", + "run": 3, + "sourcePath": "claude-code-sonnet-5-no-skills/build-vectors-001-rag-with-permissions/run-3/result.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -20633,7 +21433,7 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "preserved existing app scrape job", @@ -20641,13 +21441,13 @@ }, { "name": "configured the Supabase Metrics API scrape correctly", - "passed": false, - "judgeNotes": "Supabase scrape uses HTTP and host.docker.internal instead of HTTPS targeting .supabase.co or .supabase.red. The README’s hosted-project suggestion does not make the current prometheus.yml deployable as required." + "passed": true, + "judgeNotes": "Supabase HTTPS scrape uses the correct metrics path, Basic Auth with password_file, a valid project target, preserved app job, and matching Docker volume wiring." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README documents Secret API key creation, matching secret file placement, Compose restart/reload, correct Basic Auth endpoint, and verification via Prometheus targets or direct curl." + "judgeNotes": "README provides correct Secret API key creation, matching password_file placement, Prometheus restart, and concrete verification via the Prometheus targets page." } ], "skills": { @@ -20658,38 +21458,38 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"project metrics endpoint prometheus\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"metrics endpoint prometheus\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", - "title": "Scrape a project's metrics" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", + "url": "https://supabase.com/docs/guides/observability/metrics", "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-self-hosted", "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-cloud", "title": "Metrics API with Grafana Cloud" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", + "url": "https://supabase.com/docs/guides/observability/metrics/vendor-agnostic", "title": "Vendor-agnostic Metrics API setup" + }, + { + "url": "https://supabase.com/docs/guides/observability/access-data", + "title": "Observe the data" } ], - "resultChars": 20571 + "resultChars": 22344 } ] }, "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "run": 2, - "sourcePath": "claude-code-sonnet-5-no-skills/deploy-database-001-prometheus-metrics/run-2/result.json" + "run": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/deploy-database-001-prometheus-metrics/run-1/result.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -20710,7 +21510,7 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { "name": "preserved existing app scrape job", @@ -20719,12 +21519,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "HTTPS Supabase Metrics API scrape is correctly configured with the required path, Basic Auth password_file, project target, preserved app job, and matching read-only secret volume mount." + "judgeNotes": "HTTPS Supabase scrape uses the correct metrics path and project target, Basic Auth with password_file, preserves the app job, and mounts the secrets directory containing the password file." }, { "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README provides correct Secret API key setup, matching password_file placement, Prometheus reload, and concrete verification via the targets page." + "passed": false, + "judgeNotes": "The secret and verification steps are correct, but `docker compose ... up -d` does not reliably restart an already-running unchanged Prometheus container. The README must use an actual restart/reload or forced recreation command so the credential is picked up." } ], "skills": { @@ -20743,23 +21543,128 @@ "title": "Scrape a project's metrics" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", + "url": "https://supabase.com/docs/guides/observability/metrics", "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-self-hosted", "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "url": "https://supabase.com/docs/guides/observability/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" + }, + { + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-cloud", "title": "Metrics API with Grafana Cloud" + } + ], + "resultChars": 20878 + } + ] + }, + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", + "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", + "run": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/deploy-database-001-prometheus-metrics/run-2/result.json" + }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "deploy-database-001-prometheus-metrics", + "stage": "deploy", + "product": [ + "database" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "preserved existing app scrape job", + "passed": true + }, + { + "name": "configured the Supabase Metrics API scrape correctly", + "passed": true, + "judgeNotes": "Valid HTTPS Supabase Metrics API scrape with Basic Auth password_file, matching mounted secret path, correct endpoint and project target, while preserving the app job." + }, + { + "name": "documented live deployment and verification steps", + "passed": true, + "judgeNotes": "README provides Secret API key creation, matching password file setup, Compose/reload commands, and concrete verification through Prometheus targets and authenticated endpoint curl." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"project metrics endpoint prometheus\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", + "title": "Scrape a project's metrics" + }, + { + "url": "https://supabase.com/docs/guides/observability/metrics", + "title": "Metrics API" + }, + { + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", + "url": "https://supabase.com/docs/guides/observability/metrics/vendor-agnostic", "title": "Vendor-agnostic Metrics API setup" + }, + { + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" + } + ], + "resultChars": 20878 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"create api key management api secret sb_secret\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", + "title": "Build a User Management App with Angular" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" } ], - "resultChars": 20571 + "resultChars": 144475 } ] }, @@ -20853,7 +21758,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment via Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -20905,7 +21810,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -21100,7 +22005,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "victim session active before delete-account", @@ -21121,8 +22026,7 @@ }, { "name": "deleted user cannot sign back in", - "passed": false, - "notes": "deleted account can still sign in" + "passed": true }, { "name": "other users keep their sessions and access", @@ -21131,7 +22035,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "Correctly diagnoses the soft-delete issue, implements permanent auth blocking plus session/refresh-token revocation, closes the Data API stale-JWT window through RLS, notes JWTs remain locally valid until expiry, and accurately distinguishes publishable and secret keys." + "judgeNotes": "Correctly identifies the soft-delete/auth-session issue, implements deletion of the auth user with cascading session and refresh-token revocation, accurately explains stale JWT behavior and closes the shown Data API paths, and correctly distinguishes publishable frontend keys from server-only secret keys that bypass RLS." } ], "skills": { @@ -21196,8 +22100,8 @@ }, { "name": "diagnosed and explained session revocation", - "passed": false, - "judgeNotes": "The revocation/RLS fix and key guidance are largely correct, but it incorrectly says `supabase.auth.getUser()` only validates the JWT signature and can still succeed. `getUser()` performs a server-side Auth check and is a mitigation for stale stateless JWTs; `getClaims()` or local signature/expiry middleware are the checks that continue accepting the token until expiry." + "passed": true, + "judgeNotes": "Correctly diagnoses the soft-delete-only flow, revokes sessions and refresh tokens while blocking future sign-ins, closes Data API access via live-state RLS checks, explains stale JWT behavior, and accurately distinguishes frontend publishable keys from server-only secret keys. Minor issue: auth.getUser() performs server-side validation; getClaims() is the local-validation example." } ], "skills": { @@ -21205,7 +22109,34 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"delete user account RPC self delete auth.users\", limit: 5) { nodes { title href ... on Guide { content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", + "title": "deleteUser()" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" + } + ], + "resultChars": 21795 + } + ] }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", @@ -21262,8 +22193,8 @@ }, { "name": "diagnosed and explained session revocation", - "passed": false, - "judgeNotes": "The fix revokes sessions and bans future sign-ins, but it does not delete the auth user or remove their identity as required. It also says there is practically no remaining window without clearly warning that purely local JWT validation (for example getClaims/custom middleware) will still accept the token until expiry." + "passed": true, + "judgeNotes": "Correctly diagnoses soft deletion, implements real login/session/refresh-token revocation, closes the demonstrated Data API window with RLS, accurately explains residual stateless-JWT validity for local validation, and correctly distinguishes publishable versus server-only secret keys." } ], "skills": { @@ -21271,64 +22202,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"ban user banned_until sign in token refresh\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - }, - { - "url": "https://supabase.com/docs/guides/platform/temporary-access", - "title": "Temporary access" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - } - ], - "resultChars": 78736 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key anon key service_role key migration RLS\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", - "title": "Build a User Management App with SvelteKit" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" - } - ], - "resultChars": 139212 - } - ] + "calls": [] }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", @@ -21381,7 +22255,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified missing orders publication membership, added only public.orders to supabase_realtime, and preserved RLS, policies, and courier_locations." + "judgeNotes": "Correctly identified orders missing from supabase_realtime, added only public.orders to the existing publication, and preserved courier_locations, RLS, and policies." } ], "skills": { @@ -21442,7 +22316,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified missing publication membership and added public.orders to the existing supabase_realtime publication without altering RLS, policies, or courier_locations." + "judgeNotes": "Correctly identified the missing orders table in supabase_realtime, added only public.orders to the existing publication, and preserved RLS, policies, and courier_locations." } ], "skills": { @@ -21503,7 +22377,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified missing publication membership, added public.orders to the existing supabase_realtime publication, and preserved RLS, policies, and courier_locations." + "judgeNotes": "Correctly identified the missing orders table in supabase_realtime, added only public.orders, verified it, and preserved courier_locations, RLS, and policies." } ], "skills": { @@ -21542,17 +22416,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified `image-transform` and documented all 8 recurring gateway-level HTTP 503 failures from 07:00Z through 12:00Z on 2026-04-28." + "judgeNotes": "Identified image-transform and documented all 8 recurring HTTP 503 failures from 07:00Z through 12:00Z on 2026-04-28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": false, - "judgeNotes": "Although it notes gateway-only 503s with no runtime rows, it ultimately attributes them to function cold-start/import costs and recommends function changes. The rubric requires platform/gateway attribution, not function runtime/code remediation." + "judgeNotes": "Although it correctly notes the 503s exist only in gateway logs with no matching function invocations and an unchanged deployment, it ultimately attributes them to function cold-start/import behavior and recommends modifying/keeping warm/instrumenting the function. The rubric requires a platform-layer cause rather than function/runtime remediation." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant provided multiple concrete actions: add a cron warm-up, reduce module initialization cost, implement retry/backoff, decouple transformation from upload, and confirm runtime boot/CPU-limit errors with Supabase support." + "judgeNotes": "The assistant provided multiple concrete actions, including scheduled warm-up pings, cold-start profiling, retry/backoff, dependency decoupling, and boot instrumentation." } ], "skills": { @@ -21591,17 +22465,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform and the recurring eight HTTP 503 gateway failures across 07:00Z–12:00Z on 2026-04-28, while correctly distinguishing older billing-webhook errors." + "judgeNotes": "Identified `image-transform` as the affected function and documented all 8 recurring HTTP 503 gateway failures from 07:00Z through 12:00Z on 2026-04-28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Correctly attributes the recurring 503s to the gateway/platform layer, supported by absent function invocation logs, unchanged deployment version, successful nearby invocations, and contrast with avatar-upload’s function-level 500." + "judgeNotes": "Correctly attributes the recurring 503s to the gateway/platform layer, supported by their presence only in gateway logs with no function invocation metadata or runtime rows, while successful requests reached the unchanged deployment. It also distinguishes avatar-upload's executed 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Recommended specific actions: inspect Edge Function platform/scaling health for the exact time window, review gateway timeout settings, add retries or warm-up pings, and investigate the separate function error if it recurs." + "judgeNotes": "Provides concrete next steps, including escalating to Supabase with gateway request IDs and the exact time window, plus adding retries and 503-specific alerting." } ], "skills": { @@ -21640,17 +22514,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified `image-transform` and detailed all 8 recurring HTTP 503 gateway failures from 07:00Z–12:00Z on 2026-04-28, correctly distinguishing unrelated billing-webhook errors." + "judgeNotes": "Identified image-transform and the recurring eight HTTP 503 gateway failures across 07:00Z–12:00Z on 2026-04-28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Attributes the 503s to the gateway/router before function invocation, grounded in missing deployment/version fields on 503s, unchanged version 42 on successful retries, and distinction from avatar-upload’s function-level 500." + "judgeNotes": "Attributes the recurring 503s to the gateway/platform layer, supported by missing invocation fields, unchanged deployment ID, successful nearby invocations, and contrast with avatar-upload’s function-level 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Provides multiple concrete actions: add retries with backoff, investigate Edge Function cold-start/resource limits, keep the function warm or increase resources, decouple processing asynchronously, and add per-function 503 alerting." + "judgeNotes": "Provides multiple concrete actions: inspect function resource limits and cold-start factors, add retry/backoff, investigate the specific 09:00 application error, and escalate to infrastructure if gateway-level 503s recur." } ], "skills": { @@ -21716,7 +22590,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly identifies deny-all RLS with no policies, keeps RLS enabled, and creates authenticated SELECT and INSERT policies scoped to user_id = auth.uid() using USING and WITH CHECK." + "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and created authenticated owner-scoped SELECT and INSERT policies using auth.uid()." } ], "skills": { @@ -21782,7 +22656,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and added authenticated-only owner-scoped SELECT and INSERT policies using auth.uid()." + "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and added authenticated-only SELECT and INSERT policies scoped by user_id = auth.uid(), using USING and WITH CHECK respectively." } ], "skills": { @@ -21848,7 +22722,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly identifies deny-all RLS with no policies, keeps RLS enabled, and adds authenticated-only owner-scoped SELECT and INSERT policies using auth.uid()." + "judgeNotes": "Correctly identified deny-all RLS with no policies, kept RLS enabled, and added authenticated owner-scoped SELECT and INSERT policies using auth.uid()." } ], "skills": { @@ -21907,7 +22781,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "`supabase db push` (#19) applied `20240220000000_add_avatar_url.sql` and finished successfully. The orphan bio history was reconciled by adding local `20240115000000_add_profile_bio.sql` (#17), allowing the CLI push to proceed. No prohibited workaround was used." + "judgeNotes": "`supabase db push` (#16) applied `20240220000000_add_avatar_url.sql` and finished successfully. The orphan bio history was reconciled by adding local `20240115000000_add_profile_bio.sql` (#14), then pushing via #16. The `psql` commands were read-only; no prohibited workaround occurred." } ], "skills": { @@ -21966,7 +22840,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "`supabase db push` (#17) applied `20240220000000_add_avatar_url.sql` and finished successfully. The orphan history was reconciled by adding local `20240115000000_add_profile_bio.sql` (#15), confirmed aligned by `supabase migration list` (#16), then proceeding through the successful CLI push. The `psql` commands were read-only inspections; no prohibited workaround was used." + "judgeNotes": "`supabase db push` (#15) applied `20240220000000_add_avatar_url.sql` and finished successfully. History was reconciled by adding local `20240115000000_add_profile_bio.sql`, then the same CLI push accepted it. No prohibited workaround occurred; `psql` was read-only inspection." } ], "skills": { @@ -22025,7 +22899,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "`supabase db push` (#19) applied `20240220000000_add_avatar_url.sql` and finished successfully. The orphan history was reconciled by adding the matching local `20240115000000_add_profile_bio.sql` file (#17), confirmed by `supabase migration list` (#18), allowing the push to proceed. No prohibited direct-SQL mutation or prepared-statement workaround occurred; the `psql` commands were read-only inspections." + "judgeNotes": "`supabase db push` (#17) applied `20240220000000_add_avatar_url.sql` and finished successfully. The orphan bio history was reconciled by adding local `20240115000000_add_profile_bio.sql` (#15), confirmed aligned by `supabase migration list` (#16), then pushing. The `psql` commands were read-only inspections; no prohibited workaround occurred." } ], "skills": { @@ -22077,119 +22951,119 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_id_created_at (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" - }, - { - "name": "inserts still work", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "run": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/resolve-performance-001-slow-query-cpu-spike/run-1/result.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "observability", - "sql" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", - "passed": true - }, - { - "name": "created index covering user_id and created_at", - "passed": true - }, - { - "name": "query plan uses an index and avoids sequential scan", - "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_id_created_at (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" - }, - { - "name": "inserts still work", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "run": 2, - "sourcePath": "claude-code-sonnet-5-no-skills/resolve-performance-001-slow-query-cpu-spike/run-2/result.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "observability", - "sql" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", - "passed": true - }, - { - "name": "created index covering user_id and created_at", - "passed": true - }, - { - "name": "query plan uses an index and avoids sequential scan", - "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_id_created_at (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + }, + { + "name": "inserts still work", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "run": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/resolve-performance-001-slow-query-cpu-spike/run-1/result.json" + }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "resolve-performance-001-slow-query-cpu-spike", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "observability", + "sql" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "inspected pg_stat_statements for query performance", + "passed": true + }, + { + "name": "ran EXPLAIN on the expensive query", + "passed": true + }, + { + "name": "created index covering user_id and created_at", + "passed": true + }, + { + "name": "query plan uses an index and avoids sequential scan", + "passed": true, + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + }, + { + "name": "inserts still work", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "run": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/resolve-performance-001-slow-query-cpu-spike/run-2/result.json" + }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "resolve-performance-001-slow-query-cpu-spike", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "observability", + "sql" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "inspected pg_stat_statements for query performance", + "passed": true + }, + { + "name": "ran EXPLAIN on the expensive query", + "passed": true + }, + { + "name": "created index covering user_id and created_at", + "passed": true + }, + { + "name": "query plan uses an index and avoids sequential scan", + "passed": true, + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -22425,12 +23299,12 @@ "sourcePath": "claude-code-sonnet-5-no-skills/resolve-security-002-rls-cross-tenant-leak/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-auth-001-email-password-flow", @@ -22446,7 +23320,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "auth module loads and the driver completes", @@ -22456,7 +23330,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user aa77048d-9b87-493c-8b1e-31a01839c8b2, signUp returned {\"userId\":\"aa77048d-9b87-493c-8b1e-31a01839c8b2\"}" + "notes": "db user a3325096-94dd-4cf2-a89f-c78c255cd284, signUp returned {\"userId\":\"a3325096-94dd-4cf2-a89f-c78c255cd284\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -22471,7 +23345,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"aa77048d-9b87-493c-8b1e-31a01839c8b2\"}" + "notes": "{\"userId\":\"a3325096-94dd-4cf2-a89f-c78c255cd284\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -22485,8 +23359,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "passed": true, + "notes": "imports found in: app/src/auth.mjs" } ], "skills": { @@ -22502,158 +23376,43 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"sign up sign in auth REST API createUser getUser getSession localStorage supabase-js\", limit: 5) {\n nodes {\n ... on Guide {\n title\n href\n content\n }\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n ... on TroubleshootingGuide {\n title\n href\n content\n }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", - "title": "Login with Google" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", - "title": "Login with Figma" - } - ], - "resultChars": 86837 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"signUpWithPassword signInWithPassword supabase-js auth createClient persistSession localStorage\", limit: 10) {\n nodes {\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n ... on Guide {\n title\n href\n content\n }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", - "title": "Login with Figma" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-slack", - "title": "Login with Slack" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-github", - "title": "Login with GitHub" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-bitbucket", - "title": "Login with Bitbucket" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-zoom", - "title": "Login with Zoom" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-twitter", - "title": "Login with X / Twitter" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-gitlab", - "title": "Login with GitLab" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-azure", - "title": "Login with Azure (Microsoft)" - } - ], - "resultChars": 206448 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"signUp options data user_metadata display_name supabase-js\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n }\n }\n}", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i -C 2 'breaking-change|auth|supabase-js' | sed -n '1,200p'; curl -fsSL https://supabase.com/docs/reference/javascript/auth-signup.md | sed -n '1,180p'; curl -fsSL https://supabase.com/docs/reference/javascript/auth-signinwithpassword.md | sed -n '1,160p'; curl -fsSL https://supabase.com/docs/reference/javascript/select.md | sed -n '1,160p'; supabase --version; supabase --help | sed -n '1,180p'; supabase status -o env; psql postgresql://postgres:postgres@127.0.0.1:54322/postgres -X -v ON_ERROR_STOP=1 -c \\\"select schemaname, tablename, policyname, roles, cmd, qual, with_check from pg_policies where schemaname = 'public' and tablename = 'profiles';\\\" -c \\\"select p.oid::regprocedure, p.prosecdef, p.proacl from pg_proc p join pg_namespace n on n.oid=p.pronamespace where n.nspname='public' and p.proname='handle_new_user';\\\"\"", "pages": [ { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" + "url": "https://supabase.com/changelog.md" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" + "url": "https://supabase.com/docs/reference/javascript/auth-signup.md" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword.md" }, { - "url": "https://supabase.com/docs/reference/dart/auth-signup" + "url": "https://supabase.com/docs/reference/javascript/select.md" } ], - "resultChars": 25089 + "resultChars": 6061 }, { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"raw_user_meta_data signup data top-level auth/v1/signup gotrue\", limit: 10) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-hooks/before-user-created-hook", - "title": "Before User Created Hook" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signup" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" - }, - { - "url": "https://supabase.com/docs/guides/local-development/customizing-email-templates", - "title": "Customizing email templates" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-signup", - "title": "signUp()" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - } - ], - "resultChars": 82262 + "source": "web_search", + "query": "site:supabase.com/docs/reference/javascript/auth-signup signUp options data user metadata ...", + "pages": [] } ] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini/build-auth-001-email-password-flow/run-1/result.json" + "sourcePath": "codex-gpt-5.6/build-auth-001-email-password-flow/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-auth-001-email-password-flow", @@ -22669,7 +23428,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "auth module loads and the driver completes", @@ -22679,12 +23438,12 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user ca340fc1-7083-4dff-ac40-e4efc402ff34, signUp returned {\"userId\":\"ca340fc1-7083-4dff-ac40-e4efc402ff34\"}" + "notes": "db user 5accc7bd-7bcb-499f-ae04-b93a0f18d51a, signUp returned {\"userId\":\"5accc7bd-7bcb-499f-ae04-b93a0f18d51a\"}" }, { "name": "signup metadata reaches the profile (display name)", - "passed": false, - "notes": "profiles.display_name = \"alex-mta46447\"" + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" }, { "name": "wrong password is rejected gracefully (no throw, no session)", @@ -22694,12 +23453,12 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"ca340fc1-7083-4dff-ac40-e4efc402ff34\"}" + "notes": "{\"userId\":\"5accc7bd-7bcb-499f-ae04-b93a0f18d51a\"}" }, { "name": "getMyProfile returns the signed-in user's profile", - "passed": false, - "notes": "{\"displayName\":\"alex-mta46447\",\"plan\":\"free\"}" + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" }, { "name": "app code does not use the secret / service-role key", @@ -22708,8 +23467,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "passed": true, + "notes": "imports found in: app/src/auth.mjs" } ], "skills": { @@ -22725,170 +23484,54 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"supabase-js signUp signInWithPassword getUser getSession auth client JavaScript\", limit: 5) {\n nodes {\n __typename\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n ... on Guide {\n title\n href\n content\n }\n }\n totalCount\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", - "title": "Login with Figma" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-gitlab", - "title": "Login with GitLab" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-bitbucket", - "title": "Login with Bitbucket" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - } - ], - "resultChars": 101116 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"signInWithPassword JavaScript reference supabase-js getUser getSession signUp user_metadata display_name\", limit: 10) {\n nodes {\n __typename\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n ... on Guide {\n title\n href\n content\n }\n }\n totalCount\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", - "title": "Login with Google" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", - "title": "signInWithPassword()" - } - ], - "resultChars": 121037 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"auth/v1/signup data user_metadata supabase signup http payload\", limit: 5) {\n nodes {\n __typename\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on Guide { title href content }\n }\n totalCount\n }\n}", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"find .claude/skills/supabase-postgres-best-practices/references -maxdepth 1 -type f -printf '%f\\\\n' | sort | sed -n '1,200p'\nsupabase --version\nsupabase status -o env\ncurl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|auth|supabase-js' | head -100\ncurl -fsSL https://supabase.com/docs/reference/javascript/auth-signup.md | rg -n -i 'signUp|options|data|email|password' | head -100\ncurl -fsSL https://supabase.com/docs/reference/javascript/auth-signinwithpassword.md | rg -n -i 'signInWithPassword|email|password|error' | head -100\ncurl -fsSL https://supabase.com/docs/reference/javascript/auth-getuser.md | rg -n -i 'getUser|authenticated|network|user' | head -100\"", "pages": [ { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-hooks/before-user-created-hook", - "title": "Before User Created Hook" + "url": "https://supabase.com/changelog.md" }, { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - } - ], - "resultChars": 65093 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"reference javascript auth-signinwithpassword HTTP curl body JSON password grant\", limit: 3) {\n nodes {\n __typename\n ... on ClientLibraryFunctionReference { title href content language methodName }\n }\n totalCount\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" + "url": "https://supabase.com/docs/reference/javascript/auth-signup.md" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword.md" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-passkey-verifyauthentication" + "url": "https://supabase.com/docs/reference/javascript/auth-getuser.md" } ], - "resultChars": 2349 + "resultChars": 20556 }, { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"auth refresh token grant_type refresh_token supabase-js reference javascript\", limit: 5) {\n nodes {\n __typename\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on Guide { title href content }\n }\n totalCount\n }\n}", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"sed -n '1,260p' .claude/skills/supabase-postgres-best-practices/references/security-rls-basics.md\nsed -n '1,220p' .claude/skills/supabase-postgres-best-practices/references/security-rls-performance.md\nsed -n '1,220p' .claude/skills/supabase-postgres-best-practices/references/security-privileges.md\ncurl -fsSL 'https://supabase.com/docs/reference/javascript/auth-signup?format=markdown' | rg -n -i 'signUp|options|data|email|password' | head -100\ncurl -fsSL 'https://supabase.com/docs/reference/javascript/auth-signinwithpassword?format=markdown' | rg -n -i 'signInWithPassword|email|password' | head -100\ncurl -fsSL 'https://supabase.com/docs/reference/javascript/auth-getuser?format=markdown' | rg -n -i 'getUser|authentic|network' | head -100\"", "pages": [ { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/advanced-guide", - "title": "Advanced guide" - }, - { - "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", - "title": "Build a Supabase Integration" + "url": "https://supabase.com/docs/reference/javascript/auth-signup?format=markdown" }, { - "url": "https://supabase.com/docs/guides/auth/social-login", - "title": "Social Login" - } - ], - "resultChars": 110766 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"reference javascript auth-signup HTTP curl signup body data options\", limit: 3) {\n nodes {\n __typename\n ... on ClientLibraryFunctionReference { title href content language methodName }\n }\n totalCount\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword?format=markdown" }, { - "url": "https://supabase.com/docs/reference/dart/auth-signup" + "url": "https://supabase.com/docs/reference/javascript/auth-getuser?format=markdown" } ], - "resultChars": 3295 + "resultChars": 93732 } ] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini/build-auth-001-email-password-flow/run-2/result.json" + "sourcePath": "codex-gpt-5.6/build-auth-001-email-password-flow/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-auth-001-email-password-flow", @@ -22904,7 +23547,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "auth module loads and the driver completes", @@ -22914,7 +23557,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 6e67aa11-b23a-434b-ad2c-5be2e7029b1c, signUp returned {\"userId\":\"6e67aa11-b23a-434b-ad2c-5be2e7029b1c\"}" + "notes": "db user ee33db47-1535-42bf-b17a-088eb68f37d1, signUp returned {\"userId\":\"ee33db47-1535-42bf-b17a-088eb68f37d1\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -22929,12 +23572,12 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"6e67aa11-b23a-434b-ad2c-5be2e7029b1c\"}" + "notes": "{\"userId\":\"ee33db47-1535-42bf-b17a-088eb68f37d1\"}" }, { "name": "getMyProfile returns the signed-in user's profile", - "passed": false, - "notes": "{\"error\":\"No user is signed in\"}" + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" }, { "name": "app code does not use the secret / service-role key", @@ -22943,8 +23586,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "passed": true, + "notes": "imports found in: app/src/auth.mjs" } ], "skills": { @@ -22960,86 +23603,43 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"supabase-js signUp signInWithPassword getUser profile metadata raw_user_meta_data\", limit: 5) {\n nodes {\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n ... on Guide {\n title\n href\n content\n }\n ... on TroubleshootingGuide {\n title\n href\n content\n }\n }\n }\n}", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"set -o pipefail\ncurl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|auth|supabase-js' | head -80\nprintf '\\\\n--- auth docs snippets ---\\\\n'\ncurl -fsSL https://supabase.com/docs/reference/javascript/auth-signup.md | rg -n -i 'signUp|options|data|email|password' | head -80\ncurl -fsSL https://supabase.com/docs/reference/javascript/auth-signinwithpassword.md | rg -n -i 'signInWithPassword|email|password|error' | head -60\ncurl -fsSL https://supabase.com/docs/reference/javascript/select.md | rg -n -i '\\\\.select|single|error' | head -60\nprintf '\\\\n--- best practices ---\\\\n'\nfor f in security-rls-basics.md security-rls-performance.md security-privileges.md schema-primary-keys.md schema-foreign-key-indexes.md; do echo \\\"--- \"'$f\"; sed -n '\"'1,240p' \\\"skills/supabase-postgres-best-practices/references/\"'$f\"; done\nprintf '\"'\\\\n--- workspace state ---\\\\n'\ngit status --short\nsupabase --version\nsupabase status -o env 2>&1\nprintf '\\\\n--- npm versions ---\\\\n'\ncd app && npm view @supabase/supabase-js version\"", "pages": [ { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" + "url": "https://supabase.com/changelog.md" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" + "url": "https://supabase.com/docs/reference/javascript/auth-signup.md" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword.md" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nuxt-3", - "title": "Build a User Management App with Nuxt 3" + "url": "https://supabase.com/docs/reference/javascript/select.md" } ], - "resultChars": 43745 + "resultChars": 10472 }, { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"JavaScript reference signUp signInWithPassword getSession getUser authenticated profile\", limit: 10) {\n nodes {\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n ... on Guide {\n title\n href\n content\n }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", - "title": "signInWithPassword()" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithoauth" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinanonymously" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" - }, - { - "url": "https://supabase.com/docs/reference/csharp/auth-signinwithpassword", - "title": "SignIn(email, password)" - } - ], - "resultChars": 10286 + "source": "web_search", + "query": "site:supabase.com/docs/reference/javascript auth signUp options data signInWithPassword ...", + "pages": [] } ] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini/build-auth-001-email-password-flow/run-3/result.json" + "sourcePath": "codex-gpt-5.6/build-auth-001-email-password-flow/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-cli-001-bootstrap-app", @@ -23102,85 +23702,37 @@ "calls": [ { "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,120p'\"", - "hasContent": true, + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking|row.level|rls|auth|rest|api|migration|local' | sed -n '1,160p'; printf '%s\\\\n' '--- local development docs ---'; curl -fsSL https://supabase.com/docs/guides/local-development.md | sed -n '1,220p'; printf '%s\\\\n' '--- API security docs ---'; curl -fsSL https://supabase.com/docs/guides/api/securing-your-api.md | sed -n '1,280p'; printf '%s\\\\n' '--- RLS docs relevant excerpts ---'; curl -fsSL https://supabase.com/docs/guides/database/postgres/row-level-security.md | rg -n -C 4 'authenticated|enable row level|SELECT|select policy|privilege|grant' | sed -n '1,240p'\"", "pages": [ { "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 6693 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"local development migrations RLS exposed schema Data API anonymous authenticated grants\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/api/using-custom-schemas", - "title": "Using Custom Schemas" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - } - ], - "resultChars": 70052 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase CLI migration new db pull local declarative schema\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-pull", - "title": "Pull schema from the remote database" - }, - { - "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", - "title": "Declarative database schemas" + "url": "https://supabase.com/docs/guides/local-development.md" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" + "url": "https://supabase.com/docs/guides/api/securing-your-api.md" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", - "title": "Build a User Management App with Angular" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security.md" } ], - "resultChars": 88514 + "resultChars": 28318 } ] }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini/build-cli-001-bootstrap-app/run-1/result.json" + "sourcePath": "codex-gpt-5.6/build-cli-001-bootstrap-app/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-cli-001-bootstrap-app", @@ -23242,47 +23794,47 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security expose table data api anon authenticated select policy insert update delete migrations local development declarative schema\", limit: 5) { nodes { title href content } } }", - "hasContent": true, + "source": "web_search", + "query": "https://supabase.com/changelog.md", "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i -C 2 'breaking-change|row.level|rls|migration|local development' | head -160\ncurl -fsSL https://supabase.com/docs/guides/local-development/cli/getting-started.md | sed -n '1,220p'\ncurl -fsSL https://supabase.com/docs/guides/database/postgres/row-level-security.md | rg -n -C 3 'authenticated|create policy|enable row level|grant|anon' | head -220\ncurl -fsSL https://supabase.com/docs/guides/api/securing-your-api.md | sed -n '1,260p'\"", + "pages": [ { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/changelog.md" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/local-development/cli/getting-started.md" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security.md" }, { - "url": "https://supabase.com/docs/guides/realtime/authorization", - "title": "Realtime Authorization" + "url": "https://supabase.com/docs/guides/api/securing-your-api.md" } ], - "resultChars": 88521 + "resultChars": 28825 } ] }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini/build-cli-001-bootstrap-app/run-2/result.json" + "sourcePath": "codex-gpt-5.6/build-cli-001-bootstrap-app/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-cli-001-bootstrap-app", @@ -23344,67 +23896,38 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"RLS exposing table to Data API anon authenticated select policy insert update local development migration\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ManagementApiReference { title href content } ... on CLICommandReference { title href content } } totalCount } }", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md -o /tmp/supabase-changelog.md\nrg -n -i -C 2 'breaking.change|row.level|rls|data api|postgrest|auth' /tmp/supabase-changelog.md | sed -n '1,220p'\ncurl -fsSL https://supabase.com/docs/guides/local-development/cli/getting-started.md | sed -n '1,240p'\ncurl -fsSL https://supabase.com/docs/guides/local-development/declarative-database-schemas.md | sed -n '1,180p'\ncurl -fsSL https://supabase.com/docs/guides/database/postgres/row-level-security.md | sed -n '1,260p'\ncurl -fsSL https://supabase.com/docs/guides/api/securing-your-api.md | sed -n '1,240p'\"", "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-react", - "title": "Build a User Management App with Ionic React" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/local-development/cli/getting-started.md" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - } - ], - "resultChars": 146018 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Securing your API grants RLS authenticated anon select only rows empty response\", limit: 3) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } totalCount } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-select-returning-an-empty-data-array-and-i-have-data-in-the-table-xvOPgx", - "title": "Why is my select returning an empty data array and I have data in the table?" + "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas.md" }, { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0026_pg_graphql_anon_table_exposed", - "title": "Database Advisor: Lint 0026_pg_graphql_anon_table_exposed" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security.md" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/docs/guides/api/securing-your-api.md" } ], - "resultChars": 32294 + "resultChars": 51215 } ] }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini/build-cli-001-bootstrap-app/run-3/result.json" + "sourcePath": "codex-gpt-5.6/build-cli-001-bootstrap-app/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-cli-002-declarative-schema", @@ -23450,33 +23973,39 @@ "docs": { "calls": [ { - "source": "web_search", - "query": "https://supabase.com/changelog.md", + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | head -n 120\nfind . -maxdepth 4 -type f -print | sort\nsupabase --version\nsupabase migration --help\nif [ -f supabase/config.toml ]; then sed -n '1,240p' supabase/config.toml; fi\"", "pages": [ { "url": "https://supabase.com/changelog.md" } - ] + ], + "resultChars": 2437 }, { - "source": "web_search", - "query": "site:supabase.com changelog breaking-change supabase", - "pages": [] + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/docs/guides/local-development/declarative-database-schemas.md | sed -n '1,220p'\nsed -n '1,240p' skills/supabase-postgres-best-practices/references/schema-data-types.md\nsed -n '1,240p' supabase/schemas/products.sql\nsed -n '1,240p' supabase/migrations/20260101000000_create_products.sql\ngit status --short\nsupabase status\"", + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas.md" + } + ], + "resultChars": 1093 } ] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini/build-cli-002-declarative-schema/run-1/result.json" + "sourcePath": "codex-gpt-5.6/build-cli-002-declarative-schema/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-cli-002-declarative-schema", @@ -23523,57 +24052,40 @@ "calls": [ { "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,160p'\"", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,220p'\"", "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 9005 + "resultChars": 12609 }, { - "source": "search_docs", - "query": "query { searchDocs(query: \"declarative database schemas supabase migration generate schema_paths alter table add column\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } ... on TroubleshootingGuide { title href content } } totalCount } }", + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/docs/guides/local-development/declarative-database-schemas.md | sed -n '1,260p'\"", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", - "title": "Declarative database schemas" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" + "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas.md" } ], - "resultChars": 74593 + "resultChars": 8751 } ] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini/build-cli-002-declarative-schema/run-2/result.json" + "sourcePath": "codex-gpt-5.6/build-cli-002-declarative-schema/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-cli-002-declarative-schema", @@ -23619,47 +24131,32 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"declarative database schemas supabase db pull local schema_paths\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"sed -n '1,220p' skills/supabase-postgres-best-practices/references/schema-data-types.md; sed -n '1,220p' supabase/config.toml; sed -n '1,220p' supabase/schemas/products.sql; sed -n '1,220p' supabase/migrations/20260101000000_create_products.sql; git status --short; supabase --version; supabase status 2>&1 || true; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking.change|declarative|migration|schema' | head -n 100; curl -fsSL https://supabase.com/docs/guides/local-development/declarative-database-schemas.md | sed -n '1,160p'\"", "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", - "title": "Declarative database schemas" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-pull", - "title": "Pull schema from the remote database" + "url": "https://supabase.com/changelog.md" }, { - "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches", - "title": "Working with branches" + "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas.md" } ], - "resultChars": 64546 + "resultChars": 11981 } ] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini/build-cli-002-declarative-schema/run-3/result.json" + "sourcePath": "codex-gpt-5.6/build-cli-002-declarative-schema/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-cli-003-pg-cron-queue-workflow", @@ -23707,87 +24204,29 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"pg_cron scheduled job queue pgmq edge function enqueue message Supabase\", limit: 8) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } ... on ManagementApiReference { title href content } ... on ClientLibraryFunctionReference { title methodName language content href } ... on TroubleshootingGuide { title href content } } totalCount } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - } - ], - "resultChars": 84413 + "source": "web_search", + "query": "site:supabase.com/docs/guides/queues pgmq send read delete archive queue ...", + "pages": [] }, { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Function database access SUPABASE_DB_URL postgresjs local runtime env vars\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title methodName language content href } ... on TroubleshootingGuide { title href content } } totalCount } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - } - ], - "resultChars": 36141 + "source": "web_search", + "query": "site:supabase.com/docs/guides/cron/quickstart cron.schedule SQL job_name schedule command ...", + "pages": [] } ] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini/build-cli-003-pg-cron-queue-workflow/run-1/result.json" + "sourcePath": "codex-gpt-5.6/build-cli-003-pg-cron-queue-workflow/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-cli-003-pg-cron-queue-workflow", @@ -23814,12 +24253,12 @@ { "name": "cron command enqueues to the 'tasks' queue", "passed": true, - "notes": "queue depth 1 -> 2" + "notes": "queue depth 0 -> 1" }, { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 4) from the queue" + "notes": "function removed the seeded message (id 36) from the queue" } ], "skills": { @@ -23835,187 +24274,47 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Queues send read delete pop archive edge function local\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/architecture", - "title": "Edge Functions Architecture" - }, - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgmq", - "title": "pgmq: Queues" - } - ], - "resultChars": 24051 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"pg_cron schedule cron.job queue pgmq enqueue read delete Supabase local\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - } - ], - "resultChars": 26258 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Queues create queue SQL pgmq.create queue\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - }, - { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", - "title": "Expose Queues for local and self-hosted Supabase" - } - ], - "resultChars": 49530 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"pgmq create queue first send auto create queue Supabase\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", - "title": "Expose Queues for local and self-hosted Supabase" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - } - ], - "resultChars": 39378 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"cron.unschedule jobname cron job id unschedule Supabase\", limit: 5) { nodes { title href content } } }", - "hasContent": true, + "source": "web_search", + "query": "https://supabase.com/changelog.md", "pages": [ { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" - }, - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_cron", - "title": "pg_cron: Schedule Recurring Jobs with Cron Syntax in Postgres" + "url": "https://supabase.com/changelog.md" } - ], - "resultChars": 19469 + ] }, { - "source": "search_docs", - "query": "query { searchDocs(query: \"pg_cron unschedule signature jobname\", limit: 5) { nodes { title href content } } }", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"sed -n '241,520p' supabase/migrations/20260101000000_enable_cron_and_queues.sql\nsupabase --version\nsupabase --help | sed -n '1,200p'\nsupabase functions --help | sed -n '1,200p'\nprintf '\\\\n--- changelog relevant lines ---\\\\n'\ncurl -fsSL https://supabase.com/changelog.md | rg -in -C 2 'breaking-change|queue|cron|edge function' | sed -n '1,240p'\nprintf '\\\\n--- queue docs ---\\\\n'\ncurl -fsSL https://supabase.com/docs/guides/queues/quickstart.md | sed -n '1,260p'\nprintf '\\\\n--- cron docs ---\\\\n'\ncurl -fsSL https://supabase.com/docs/guides/cron/quickstart.md | sed -n '1,240p'\nprintf '\\\\n--- function docs ---\\\\n'\ncurl -fsSL https://supabase.com/docs/guides/functions/quickstart.md | sed -n '1,260p'\"", "pages": [ { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" + "url": "https://supabase.com/changelog.md" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" + "url": "https://supabase.com/docs/guides/queues/quickstart.md" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" + "url": "https://supabase.com/docs/guides/cron/quickstart.md" }, { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" + "url": "https://supabase.com/docs/guides/functions/quickstart.md" } ], - "resultChars": 33082 + "resultChars": 40465 } ] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini/build-cli-003-pg-cron-queue-workflow/run-2/result.json" + "sourcePath": "codex-gpt-5.6/build-cli-003-pg-cron-queue-workflow/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-cli-003-pg-cron-queue-workflow", @@ -24042,12 +24341,12 @@ { "name": "cron command enqueues to the 'tasks' queue", "passed": true, - "notes": "queue depth 1 -> 2" + "notes": "queue depth 0 -> 1" }, { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 4) from the queue" + "notes": "function removed the seeded message (id 36) from the queue" } ], "skills": { @@ -24055,136 +24354,45 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "loaded": [] }, "docs": { "calls": [ { "source": "web_search", - "query": "site:supabase.com/changelog.md supabase changelog markdown breaking-change pg_cron pgmq edge functions", + "query": "site:supabase.com/docs/guides/queues pgmq Edge Function pop delete ...", "pages": [] }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, { "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,120p'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 6693 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"pg_cron schedule a function every minute Supabase\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on CLICommandReference { title href content } ... on ManagementApiReference { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } totalCount } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - } - ], - "resultChars": 58699 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"pgmq pop send queue Supabase\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on CLICommandReference { title href content } ... on ManagementApiReference { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } totalCount } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", - "title": "Expose Queues for local and self-hosted Supabase" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - } - ], - "resultChars": 39520 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"pgmq create queue if exists behavior\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on CLICommandReference { title href content } ... on ManagementApiReference { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } totalCount } }", - "hasContent": true, + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md -o /tmp/supabase-changelog.md && rg -n -i -C 2 'breaking-change|queue|pgmq|cron|edge function' /tmp/supabase-changelog.md | sed -n '1,240p'; curl -fsSL https://supabase.com/docs/guides/queues/quickstart.md | sed -n '1,260p'; curl -fsSL https://supabase.com/docs/guides/cron/quickstart.md | sed -n '1,240p'; curl -fsSL https://supabase.com/docs/guides/functions/quickstart-dashboard.md | sed -n '1,180p'\"", "pages": [ { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/queues/quickstart.md" }, { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" + "url": "https://supabase.com/docs/guides/cron/quickstart.md" }, { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard.md" } ], - "resultChars": 68868 + "resultChars": 22140 } ] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini/build-cli-003-pg-cron-queue-workflow/run-3/result.json" + "sourcePath": "codex-gpt-5.6/build-cli-003-pg-cron-queue-workflow/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-dataapi-001-relational-report", @@ -24233,36 +24441,53 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,160p'\"", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -n 120'", "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 9005 + "resultChars": 6742 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/docs/reference/javascript/select.md | sed -n '1,260p'\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/select.md" + } + ], + "resultChars": 49 + }, + { + "source": "web_search", + "query": "site:supabase.com/docs secret key apikey header REST Data API Supabase", + "pages": [] } ] }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini/build-dataapi-001-relational-report/run-1/result.json" + "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-dataapi-001-relational-report", @@ -24277,7 +24502,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "report runs and prints JSON", @@ -24296,8 +24521,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "passed": true, + "notes": "imports found in: app/report.mjs" }, { "name": "report queries via the Data API, not raw SQL", @@ -24316,20 +24541,48 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,220p'\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 12609 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/docs/guides/api/api-keys.md | sed -n '1,260p'; curl -fsSL https://supabase.com/docs/guides/api/using-custom-schemas.md | sed -n '1,120p'; curl -fsSL https://supabase.com/docs/reference/javascript/select.md | sed -n '1,220p'\"", + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/api-keys.md" + }, + { + "url": "https://supabase.com/docs/guides/api/using-custom-schemas.md" + }, + { + "url": "https://supabase.com/docs/reference/javascript/select.md" + } + ], + "resultChars": 20764 + } + ] }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini/build-dataapi-001-relational-report/run-2/result.json" + "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-dataapi-001-relational-report", @@ -24344,7 +24597,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "report runs and prints JSON", @@ -24363,8 +24616,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "passed": true, + "notes": "imports found in: app/report.mjs" }, { "name": "report queries via the Data API, not raw SQL", @@ -24385,69 +24638,51 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js select order group by nested relation order function aggregate\", limit: 5) { nodes { title href content } } }", - "hasContent": true, + "source": "web_search", + "query": "https://supabase.com/changelog.md", "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/using-modifiers-order" - }, - { - "url": "https://supabase.com/docs/guides/database/joins-and-nesting", - "title": "Querying Joins and Nested tables" - }, + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"sed -n '1,220p' app/package.json; if [ -f app/package-lock.json ]; then sed -n '1,100p' app/package-lock.json; fi; curl -fsSL https://supabase.com/changelog.md | sed -n '1,180p'; curl -fsSL https://supabase.com/docs/reference/javascript/select.md | sed -n '1,180p'\"", + "pages": [ { - "url": "https://supabase.com/docs/guides/api/sql-to-api", - "title": "Converting SQL to JavaScript API" + "url": "https://supabase.com/changelog.md" }, { - "url": "https://supabase.com/docs/guides/api/rest/generating-types", - "title": "Generating TypeScript Types" + "url": "https://supabase.com/docs/reference/javascript/select.md" } ], - "resultChars": 38575 + "resultChars": 49 }, { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js service role Node.js server createClient env SUPABASE_URL SUPABASE_SECRET_KEY\", limit: 5) { nodes { title href content } } }", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"git status --short; git log --oneline --all -5; find . -maxdepth 3 -type f -not -path './.git/*' -print | sort; node --version; curl -fsSL https://supabase.com/docs/reference/javascript/select | sed -n '1,80p'\"", "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", - "title": "Build a User Management App with Refine" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", - "title": "Performing administration tasks on the server side with a secret key" + "url": "https://supabase.com/docs/reference/javascript/select" } ], - "resultChars": 85729 + "resultChars": 29768 } ] }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini/build-dataapi-001-relational-report/run-3/result.json" + "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-dataapi-002-restock-alert-report", @@ -24503,479 +24738,24 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"REST API service_role key PostgREST Authorization apikey local backend JavaScript fetch\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#before-you-begin", - "title": "Before you begin" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#enabling-the-envoy-gateway", - "title": "Enabling the Envoy gateway" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#verify", - "title": "Verify" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#architecture", - "title": "Architecture" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#configuration-file-structure", - "title": "Configuration file structure" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#how-the-configuration-is-rendered-at-startup", - "title": "How the configuration is rendered at startup" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#routes", - "title": "Routes" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#authentication", - "title": "Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#dashboard-basic-auth", - "title": "Dashboard basic auth" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#api-key-enforcement-on-protected-routes", - "title": "API key enforcement on protected routes" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#opaque-key-translation", - "title": "Opaque key translation" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#forwarded-headers-and-cors", - "title": "Forwarded headers and CORS" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#x-forwarded-headers", - "title": "X-Forwarded headers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#cors", - "title": "CORS" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#security-hardening", - "title": "Security hardening" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#customizing-the-configuration", - "title": "Customizing the configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#admin-interface", - "title": "Admin interface" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#logs", - "title": "Logs" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#common-issues", - "title": "Common issues" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#see-also", - "title": "See also" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles", - "title": "Postgres Roles" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles#users-vs-roles", - "title": "Users vs roles" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles#creating-roles", - "title": "Creating roles" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles#creating-users", - "title": "Creating users" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles#passwords", - "title": "Passwords" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles#special-symbols-in-passwords", - "title": "Special symbols in passwords" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles#changing-your-project-password", - "title": "Changing your project password" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles#granting-permissions", - "title": "Granting permissions" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles#revoking-permissions", - "title": "Revoking permissions" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles#role-hierarchy", - "title": "Role hierarchy" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles#role-inheritance", - "title": "Role inheritance" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles#preventing-inheritance", - "title": "Preventing inheritance" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles#supabase-roles", - "title": "Supabase roles" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles#postgres", - "title": "postgres" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles#anon", - "title": "anon" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles#authenticator", - "title": "authenticator" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles#authenticated", - "title": "authenticated" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles#service_role", - "title": "service_role" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles#supabase_auth_admin", - "title": "supabase_auth_admin" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles#supabase_storage_admin", - "title": "supabase_storage_admin" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles#supabase_etl_admin", - "title": "supabase_etl_admin" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles#dashboard_user", - "title": "dashboard_user" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles#supabase_admin", - "title": "supabase_admin" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data#connecting-your-app-securely", - "title": "Connecting your app securely" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data#data-api", - "title": "Data API" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data#edge-functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data#direct-database-connections", - "title": "Direct database connections" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data#frontend-access", - "title": "Frontend access" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data#more-information", - "title": "More information" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#database", - "title": "Database" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#postgres-database", - "title": "Postgres database" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#vector-database", - "title": "Vector database" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#auto-generated-rest-api-via-postgrest", - "title": "Auto-generated REST API via PostgREST" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#auto-generated-graphql-api-via-pg_graphql", - "title": "Auto-generated GraphQL API via pg_graphql" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#database-webhooks", - "title": "Database webhooks" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#secrets-and-encryption", - "title": "Secrets and encryption" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#replication", - "title": "Replication" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#platform", - "title": "Platform" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#database-backups", - "title": "Database backups" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#custom-domains", - "title": "Custom domains" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#network-restrictions", - "title": "Network restrictions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#ssl-enforcement", - "title": "SSL enforcement" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#branching", - "title": "Branching" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#terraform-provider", - "title": "Terraform provider" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#read-replicas", - "title": "Read replicas" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#log-drains", - "title": "Log drains" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#studio", - "title": "Studio" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#studio-single-sign-on", - "title": "Studio Single Sign-On" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#realtime", - "title": "Realtime" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#postgres-changes", - "title": "Postgres changes" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#broadcast", - "title": "Broadcast" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#presence", - "title": "Presence" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#email-login", - "title": "Email login" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#social-login", - "title": "Social login" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#phone-logins", - "title": "Phone logins" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#passwordless-login", - "title": "Passwordless login" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#authorization-via-row-level-security", - "title": "Authorization via Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#captcha-protection", - "title": "CAPTCHA protection" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#server-side-auth", - "title": "Server-Side Auth" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#storage", - "title": "Storage" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#file-storage", - "title": "File storage" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#content-delivery-network", - "title": "Content Delivery Network" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#smart-content-delivery-network", - "title": "Smart Content Delivery Network" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#image-transformations", - "title": "Image transformations" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#resumable-uploads", - "title": "Resumable uploads" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#s3-compatibility", - "title": "S3 compatibility" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#edge-functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#deno-edge-functions", - "title": "Deno Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#regional-invocations", - "title": "Regional invocations" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#npm-compatibility", - "title": "NPM compatibility" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#project-management", - "title": "Project management" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#cli", - "title": "CLI" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#management-api", - "title": "Management API" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#client-libraries", - "title": "Client libraries" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#feature-status", - "title": "Feature status" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#private-alpha", - "title": "Private alpha" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#public-alpha", - "title": "Public alpha" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#beta", - "title": "Beta" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#generally-available", - "title": "Generally available" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth#authenticated-user-calls", - "title": "Authenticated user calls" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth#service-to-service-calls", - "title": "Service-to-service calls" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth#public-functions", - "title": "Public functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth#external-webhooks", - "title": "External webhooks" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth#combining-modes", - "title": "Combining modes" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth#custom-error-responses", - "title": "Custom error responses" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth#environment-variables", - "title": "Environment variables" - } - ], - "resultChars": 140109 + "source": "web_search", + "query": "site:supabase.com/docs reference JavaScript select foreign tables joins service role secret key Authorization apikey", + "pages": [] } ] }, "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini/build-dataapi-002-restock-alert-report/run-1/result.json" + "sourcePath": "codex-gpt-5.6/build-dataapi-002-restock-alert-report/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-dataapi-002-restock-alert-report", @@ -25029,20 +24809,47 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/docs/guides/database/joins-and-nesting.md | sed -n '1,240p'\ncurl -fsSL https://supabase.com/docs/guides/api/api-keys.md | sed -n '1,260p'\"", + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/joins-and-nesting.md" + }, + { + "url": "https://supabase.com/docs/guides/api/api-keys.md" + } + ], + "resultChars": 17000 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/docs/guides/api/api-keys.md | rg -n -C 4 'apikey|Authorization' | head -120\ncurl -fsSL https://supabase.com/docs/reference/javascript/select.md | sed -n '1,220p'\"", + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/api-keys.md" + }, + { + "url": "https://supabase.com/docs/reference/javascript/select.md" + } + ], + "resultChars": 1781 + } + ] }, "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini/build-dataapi-002-restock-alert-report/run-2/result.json" + "sourcePath": "codex-gpt-5.6/build-dataapi-002-restock-alert-report/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-dataapi-002-restock-alert-report", @@ -25098,46 +24905,34 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createClient select service_role key server-side node\", limit: 5) { nodes { title href content ... on ClientLibraryFunctionReference { language methodName } } } }", - "hasContent": true, + "source": "web_search", + "query": "site:supabase.com/docs REST API secret key apikey Authorization header Supabase ...", + "pages": [] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking.change|rest api|api key|postgrest' | head -n 160; supabase --version; supabase status -o env | sed -E 's/=.*/=/'\"", "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/quickstarts/astrojs", - "title": "Use Supabase with Astro" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 60026 + "resultChars": 931 } ] }, "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini/build-dataapi-002-restock-alert-report/run-3/result.json" + "sourcePath": "codex-gpt-5.6/build-dataapi-002-restock-alert-report/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-database-001-migrate-postgres-to-supabase", @@ -25186,47 +24981,43 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"pg_restore local binary dump Supabase CLI restore dump\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on CLICommandReference { title href content } ... on TroubleshootingGuide { title href content } } totalCount } }", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i -C 2 'breaking.change|restore|dump|local development|postgres' | sed -n '1,180p'; curl -fsSL https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres.md | sed -n '1,240p'\"", "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/download-logical-backups", - "title": "How to download logical backups in Supabase with physical backups enabled?" - }, - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" + "url": "https://supabase.com/changelog.md" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" - }, + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres.md" + } + ], + "resultChars": 9740 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres.md | sed -n '240,520p'\"", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/reference/cli/supabase-db-dump", - "title": "Dumps data or schemas from the remote database" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres.md" } ], - "resultChars": 37470 + "resultChars": 7504 } ] }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini/build-database-001-migrate-postgres-to-supabase/run-1/result.json" + "sourcePath": "codex-gpt-5.6/build-database-001-migrate-postgres-to-supabase/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-database-001-migrate-postgres-to-supabase", @@ -25276,57 +25067,31 @@ "calls": [ { "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,120p'\"", - "hasContent": true, + "query": "/bin/bash -lc \"file source.dump\npg_restore --version\npg_restore --list source.dump\nsupabase --version\nsupabase --help\nsupabase init --help\nsupabase start --help\ncurl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|restore|migration|postgres' | head -n 120\ncurl -fsSL https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres.md | sed -n '1,220p'\"", "pages": [ { "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 6693 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI restore database dump local start restore pg_restore\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } ... on ManagementApiReference { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-start", - "title": "Starts local Postgres database" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", - "title": "Migrate from Vercel Postgres to Supabase" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres.md" } ], - "resultChars": 38865 + "resultChars": 23911 } ] }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini/build-database-001-migrate-postgres-to-supabase/run-2/result.json" + "sourcePath": "codex-gpt-5.6/build-database-001-migrate-postgres-to-supabase/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-database-001-migrate-postgres-to-supabase", @@ -25339,12 +25104,27 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { - "name": "scorer completed without errors", - "passed": false, - "notes": "query failed: psql: error: connection to server at \"127.0.0.1\", port 54322 failed: FATAL: password authentication failed for user \"postgres\"\n" + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true } ], "skills": { @@ -25363,15 +25143,15 @@ "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini/build-database-001-migrate-postgres-to-supabase/run-3/result.json" + "sourcePath": "codex-gpt-5.6/build-database-001-migrate-postgres-to-supabase/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-functions-004-service-role-bypass", @@ -25408,12 +25188,12 @@ { "name": "user A cannot force-read user B note", "passed": true, - "notes": "status=403" + "notes": "status=200" }, { "name": "user B cannot force-read user A note", "passed": true, - "notes": "status=403" + "notes": "status=200" } ], "skills": { @@ -25428,9 +25208,29 @@ }, "docs": { "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i -C 2 'breaking-change|edge function|jwt|auth' | head -n 120\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 8298 + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions auth get user JWT verify_jwt private notes RLS security definer\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on ManagementApiReference { title href content } } } }", + "query": "query { searchDocs(query: \"Edge Functions authentication Authorization header createClient user JWT RLS\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -25438,38 +25238,38 @@ "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" }, { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" + "url": "https://supabase.com/docs/guides/auth/auth-mfa", + "title": "Multi-Factor Authentication" } ], - "resultChars": 52639 + "resultChars": 71385 } ] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini/build-functions-004-service-role-bypass/run-1/result.json" + "sourcePath": "codex-gpt-5.6/build-functions-004-service-role-bypass/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-functions-004-service-role-bypass", @@ -25501,7 +25301,7 @@ { "name": "reads only with the caller's JWT", "passed": true, - "notes": "bearer_tokens=1, all_match=true" + "notes": "bearer_tokens=2, all_match=true" }, { "name": "user A cannot force-read user B note", @@ -25526,53 +25326,68 @@ }, "docs": { "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,180p'\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 10168 + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Function verify JWT getUser auth.uid notes RLS\", limit: 5) { nodes { title href content } } }", + "query": "query {\n auth: searchDocs(query: \"Edge Functions authenticate user Authorization header createClient global headers getUser RLS\", limit: 5) {\n nodes { title href content }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" }, { - "url": "https://supabase.com/docs/guides/auth/third-party/firebase-auth", - "title": "Firebase Auth" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" }, { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" } ], - "resultChars": 54213 - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/guides/functions/auth Securing Edge Functions verify_jwt RLS", - "pages": [] + "resultChars": 32605 } ] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini/build-functions-004-service-role-bypass/run-2/result.json" + "sourcePath": "codex-gpt-5.6/build-functions-004-service-role-bypass/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-functions-004-service-role-bypass", @@ -25629,9 +25444,57 @@ }, "docs": { "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i -C 2 'breaking-change|edge function|jwt|auth' | head -160\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 10982 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions authentication Authorization header createClient user JWT RLS service role\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" + } + ], + "resultChars": 63849 + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions verify_jwt auth uid getUser private notes RLS\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"Securing Edge Functions JWT Authorization header getUser createClient RLS\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -25639,38 +25502,50 @@ "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/auth/third-party/firebase-auth", - "title": "Firebase Auth" + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" + }, + { + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/getting-started", + "title": "Getting Started with OAuth 2.1 Server" + }, + { + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" } ], - "resultChars": 65046 + "resultChars": 104555 } ] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini/build-functions-004-service-role-bypass/run-3/result.json" + "sourcePath": "codex-gpt-5.6/build-functions-004-service-role-bypass/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-functions-005-dual-auth-user-secret", @@ -25688,7 +25563,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "seed rows present", @@ -25698,42 +25573,42 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 501: Error response

Error response

Error code: 501

Me" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"MISSING_CREDENTIALS\",\"message\":\"[@supabase/server] No credentials found on the request. This endpoint accepts auth mode(s):" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\": \"75461db0-3c52-4c50-ae22-e9ae9c1cb85d\", \"metric\": \"steps_a_mta4e8hg\", \"value\": 111}]" + "notes": "status 200: [{\"user_id\":\"da88dff8-cc7c-48fd-a14d-04da3c087356\",\"metric\":\"steps_a_mtsfve92\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", - "passed": false, - "notes": "status 501: Error response

Error response

Error code: 501

Me" + "passed": true, + "notes": "status 200: [{\"user_id\":\"da88dff8-cc7c-48fd-a14d-04da3c087356\",\"metric\":\"steps_a_mtsfve92\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: [{\"user_id\": \"54c39b95-48e5-4ff9-85a6-b57173cf2637\", \"metric\": \"steps_b_mta4e8hg\", \"value\": 222}]" + "notes": "status 200: [{\"user_id\":\"46dad85b-5635-474d-9334-cf92822ebcf0\",\"metric\":\"steps_b_mtsfve92\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"error\": \"Unauthorized.\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 501: Error response

Error response

Error code: 501

Me" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_JWT\",\"message\":\"[@supabase/server] The JWT in the Authorization header failed verification: its header is missing \\" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"error\": \"Unauthorized.\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" }, { "name": "implementation uses @supabase/server", - "passed": false, - "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" + "passed": true, + "notes": "imports @supabase/server / withSupabase" } ], "skills": { @@ -25749,139 +25624,34 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"Edge Functions verify user access token service role api key request auth header user stats\", limit: 5) {\n nodes {\n ... on Guide {\n title\n href\n content\n }\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n ... on TroubleshootingGuide {\n title\n href\n content\n }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports", - "title": "Reports" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - } - ], - "resultChars": 74606 + "source": "web_search", + "query": "site:supabase.com/docs/guides/functions function configuration verify_jwt config.toml Edge Functions ...", + "pages": [] }, { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"@supabase/server withSupabase auth none apikey user service role env vars\", limit: 10) {\n nodes {\n ... on Guide {\n title\n href\n content\n }\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" - }, - { - "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", - "title": "Build a Supabase Integration" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-environment", - "title": "Development Environment" - } - ], - "resultChars": 157426 + "source": "web_search", + "query": "site:supabase.com/docs/guides/functions/auth \"auth: ['user'\" secret multiple modes withSupabase ...", + "pages": [] }, { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"config.toml functions entrypoint import_map enabled verify_jwt function block\", limit: 10) {\n nodes {\n ... on Guide { title href content }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - }, - { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/dependencies", - "title": "Managing dependencies" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/cloudflare-turnstile", - "title": "CAPTCHA support with Cloudflare Turnstile" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/github-actions", - "title": "GitHub Actions" - } - ], - "resultChars": 58649 + "source": "web_search", + "query": "\"failed to determine entrypoint\" \"supabase functions serve\" ...", + "pages": [] } ] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini/build-functions-005-dual-auth-user-secret/run-1/result.json" + "sourcePath": "codex-gpt-5.6/build-functions-005-dual-auth-user-secret/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-functions-005-dual-auth-user-secret", @@ -25899,7 +25669,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "seed rows present", @@ -25914,17 +25684,17 @@ { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"b02930af-2ad5-4a98-9165-326cc453a88c\",\"metric\":\"steps_a_mta4f1i6\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"8eacdfd8-f18e-4724-bdaa-ffe0dd0def9f\",\"metric\":\"steps_a_mtsfwbvj\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"b02930af-2ad5-4a98-9165-326cc453a88c\",\"metric\":\"steps_a_mta4f1i6\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"8eacdfd8-f18e-4724-bdaa-ffe0dd0def9f\",\"metric\":\"steps_a_mtsfwbvj\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"5682e0e7-ecb2-44d7-b336-2c1d5c1767c9\",\"metric\":\"steps_b_mta4f1i6\",\"value\":222}]" + "passed": false, + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { "name": "non-service key is not granted service access", @@ -25960,683 +25730,35 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"Deno Edge Functions getSession user access token verify JWT service_role key\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers#understanding-authorization-headers", - "title": "Understanding authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers#the-verify_jwt-platform-check", - "title": "The verify_jwt platform check" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#database", - "title": "Database" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#postgres-database", - "title": "Postgres database" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#vector-database", - "title": "Vector database" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#auto-generated-rest-api-via-postgrest", - "title": "Auto-generated REST API via PostgREST" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#auto-generated-graphql-api-via-pg_graphql", - "title": "Auto-generated GraphQL API via pg_graphql" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#database-webhooks", - "title": "Database webhooks" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#secrets-and-encryption", - "title": "Secrets and encryption" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#replication", - "title": "Replication" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#platform", - "title": "Platform" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#database-backups", - "title": "Database backups" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#custom-domains", - "title": "Custom domains" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#network-restrictions", - "title": "Network restrictions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#ssl-enforcement", - "title": "SSL enforcement" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#branching", - "title": "Branching" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#terraform-provider", - "title": "Terraform provider" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#read-replicas", - "title": "Read replicas" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#log-drains", - "title": "Log drains" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#studio", - "title": "Studio" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#studio-single-sign-on", - "title": "Studio Single Sign-On" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#realtime", - "title": "Realtime" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#postgres-changes", - "title": "Postgres changes" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#broadcast", - "title": "Broadcast" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#presence", - "title": "Presence" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#email-login", - "title": "Email login" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#social-login", - "title": "Social login" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#phone-logins", - "title": "Phone logins" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#passwordless-login", - "title": "Passwordless login" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#authorization-via-row-level-security", - "title": "Authorization via Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#captcha-protection", - "title": "CAPTCHA protection" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#server-side-auth", - "title": "Server-Side Auth" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#storage", - "title": "Storage" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#file-storage", - "title": "File storage" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#content-delivery-network", - "title": "Content Delivery Network" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#smart-content-delivery-network", - "title": "Smart Content Delivery Network" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#image-transformations", - "title": "Image transformations" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#resumable-uploads", - "title": "Resumable uploads" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#s3-compatibility", - "title": "S3 compatibility" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#edge-functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#deno-edge-functions", - "title": "Deno Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#regional-invocations", - "title": "Regional invocations" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#npm-compatibility", - "title": "NPM compatibility" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#project-management", - "title": "Project management" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#cli", - "title": "CLI" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#management-api", - "title": "Management API" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#client-libraries", - "title": "Client libraries" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#feature-status", - "title": "Feature status" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#private-alpha", - "title": "Private alpha" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#public-alpha", - "title": "Public alpha" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#beta", - "title": "Beta" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features#generally-available", - "title": "Generally available" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#architecture", - "title": "Architecture" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#database-schema", - "title": "Database schema" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#edge-function-websocket-proxy", - "title": "Edge Function (WebSocket proxy)" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#browser-client", - "title": "Browser client" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#why-this-pattern-works", - "title": "Why this pattern works" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#next-steps", - "title": "Next steps" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets#creating-websocket-servers", - "title": "Creating WebSocket servers" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets#outbound-websockets", - "title": "Outbound WebSockets" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets#authentication", - "title": "Authentication" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets#testing-websockets-locally", - "title": "Testing WebSockets locally" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields#jwt-structure-overview", - "title": "JWT structure overview" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields#required-claims", - "title": "Required claims" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields#optional-claims", - "title": "Optional claims" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields#special-claims", - "title": "Special claims" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields#field-value-constraints", - "title": "Field value constraints" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields#authenticator-assurance-level-aal", - "title": "Authenticator assurance level (aal)" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields#role-values-role", - "title": "Role values (role)" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields#audience-values-aud", - "title": "Audience values (aud)" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields#authentication-methods-amrmethod", - "title": "Authentication methods (amr.method)" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields#jwt-examples", - "title": "JWT examples" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields#authenticated-user-token", - "title": "Authenticated user token" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields#anonymous-user-token", - "title": "Anonymous user token" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields#service-role-token", - "title": "Service role token" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields#language-specific-considerations", - "title": "Language-Specific considerations" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields#rust", - "title": "Rust" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields#typescriptjavascript", - "title": "TypeScript/JavaScript" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields#python", - "title": "Python" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields#go", - "title": "Go" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields#validation-guidelines", - "title": "Validation guidelines" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields#security-considerations", - "title": "Security considerations" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields#related-documentation", - "title": "Related documentation" - } - ], - "resultChars": 102561 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions auth verify_jwt anon service_role user token apikey header\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#how-to-create-mint-jwts-if-access-to-the-private-key-or-shared-secret-is-not-possible", - "title": "How to create (mint) JWTs if access to the private key or shared secret is not possible?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#why-is-a-5-minute-wait-imposed-when-changing-signing-key-states", - "title": "Why is a 5 minute wait imposed when changing signing key states?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#why-is-deleting-the-legacy-jwt-secret-disallowed", - "title": "Why is deleting the legacy JWT secret disallowed?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#why-does-revoking-the-legacy-jwt-secret-require-disabling-of-anon-and-service_role-api-keys", - "title": "Why does revoking the legacy JWT secret require disabling of anon and service_role API keys?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#using-jwt-based-anon-key-in-a-mobile-desktop-or-cli-application-and-need-to-rotate-a-service_role-jwt-secret", - "title": "Using JWT-based anon key in a mobile, desktop, or CLI application and need to rotate a service_role JWT secret?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#overview", - "title": "Overview" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#benefits-of-the-signing-keys-system", - "title": "Benefits of the signing keys system" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#getting-started", - "title": "Getting started" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#rotating-and-revoking-keys", - "title": "Rotating and revoking keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#lifetime-of-a-signing-key", - "title": "Lifetime of a signing key" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#public-key-discovery-and-caching", - "title": "Public key discovery and caching" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#choosing-the-right-signing-algorithm", - "title": "Choosing the right signing algorithm" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#frequently-asked-questions", - "title": "Frequently asked questions" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#why-is-it-not-possible-to-extract-the-private-key-or-shared-secret-from-supabase", - "title": "Why is it not possible to extract the private key or shared secret from Supabase?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#why-are-anon-and-service_role-jwt-based-keys-no-longer-recommended", - "title": "Why are anon and service_role JWT-based keys no longer recommended?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#can-you-still-use-an-old-anon-and-service-role-api-keys-after-enabling-the-publishable-and-secret-keys", - "title": "Can you still use an old anon and service-role API keys after enabling the publishable and secret keys?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#how-to-deactivate-the-anon-and-service_role-jwt-based-api-keys-after-moving-to-publishable-and-secret-keys", - "title": "How to deactivate the anon and service_role JWT-based API keys after moving to publishable and secret keys?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#how-are-publishable-and-secret-keys-implemented-on-the-hosted-platform", - "title": "How are publishable and secret keys implemented on the hosted platform?" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#setting-up-auth-context", - "title": "Setting up auth context" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#fetching-the-user", - "title": "Fetching the user" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#example", - "title": "Example" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#before-you-start", - "title": "Before you start" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-1-create-the-new-api-keys", - "title": "Step 1: Create the new API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-2-swap-the-publishable-key-in-client-code", - "title": "Step 2: Swap the publishable key in client code" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-3-swap-the-secret-key-in-backend-code", - "title": "Step 3: Swap the secret key in backend code" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#database-webhooks-and-pg_net", - "title": "Database Webhooks and pg_net" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-4-update-edge-functions", - "title": "Step 4: Update Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-1-read-the-new-keys-from-the-environment", - "title": "Option 1: Read the new keys from the environment" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-2-adopt-the-supabaseserver-sdk", - "title": "Option 2: Adopt the @supabase/server SDK" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-5-verify-nothing-uses-the-legacy-keys", - "title": "Step 5: Verify nothing uses the legacy keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-6-deactivate-the-legacy-keys", - "title": "Step 6: Deactivate the legacy keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#known-limitations", - "title": "Known limitations" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#next-steps", - "title": "Next steps" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers#understanding-authorization-headers", - "title": "Understanding authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers#the-verify_jwt-platform-check", - "title": "The verify_jwt platform check" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#overview", - "title": "Overview" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#publishable-keys", - "title": "Publishable keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#interaction-with-supabase-auth", - "title": "Interaction with Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#security-considerations", - "title": "Security considerations" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#what-secret-keys-allow-access-to", - "title": "What secret keys allow access to" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#best-practices-for-handling-secret-keys", - "title": "Best practices for handling secret keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#what-to-do-if-a-secret-key-or-service_role-has-been-leaked-or-compromised", - "title": "What to do if a secret key or service_role has been leaked or compromised?" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#known-limitations-and-compatibility-differences", - "title": "Known limitations and compatibility differences" - } - ], - "resultChars": 145199 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions supabase-js createClient Deno import jsr supabase-js environment variables\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/dependencies", - "title": "Managing dependencies" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - } - ], - "resultChars": 42217 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions entrypoint index.ts failed to determine entrypoint local serve function directory config.toml\", limit: 10) { nodes { title href content } } }", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"supabase --version; supabase functions --help | sed -n '1,220p'; printf '\\\\nCHANGELOG MATCHES\\\\n'; curl -fsSL https://supabase.com/changelog.md | rg -n -i 'edge function|jwt|breaking-change' | head -100; printf '\\\\nFUNCTION AUTH DOCS\\\\n'; curl -fsSL https://supabase.com/docs/guides/functions/auth.md | sed -n '1,260p'; printf '\\\\nENV DOCS\\\\n'; curl -fsSL https://supabase.com/docs/guides/functions/secrets.md | sed -n '1,220p'; printf '\\\\nSERVE HELP\\\\n'; supabase functions serve --help | sed -n '1,220p'\"", "pages": [ { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart", - "title": "Getting Started with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/issues-serving-edge-functions-locally", - "title": "Issues serving Edge Functions locally" - }, - { - "url": "https://supabase.com/docs/guides/local-development/managing-config", - "title": "Managing config and secrets" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/unable-to-deploy-edge-function", - "title": "Unable to deploy Edge Function" + "url": "https://supabase.com/changelog.md" }, { - "url": "https://supabase.com/docs/guides/functions/unit-test", - "title": "Testing your Edge Functions" + "url": "https://supabase.com/docs/guides/functions/auth.md" }, { - "url": "https://supabase.com/docs/guides/deployment", - "title": "Deployment & Branching" + "url": "https://supabase.com/docs/guides/functions/secrets.md" } ], - "resultChars": 48273 - }, - { - "source": "web_search", - "query": "\"/home/node/.supabase/profile\" supabase functions serve profile file", - "pages": [] - }, - { - "source": "web_search", - "query": "site:github.com/supabase/cli \"failed to determine entrypoint\" \"supabase functions serve\"", - "pages": [] - }, - { - "source": "web_search", - "query": "site:github.com/supabase/cli \"supabase/.temp/profile\"", - "pages": [] - }, - { - "source": "web_search", - "query": "Supabase CLI v2.115.0 linux amd64 release asset name github", - "pages": [] + "resultChars": 36508 } ] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini/build-functions-005-dual-auth-user-secret/run-2/result.json" + "sourcePath": "codex-gpt-5.6/build-functions-005-dual-auth-user-secret/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-functions-005-dual-auth-user-secret", @@ -26664,37 +25786,37 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"MISSING_CREDENTIALS\",\"message\":\"[@supabase/server] No credentials found on the request. This endpoint accepts auth mode(s):" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"0d03d215-61c1-4b57-bd96-686ee1aea40e\",\"metric\":\"steps_a_mta4ee09\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"83533f0f-675f-48b6-99a4-3e7880a1dec4\",\"metric\":\"steps_a_mtsfydik\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"0d03d215-61c1-4b57-bd96-686ee1aea40e\",\"metric\":\"steps_a_mta4ee09\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"83533f0f-675f-48b6-99a4-3e7880a1dec4\",\"metric\":\"steps_a_mtsfydik\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"4e3a973a-e9f1-4f0e-9dcc-d130908e8777\",\"metric\":\"steps_b_mta4ee09\",\"value\":222}]" + "notes": "status 200: [{\"user_id\":\"d565816b-fc74-4bbd-a9b3-ae18d38f25f2\",\"metric\":\"steps_b_mtsfydik\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_JWT\",\"message\":\"[@supabase/server] The JWT in the Authorization header failed verification: its header is missing \\" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" }, { "name": "implementation uses @supabase/server", @@ -26715,86 +25837,29 @@ "docs": { "calls": [ { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,220p'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 12609 + "source": "web_search", + "query": "site:supabase.com/docs/guides/functions auth JWT verify_jwt Edge Functions apikey service role ...", + "pages": [] }, { - "source": "search_docs", - "query": "{ searchDocs(query: \"Supabase Edge Function verify JWT auth getUser service role apikey header Deno.env.get authorization header\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - } - ], - "resultChars": 43135 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Supabase Edge Function config.toml verify_jwt per function config\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" - } - ], - "resultChars": 33663 + "source": "web_search", + "query": "site:supabase.com/docs \"auth: ['user', 'secret']\" withSupabase ...", + "pages": [] } ] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini/build-functions-005-dual-auth-user-secret/run-3/result.json" + "sourcePath": "codex-gpt-5.6/build-functions-005-dual-auth-user-secret/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-rls-003-org-roles-permissions", @@ -26878,54 +25943,73 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security policies auth.uid memberships documents\", limit: 5) { nodes { title href content } } }", + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,240p'\"", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 13905 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Row Level Security policies SELECT INSERT UPDATE DELETE auth.uid organization multi tenant\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" }, { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", "title": "Column Level Security" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" } ], - "resultChars": 45661 + "resultChars": 79622 } ] }, "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini/build-rls-003-org-roles-permissions/run-1/result.json" + "sourcePath": "codex-gpt-5.6/build-rls-003-org-roles-permissions/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-rls-003-org-roles-permissions", @@ -27017,7 +26101,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"row level security policies auth.uid select update with check team members organization membership\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ClientLibraryFunctionReference { title href href content language methodName } } } }", + "query": "query { searchDocs(query: \"Row Level Security policies auth.uid organization multi tenant UPDATE USING WITH CHECK\", limit: 4) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -27025,38 +26109,54 @@ "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" }, { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" + "url": "https://supabase.com/docs/guides/troubleshooting/rls-simplified-BJTcS8", + "title": "RLS Simplified" }, { "url": "https://supabase.com/docs/guides/getting-started/features", "title": "Features" - }, + } + ], + "resultChars": 68493 + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|row level|rls|policy' | head -80\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 73782 + "resultChars": 8361 } ] }, "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini/build-rls-003-org-roles-permissions/run-2/result.json" + "sourcePath": "codex-gpt-5.6/build-rls-003-org-roles-permissions/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-rls-003-org-roles-permissions", @@ -27148,46 +26248,66 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"row level security org membership access control policies\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Postgres row level security policies multi tenant organization auth.uid update using with check\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { "url": "https://supabase.com/docs/guides/getting-started/features", "title": "Features" }, { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" + "url": "https://supabase.com/docs/guides/troubleshooting/rls-simplified-BJTcS8", + "title": "RLS Simplified" }, { "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", "title": "Column Level Security" - }, + } + ], + "resultChars": 74495 + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|row.level|rls|postgres' | sed -n '1,120p'\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 74443 + "resultChars": 14130 } ] }, "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini/build-rls-003-org-roles-permissions/run-3/result.json" + "sourcePath": "codex-gpt-5.6/build-rls-003-org-roles-permissions/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-storage-001-private-bucket-access", @@ -27219,7 +26339,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 01a03e35-465c-774b-a40d-97f52145d486/receipt-alpha.pdf, 01a03e35-465c-774b-a40d-97f52145d486/receipt-beta.pdf" + "notes": "saw: 01a0803d-80f2-73ce-9dda-dfc0c1f49086/receipt-alpha.pdf, 01a0803d-80f2-73ce-9dda-dfc0c1f49086/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -27240,7 +26360,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Private user-files bucket, authenticated owner-scoped SELECT/INSERT policies with RLS enabled, and short-lived createSignedUrl code are all provided." + "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT policies, intact RLS, and expiring createSignedUrl sharing are all provided." } ], "skills": { @@ -27256,40 +26376,15 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"storage bucket policies signed upload download signed URLs createBucket upload file path owner user id\", limit: 5) {\n nodes {\n __typename\n ... on Guide {\n title\n href\n content\n }\n ... on ClientLibraryFunctionReference {\n title\n href\n language\n methodName\n }\n ... on TroubleshootingGuide {\n title\n href\n content\n }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", - "title": "Why can't I upload/list/etc my public bucket?" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" - }, - { - "url": "https://supabase.com/docs/reference/dart/file-buckets-createsigneduploadurl" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" - } - ], - "resultChars": 5298 + "source": "web_search", + "query": "site:supabase.com/docs/guides/storage security access control storage.foldername auth.uid bucket_id ...", + "pages": [] }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"storage access control object ownership RLS policy storage.objects path like auth.uid create policy bucket private signed URL\", limit: 10) {\n nodes {\n __typename\n ... on Guide {\n title\n href\n content\n }\n ... on ClientLibraryFunctionReference {\n title\n href\n language\n methodName\n }\n }\n }\n}", + "query": "query { searchDocs(query: \"Storage RLS foldername auth.uid private bucket createSignedUrl JavaScript\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, { "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", "title": "Storage Buckets" @@ -27298,73 +26393,6 @@ "url": "https://supabase.com/docs/guides/security/product-security", "title": "Secure configuration of Supabase products" }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/design", - "title": "The Storage Schema" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", - "title": "Custom Roles" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - } - ], - "resultChars": 45181 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"javascript storage from createSignedUrl download signed url reference\", limit: 5) {\n nodes {\n __typename\n ... on ClientLibraryFunctionReference {\n title\n href\n language\n methodName\n }\n ... on Guide {\n title\n href\n content\n }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", - "title": "Storage Image Transformations" - }, - { - "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" - } - ], - "resultChars": 20411 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"storage create bucket policy select insert update delete owner_id foldername signed url createSignedUrl javascript\", limit: 8) {\n nodes {\n __typename\n ... on Guide {\n title\n href\n content\n }\n ... on ClientLibraryFunctionReference {\n title\n href\n language\n methodName\n }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createbucket" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, { "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" }, @@ -27372,40 +26400,26 @@ "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-updatebucket" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-from" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" } ], - "resultChars": 5537 - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/reference/javascript file-buckets-createsignedurl supabase-js createSignedUrl expiresIn", - "pages": [] + "resultChars": 15699 } ] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini/build-storage-001-private-bucket-access/run-1/result.json" + "sourcePath": "codex-gpt-5.6/build-storage-001-private-bucket-access/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-storage-001-private-bucket-access", @@ -27437,7 +26451,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 01a03e35-08a2-75d2-ab47-3328a2fcb89f/receipt-alpha.pdf, 01a03e35-08a2-75d2-ab47-3328a2fcb89f/receipt-beta.pdf" + "notes": "saw: 01a0803d-08ec-7295-910b-476e18af7539/receipt-alpha.pdf, 01a0803d-08ec-7295-910b-476e18af7539/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -27458,7 +26472,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Private bucket, RLS-enabled owner-scoped SELECT/INSERT policies, and expiring createSignedUrl code are provided. Service role usage is explicitly server-side." + "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT policies, RLS retained, and a short-lived signed-URL helper were provided." } ], "skills": { @@ -27474,151 +26488,88 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase storage bucket public private policies folders auth.uid upload download RLS\", limit: 5) { nodes { __typename title href content } } }", - "hasContent": true, + "source": "web_search", + "query": "https://supabase.com/changelog.md", "pages": [ { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/changelog.md" } - ], - "resultChars": 28721 + ] }, { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Storage policies create bucket signed URLs download temporary link user owner path auth.uid()\", limit: 5) { nodes { __typename title href content } } }", + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i -C 2 'breaking|storage|signed url' | sed -n '1,240p'\"", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, - { - "url": "https://supabase.com/docs/guides/storage", - "title": "Storage" - }, - { - "url": "https://supabase.com/docs/guides/platform/delete-project", - "title": "Deleting Your Project" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - }, - { - "url": "https://supabase.com/docs/guides/storage/s3/compatibility", - "title": "S3 Compatibility" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 55052 + "resultChars": 16877 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createSignedUrl temporary link storage.from bucket docs\", limit: 3) { nodes { __typename title href content } } }", + "query": "query {\n storage: searchDocs(query: \"Storage access control foldername RLS policy private bucket signed URL createSignedUrl\", limit: 5) {\n nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", - "title": "Manage Storage Image Transformations usage" + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" }, - { - "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads", - "title": "Resumable Uploads" - } - ], - "resultChars": 27016 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"storage.objects policy allow_any_operation object.get_authenticated_info object.get_authenticated\", limit: 5) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ { "url": "https://supabase.com/docs/guides/storage/security/access-control", "title": "Storage Access Control" }, - { - "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", - "title": "Copy Objects" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", "title": "Configure S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" } ], - "resultChars": 24850 + "resultChars": 19578 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"storage.allow_only_operation object.get_authenticated object.list storage.objects select policy\", limit: 5) { nodes { __typename title href content } } }", + "query": "query {\n signed: searchDocs(query: \"JavaScript createSignedUrl(path expiresIn) private bucket RLS select policy\", limit: 5) {\n nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" }, { - "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", - "title": "Copy Objects" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" }, { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" }, { - "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", - "title": "Custom Roles" + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" } ], - "resultChars": 25293 + "resultChars": 8800 } ] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini/build-storage-001-private-bucket-access/run-2/result.json" + "sourcePath": "codex-gpt-5.6/build-storage-001-private-bucket-access/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-storage-001-private-bucket-access", @@ -27650,7 +26601,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 01a03e34-dccb-758f-8451-49e49bdfa05b/receipt-alpha.pdf, 01a03e34-dccb-758f-8451-49e49bdfa05b/receipt-beta.pdf" + "notes": "saw: 01a0803d-62c9-71f9-85f4-e22de541e44c/receipt-alpha.pdf, 01a0803d-62c9-71f9-85f4-e22de541e44c/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -27671,7 +26622,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Private bucket, RLS enabled, authenticated owner-scoped SELECT/INSERT policies, and expiring createSignedUrl code are all provided." + "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT policies, RLS retained, and short-lived signed URL helper are all present." } ], "skills": { @@ -27687,140 +26638,61 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Storage policies create bucket RLS storage.objects user_id folder path authenticated\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"sed -n '1,260p' skills/supabase-postgres-best-practices/references/security-rls-basics.md && sed -n '1,300p' skills/supabase-postgres-best-practices/references/security-rls-performance.md && sed -n '1,260p' skills/supabase-postgres-best-practices/references/security-privileges.md && curl -fsSL https://supabase.com/changelog.md | rg -n -i -m 20 'breaking-change|storage|signed url|rls'\"", "pages": [ { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", - "title": "Custom Roles" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/supabase-storage-inefficient-folder-operations-and-hierarchical-rls-challenges-b05a4d", - "title": "Supabase Storage: Inefficient folder operations and hierarchical RLS challenges" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 28736 + "resultChars": 9408 }, { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase storage create bucket sql storage.buckets bucket_id public false file_size_limit\", limit: 10) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/creating-buckets", - "title": "Creating Buckets" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/amazon-bedrock-image-generator", - "title": "Generate Images with Amazon Bedrock" - }, - { - "url": "https://supabase.com/docs/guides/storage/quickstart", - "title": "Storage Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/guides/storage", - "title": "Storage" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/copy-from-platform-s3", - "title": "Copy Storage Objects from Platform" - }, - { - "url": "https://supabase.com/docs/guides/storage/vector/limits", - "title": "Vector Bucket Limits" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-size", - "title": "Manage Storage size usage" - }, - { - "url": "https://supabase.com/docs/guides/storage/analytics/creating-analytics-buckets", - "title": "Creating Analytics Buckets" - } - ], - "resultChars": 61661 + "source": "web_search", + "query": "site:supabase.com/docs/guides/storage security access-control storage.objects foldername auth.uid bucket_id ...", + "pages": [] }, { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase storage create bucket JavaScript createSignedUrl temporary link expires\", limit: 10) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "query": "query { searchDocs(query: \"Storage private bucket RLS foldername owner_id createSignedUrl expiresIn\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/amazon-bedrock-image-generator", - "title": "Generate Images with Amazon Bedrock" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/creating-buckets", - "title": "Creating Buckets" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", - "title": "Manage Storage Image Transformations usage" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", - "title": "Storage Image Transformations" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createbucket" + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" }, { "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" }, { - "url": "https://supabase.com/docs/guides/storage", - "title": "Storage" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" }, { - "url": "https://supabase.com/docs/guides/storage/analytics/creating-analytics-buckets", - "title": "Creating Analytics Buckets" + "url": "https://supabase.com/docs/guides/storage/cdn/smart-cdn", + "title": "Smart CDN" } ], - "resultChars": 49872 + "resultChars": 13777 } ] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini/build-storage-001-private-bucket-access/run-3/result.json" + "sourcePath": "codex-gpt-5.6/build-storage-001-private-bucket-access/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-tests-001-rls-tenant-isolation", @@ -27839,17 +26711,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" + "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "6 passed, 0 failed" + "notes": "22 passed, 0 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies `posts` as having the tenant-isolation flaw, explains that its policy allowed access based on any membership rather than row organization membership, and validates the fix with passing pgTAP results. It does not blame `notes`." + "judgeNotes": "Correctly identifies `posts` as cross-tenant readable because its policy fails to match `org_id`, then validates the hardened policy with passing pgTAP tests." } ], "skills": { @@ -27865,323 +26737,39 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"RLS testing pgtap Supabase\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"find .claude/skills/supabase-postgres-best-practices/references -maxdepth 1 -type f -printf '%f\\\\n' | sort | rg 'security|rls|index' && curl -fsSL https://supabase.com/changelog.md | sed -n '1,180p'\"", "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview#testing-approaches", - "title": "Testing approaches" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview#database-unit-testing-with-pgtap", - "title": "Database unit testing with pgTAP" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview#application-level-testing", - "title": "Application-Level testing" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview#test-isolation-strategies", - "title": "Test isolation strategies" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview#continuous-integration-testing", - "title": "Continuous integration testing" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview#best-practices", - "title": "Best practices" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview#real-world-examples", - "title": "Real-World examples" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview#troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview#additional-resources", - "title": "Additional resources" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#row-level-security-in-supabase", - "title": "Row Level Security in Supabase" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#policies", - "title": "Policies" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#enabling-row-level-security", - "title": "Enabling Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#auto-enable-rls-for-new-tables", - "title": "Auto-enable RLS for new tables" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#authenticated-and-unauthenticated-roles", - "title": "Authenticated and unauthenticated roles" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#creating-policies", - "title": "Creating policies" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#select-policies", - "title": "SELECT policies" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#insert-policies", - "title": "INSERT policies" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#update-policies", - "title": "UPDATE policies" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#delete-policies", - "title": "DELETE policies" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#views", - "title": "Views" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#helper-functions", - "title": "Helper functions" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#authuid", - "title": "auth.uid()" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#authjwt", - "title": "auth.jwt()" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#mfa", - "title": "MFA" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#bypassing-row-level-security", - "title": "Bypassing Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#rls-performance-recommendations", - "title": "RLS performance recommendations" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#add-indexes", - "title": "Add indexes" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#call-functions-with-select", - "title": "Call functions with select" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-1", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#add-filters-to-every-query", - "title": "Add filters to every query" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-2", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#use-security-definer-functions", - "title": "Use security definer functions" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#minimize-joins", - "title": "Minimize joins" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-3", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#specify-roles-in-your-policies", - "title": "Specify roles in your policies" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-4", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#more-resources", - "title": "More resources" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#using-databasedev", - "title": "Using database.dev" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#setting-up-dbdev", - "title": "Setting up dbdev" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#installing-test-helpers", - "title": "Installing test helpers" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#test-helper-benefits", - "title": "Test helper benefits" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#schema-wide-row-level-security-testing", - "title": "Schema-wide Row Level Security testing" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#test-file-organization", - "title": "Test file organization" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#creating-a-pre-test-hook", - "title": "Creating a pre-test hook" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#benefits", - "title": "Benefits" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#example-advanced-rls-testing", - "title": "Example: Advanced RLS testing" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#not-another-todo-app-testing-complex-organizations", - "title": "Not another todo app: Testing complex organizations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#system-overview", - "title": "System overview" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#what-makes-this-complex", - "title": "What makes this complex?" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#testing-focus-areas", - "title": "Testing focus areas" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#1-app-schema-definitions", - "title": "1. App schema definitions" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#2-grant-role-privileges", - "title": "2. Grant role privileges" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#3-rls-policies-declaration", - "title": "3. RLS policies declaration" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#4-test-cases", - "title": "4. Test cases:" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#additional-resources", - "title": "Additional resources" - }, - { - "url": "https://supabase.com/docs/guides/database/testing", - "title": "Testing Your Database" - }, - { - "url": "https://supabase.com/docs/guides/database/testing#testing-using-the-supabase-cli", - "title": "Testing using the Supabase CLI" - }, - { - "url": "https://supabase.com/docs/guides/database/testing#creating-a-test", - "title": "Creating a test" - }, - { - "url": "https://supabase.com/docs/guides/database/testing#writing-tests", - "title": "Writing tests" - }, - { - "url": "https://supabase.com/docs/guides/database/testing#running-tests", - "title": "Running tests" - }, - { - "url": "https://supabase.com/docs/guides/database/testing#more-resources", - "title": "More resources" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting#testing-your-database", - "title": "Testing your database" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting#test-helpers", - "title": "Test helpers" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting#running-database-tests-in-ci", - "title": "Running database tests in CI" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting#testing-your-edge-functions", - "title": "Testing your Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting#testing-auth-emails", - "title": "Testing Auth emails" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting#accessing-mailpit", - "title": "Accessing Mailpit" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting#going-into-production", - "title": "Going into production" - }, + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 10423 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"sed -n '1,260p' .claude/skills/supabase-postgres-best-practices/references/security-rls-basics.md && sed -n '1,260p' .claude/skills/supabase-postgres-best-practices/references/security-rls-performance.md && sed -n '1,240p' .claude/skills/supabase-postgres-best-practices/references/security-privileges.md && curl -fsSL https://supabase.com/docs/guides/database/postgres/row-level-security.md | sed -n '1,240p'\"", + "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting#linting-your-database", - "title": "Linting your database" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security.md" } ], - "resultChars": 203142 + "resultChars": 15278 } ] }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini/build-tests-001-rls-tenant-isolation/run-1/result.json" + "sourcePath": "codex-gpt-5.6/build-tests-001-rls-tenant-isolation/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-tests-001-rls-tenant-isolation", @@ -28205,12 +26793,12 @@ { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "6 passed, 0 failed" + "notes": "6 passed, 7 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies `posts` as the broken tenant-isolation policy, explains the cross-org membership flaw, states `notes` was correct, and validates the fix with passing pgTAP results." + "judgeNotes": "Correctly identifies the broken `posts` RLS policy and cross-tenant read exposure, grounded in failing pgTAP assertions. It also correctly distinguishes `notes` as isolated absent membership escalation." } ], "skills": { @@ -28226,47 +26814,54 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase database tests pgTAP local test migrations\", limit: 5) { nodes { title href content } } }", - "hasContent": true, + "source": "web_search", + "query": "https://supabase.com/changelog.md", "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" - }, + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,220p'; curl -fsSL https://supabase.com/docs/guides/local-development/testing/pgtap.md | sed -n '1,260p'; curl -fsSL https://supabase.com/docs/guides/database/postgres/row-level-security.md | sed -n '1,300p'\"", + "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" + "url": "https://supabase.com/changelog.md" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-test-db", - "title": "Tests local database with pgTAP" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap.md" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgtap", - "title": "pgTAP: Unit Testing" - }, + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security.md" + } + ], + "resultChars": 12375 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/docs/guides/database/postgres/row-level-security.md | sed -n '/#### Policy tests/,/### Expose a view safely/p'; supabase --version; supabase test --help; supabase test db --help\"", + "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security.md" } ], - "resultChars": 46077 + "resultChars": 3186 } ] }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini/build-tests-001-rls-tenant-isolation/run-2/result.json" + "sourcePath": "codex-gpt-5.6/build-tests-001-rls-tenant-isolation/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-tests-001-rls-tenant-isolation", @@ -28290,12 +26885,12 @@ { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "6 passed, 0 failed" + "notes": "6 passed, 5 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies `posts` as the cross-organization isolation flaw, fixes the missing `org_id` membership correlation, and validates the corrected behavior with passing pgTAP results. It does not blame `notes` or dismiss the tests." + "judgeNotes": "Correctly identifies `posts` as leaking cross-tenant rows, explains the missing organization match, and grounds the conclusion in the pgTAP failures while recognizing `notes` isolation works." } ], "skills": { @@ -28311,47 +26906,39 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase database tests RLS pgtap local test database\", limit: 5) { nodes { title href content } } }", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"find .claude/skills/supabase-postgres-best-practices/references -maxdepth 1 -type f -printf '%f\\\\n' | sort && curl -fsSL https://supabase.com/changelog.md | sed -n '1,180p' && find . -maxdepth 4 -type f -not -path './.git/*' -printf '%p\\\\n' | sort\"", "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" - }, - { - "url": "https://supabase.com/docs/guides/database/testing", - "title": "Testing Your Database" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - }, + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 23402 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"rg --files -g '\"'!.*'\"' -g '\"'!*skills*'\"' . | sort; supabase --version; supabase test --help; curl -fsSL https://supabase.com/docs/guides/database/postgres/row-level-security.md | rg -n -m 12 'auth.uid|testing|policy|RLS|index'\"", + "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security.md" } ], - "resultChars": 68836 + "resultChars": 1705 } ] }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini/build-tests-001-rls-tenant-isolation/run-3/result.json" + "sourcePath": "codex-gpt-5.6/build-tests-001-rls-tenant-isolation/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-vectors-001-rag-with-permissions", @@ -28376,12 +26963,12 @@ { "name": "HNSW index on the embedding column", "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" }, { "name": "index operator class matches the search operator", "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" }, { "name": "user A search returns only own sections, best match first", @@ -28413,123 +27000,71 @@ "docs": { "calls": [ { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -s https://supabase.com/changelog.md | rg -n 'breaking-change|pgvector|vector|RLS|embeddings|semantic search' -i\"", + "source": "web_search", + "query": "https://supabase.com/changelog.md", "pages": [ { "url": "https://supabase.com/changelog.md" } - ], - "resultChars": 7953 + ] }, { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase semantic search pgvector rpc match_documents row level security\", limit: 5) { nodes { title href ... on Guide { content } ... on TroubleshootingGuide { content } ... on ClientLibraryFunctionReference { content language methodName } } } }", + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,240p'\"", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 69003 + "resultChars": 13905 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase pg_net http_post edge function webhook\", limit: 5) { nodes { title href ... on Guide { content } } } }", + "query": "query { searchDocs(query: \"pgvector semantic search vector embeddings HNSW RPC row level security\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/database/webhooks", - "title": "Database Webhooks" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/webhook-debugging-guide-M8sk47", - "title": "Webhook debugging guide" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - } - ], - "resultChars": 30866 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase database webhook edge function pg_net http_post\", limit: 5) { nodes { title href ... on Guide { content } ... on TroubleshootingGuide { content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" + "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", + "title": "Semantic Search" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/webhook-debugging-guide-M8sk47", - "title": "Webhook debugging guide" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" }, { - "url": "https://supabase.com/docs/guides/database/webhooks", - "title": "Database Webhooks" + "url": "https://supabase.com/docs/guides/ai/going-to-prod", + "title": "Going to Production" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/ai/vector-indexes", + "title": "Vector indexes" }, { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" + "url": "https://supabase.com/docs/guides/ai/vector-indexes/ivf-indexes", + "title": "IVFFlat indexes" } ], - "resultChars": 41913 - }, - { - "source": "web_search", - "query": "Hugging Face inference API Supabase gte-small feature extraction endpoint", - "pages": [] - }, - { - "source": "web_search", - "query": "5784555bd141446ca98e.supabase.red DNS A record", - "pages": [] + "resultChars": 49352 } ] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini/build-vectors-001-rag-with-permissions/run-1/result.json" + "sourcePath": "codex-gpt-5.6/build-vectors-001-rag-with-permissions/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-vectors-001-rag-with-permissions", @@ -28554,12 +27089,12 @@ { "name": "HNSW index on the embedding column", "passed": true, - "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" + "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { "name": "index operator class matches the search operator", "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { "name": "user A search returns only own sections, best match first", @@ -28591,64 +27126,19 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"exposing a table to the Data API RLS Supabase docs\", limit: 5) { nodes { title href content } } }", + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md'", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0027_pg_graphql_authenticated_table_exposed", - "title": "Database Advisor: Lint 0027_pg_graphql_authenticated_table_exposed" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/quickstarts/refine", - "title": "Use Supabase with Refine" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres", - "title": "Connect to your database" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/quickstarts/flask", - "title": "Use Supabase with Python" - } - ], - "resultChars": 54934 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"pgvector semantic search row level security Supabase docs\", limit: 5) { nodes { title href content ... on Guide { subsections { totalCount } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 39356 + "resultChars": 94610 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"create function match_documents vector search Supabase docs RLS\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"pgvector semantic search match_documents RPC HNSW cosine RLS auth.uid\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -28656,38 +27146,38 @@ "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/ai/langchain", - "title": "LangChain" + "url": "https://supabase.com/docs/guides/ai/hybrid-search", + "title": "Hybrid search" }, { "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", "title": "RAG with Permissions" }, { - "url": "https://supabase.com/docs/guides/ai-tools/mcp", - "title": "Supabase MCP Server" + "url": "https://supabase.com/docs/guides/ai/vector-columns", + "title": "Vector columns" }, { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" + "url": "https://supabase.com/docs/guides/ai/langchain", + "title": "LangChain" } ], - "resultChars": 61478 + "resultChars": 57701 } ] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini/build-vectors-001-rag-with-permissions/run-2/result.json" + "sourcePath": "codex-gpt-5.6/build-vectors-001-rag-with-permissions/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "build-vectors-001-rag-with-permissions", @@ -28712,12 +27202,12 @@ { "name": "HNSW index on the embedding column", "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops) WHERE (embedding IS NOT NULL)" + "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" }, { "name": "index operator class matches the search operator", "passed": true, - "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops) WHERE (embedding IS NOT NULL)" + "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" }, { "name": "user A search returns only own sections, best match first", @@ -28748,132 +27238,249 @@ }, "docs": { "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase pgvector semantic search RLS documents owned by user\", limit: 5) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"semantic search pgvector HNSW vector cosine row level security RPC match_documents\", limit: 6) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", + "title": "Semantic Search" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/semantic-search#create-the-database-table-and-webhook", + "title": "Create the database table and webhook" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/semantic-search#create-a-database-function-and-rpc", + "title": "Create a Database Function and RPC" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/semantic-search#query-vectors-in-supabase-edge-functions", + "title": "Query vectors in Supabase Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#when-to-use-semantic-search", + "title": "When to use semantic search" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#how-semantic-search-works", + "title": "How semantic search works" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#embedding-models", + "title": "Embedding models" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#semantic-search-in-postgres", + "title": "Semantic search in Postgres" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#similarity-metric", + "title": "Similarity metric" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#calling-from-your-application", + "title": "Calling from your application" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#filtering-vector-search-by-metadata", + "title": "Filtering vector search by metadata" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#next-steps", + "title": "Next steps" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#pgvector-index-tuning", + "title": "pgvector index tuning" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#ivfflat-index", + "title": "IVFFlat index" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#hnsw-index", + "title": "HNSW index" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#see-also", + "title": "See also" }, { "url": "https://supabase.com/docs/guides/ai/hybrid-search", "title": "Hybrid search" }, { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" + "url": "https://supabase.com/docs/guides/ai/hybrid-search#use-cases-for-hybrid-search", + "title": "Use cases for hybrid search" + }, + { + "url": "https://supabase.com/docs/guides/ai/hybrid-search#when-to-consider-hybrid-search", + "title": "When to consider hybrid search" + }, + { + "url": "https://supabase.com/docs/guides/ai/hybrid-search#how-to-combine-search-methods", + "title": "How to combine search methods" + }, + { + "url": "https://supabase.com/docs/guides/ai/hybrid-search#reciprocal-ranked-fusion-rrf", + "title": "Reciprocal Ranked Fusion (RRF)" + }, + { + "url": "https://supabase.com/docs/guides/ai/hybrid-search#smoothing-constant-k", + "title": "Smoothing constant k" + }, + { + "url": "https://supabase.com/docs/guides/ai/hybrid-search#hybrid-search-in-postgres", + "title": "Hybrid search in Postgres" + }, + { + "url": "https://supabase.com/docs/guides/ai/hybrid-search#running-hybrid-search", + "title": "Running hybrid search" + }, + { + "url": "https://supabase.com/docs/guides/ai/hybrid-search#see-also", + "title": "See also" }, { "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors", "title": "Querying Vectors" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - } - ], - "resultChars": 88393 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase row level security authenticated ownership policy documents table\", limit: 5) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors#basic-similarity-search", + "title": "Basic similarity search" + }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors#semantic-search", + "title": "Semantic search" + }, + { + "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors#filtered-similarity-search", + "title": "Filtered similarity search" + }, + { + "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors#retrieving-specific-vectors", + "title": "Retrieving specific vectors" + }, + { + "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors#listing-vectors", + "title": "Listing vectors" + }, + { + "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors#hybrid-search-vectors--relational-data", + "title": "Hybrid search: Vectors + relational data" + }, + { + "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors#real-world-examples", + "title": "Real-world examples" + }, + { + "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors#rag-retrieval-augmented-generation", + "title": "RAG (retrieval-augmented generation)" + }, + { + "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors#product-recommendations", + "title": "Product recommendations" + }, + { + "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors#filtering-before-similarity-search", + "title": "Filtering before similarity search" + }, + { + "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors#next-steps", + "title": "Next steps" }, { "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", "title": "RAG with Permissions" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions#example", + "title": "Example" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions#alternative-scenarios", + "title": "Alternative scenarios" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - } - ], - "resultChars": 56870 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"gte-small dimensions Supabase.ai.Session embedding size\", limit: 5) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions#documents-owned-by-multiple-people", + "title": "Documents owned by multiple people" + }, { - "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", - "title": "Choosing your Compute Add-on" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions#user-and-document-data-live-outside-of-supabase", + "title": "User and document data live outside of Supabase" }, { - "url": "https://supabase.com/docs/guides/functions/ai-models", - "title": "Running AI Models" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions#direct-postgres-connection", + "title": "Direct Postgres connection" }, { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions#custom-jwt-with-rest-api", + "title": "Custom JWT with REST API" }, { - "url": "https://supabase.com/docs/guides/ai/concepts", - "title": "Concepts" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions#other-scenarios", + "title": "Other scenarios" }, { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" - } - ], - "resultChars": 62354 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase database webhooks edge function trigger http_request pg_net\", limit: 5) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/ai/vector-columns", + "title": "Vector columns" + }, { - "url": "https://supabase.com/docs/guides/database/webhooks", - "title": "Database Webhooks" + "url": "https://supabase.com/docs/guides/ai/vector-columns#usage", + "title": "Usage" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" + "url": "https://supabase.com/docs/guides/ai/vector-columns#enable-the-extension", + "title": "Enable the extension" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/debugging", - "title": "Debugging guide" + "url": "https://supabase.com/docs/guides/ai/vector-columns#create-a-table-to-store-vectors", + "title": "Create a table to store vectors" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/webhook-debugging-guide-M8sk47", - "title": "Webhook debugging guide" + "url": "https://supabase.com/docs/guides/ai/vector-columns#storing-a-vector--embedding", + "title": "Storing a vector / embedding" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/ai/vector-columns#querying-a-vector--embedding", + "title": "Querying a vector / embedding" + }, + { + "url": "https://supabase.com/docs/guides/ai/vector-columns#indexes", + "title": "Indexes" } ], - "resultChars": 53760 + "resultChars": 206750 } ] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini/build-vectors-001-rag-with-permissions/run-3/result.json" + "sourcePath": "codex-gpt-5.6/build-vectors-001-rag-with-permissions/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "deploy-database-001-prometheus-metrics", @@ -28886,7 +27493,7 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "preserved existing app scrape job", @@ -28894,13 +27501,13 @@ }, { "name": "configured the Supabase Metrics API scrape correctly", - "passed": false, - "judgeNotes": "Supabase Basic Auth uses a hardcoded placeholder password instead of password_file, and docker-compose.yml does not mount or provide the required password file/secret. The README also instructs hardcoding the Secret API key." + "passed": true, + "judgeNotes": "HTTPS Supabase metrics scrape is correctly configured with the required path, project target, Basic Auth password_file, preserved app job, and matching Compose secret mount." }, { "name": "documented live deployment and verification steps", - "passed": false, - "judgeNotes": "README lacks matching secret-file setup and concrete verification via Prometheus targets or PromQL/Grafana. It instructs embedding the API key directly in prometheus.yml instead." + "passed": true, + "judgeNotes": "README documents Secret API key creation, matching Compose secret file provisioning, stack recreation, and concrete verification via Prometheus targets or PromQL." } ], "skills": { @@ -28909,50 +27516,39 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"Supabase metrics Prometheus monitoring\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", - "hasContent": true, + "source": "web_search", + "query": "https://supabase.com/changelog.md", "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" + "url": "https://supabase.com/changelog.md" } - ], - "resultChars": 19981 + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/guides/platform/metrics Supabase metrics Prometheus endpoint service_role ...", + "pages": [] } ] }, "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini/deploy-database-001-prometheus-metrics/run-1/result.json" + "sourcePath": "codex-gpt-5.6/deploy-database-001-prometheus-metrics/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "deploy-database-001-prometheus-metrics", @@ -28965,21 +27561,21 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "preserved existing app scrape job", - "passed": false + "passed": true }, { "name": "configured the Supabase Metrics API scrape correctly", - "passed": false, - "judgeNotes": "prometheus.yml is empty, so the app and Supabase scrape jobs are missing. Compose injects the secret into a rendered template instead of mounting a password_file, and the referenced template is not provided." + "passed": true, + "judgeNotes": "HTTPS Supabase metrics scrape uses the correct path and Basic Auth password_file; Compose mounts the matching secret, and the app scrape remains intact." }, { "name": "documented live deployment and verification steps", - "passed": false, - "judgeNotes": "README does not explain creating a Secret API key or placing the matching secret file; it uses an environment variable instead. Thus the required secret setup is missing/mismatched, despite restart and Prometheus target verification steps." + "passed": true, + "judgeNotes": "README covers creating a dedicated Secret API key, securely writing the matching Compose secret file, starting/restarting or reloading the stack, and verifying the Supabase target is UP in Prometheus." } ], "skills": { @@ -28988,53 +27584,35 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"Prometheus metrics Supabase project observability\", limit: 5) {\n nodes {\n ... on Guide {\n title\n href\n content\n }\n ... on TroubleshootingGuide {\n title\n href\n content\n }\n ... on CLICommandReference {\n title\n href\n content\n }\n ... on ManagementApiReference {\n title\n href\n content\n }\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n }\n }\n totalCount\n }\n}", - "hasContent": true, + "source": "web_search", + "query": "https://supabase.com/changelog.md", "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" + "url": "https://supabase.com/changelog.md" } - ], - "resultChars": 29112 + ] } ] }, "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini/deploy-database-001-prometheus-metrics/run-2/result.json" + "sourcePath": "codex-gpt-5.6/deploy-database-001-prometheus-metrics/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "deploy-database-001-prometheus-metrics", @@ -29047,21 +27625,21 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "preserved existing app scrape job", - "passed": false + "passed": true }, { "name": "configured the Supabase Metrics API scrape correctly", - "passed": false, - "judgeNotes": "Supabase Basic Auth uses `password` instead of required `password_file`, and docker-compose does not mount the password file via a volume or Compose secret." + "passed": true, + "judgeNotes": "Meets all requirements: HTTPS Supabase target, correct metrics path, Basic Auth with password_file, matching Compose secret, and preserved app scrape." }, { "name": "documented live deployment and verification steps", - "passed": false, - "judgeNotes": "README uses environment variables and does not instruct placing the Secret API key in a matching secret file, as required. Restart and Prometheus target verification are otherwise concrete." + "passed": true, + "judgeNotes": "README documents creating a dedicated Secret API key, storing it in the matching Compose secret file, starting/reloading Prometheus, and verifying the Supabase target is UP. Endpoint and Basic Auth setup are consistent." } ], "skills": { @@ -29077,43 +27655,49 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"project metrics observability prometheus grafana\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", - "hasContent": true, + "source": "web_search", + "query": "site:supabase.com/docs/guides/platform/metrics Prometheus Supabase metrics endpoint service_role basic_auth ...", + "pages": [] + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,220p'; printf '\\\\nMETRICS DOC\\\\n'; curl -fsSL https://supabase.com/docs/guides/observability/metrics.md | sed -n '1,300p'; printf '\\\\nVENDOR DOC\\\\n'; curl -fsSL https://supabase.com/docs/guides/observability/metrics/vendor-agnostic.md | sed -n '1,360p'\"", + "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" + "url": "https://supabase.com/changelog.md" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" + "url": "https://supabase.com/docs/guides/observability/metrics.md" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" + "url": "https://supabase.com/docs/guides/observability/metrics/vendor-agnostic.md" } ], - "resultChars": 19981 + "resultChars": 10390 } ] }, "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini/deploy-database-001-prometheus-metrics/run-3/result.json" + "sourcePath": "codex-gpt-5.6/deploy-database-001-prometheus-metrics/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "deploy-functions-001-edge-function-secrets", @@ -29140,7 +27724,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -29160,783 +27744,35 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions secrets deploy env var WEATHER_API_KEY\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"sed -n '1,260p' /tmp/sandbox-b5834403/.agents/skills/supabase/SKILL.md && sed -n '1,260p' /tmp/sandbox-b5834403/.agents/skills/supabase-postgres-best-practices/SKILL.md && curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking|edge function|secret' | head -80 && curl -fsSL https://supabase.com/docs/guides/functions/secrets.md | sed -n '1,220p' && curl -fsSL https://supabase.com/docs/guides/functions/deploy.md | sed -n '1,220p'\"", "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#before-you-start", - "title": "Before you start" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-1-create-the-new-api-keys", - "title": "Step 1: Create the new API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-2-swap-the-publishable-key-in-client-code", - "title": "Step 2: Swap the publishable key in client code" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-3-swap-the-secret-key-in-backend-code", - "title": "Step 3: Swap the secret key in backend code" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#database-webhooks-and-pg_net", - "title": "Database Webhooks and pg_net" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-4-update-edge-functions", - "title": "Step 4: Update Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-1-read-the-new-keys-from-the-environment", - "title": "Option 1: Read the new keys from the environment" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-2-adopt-the-supabaseserver-sdk", - "title": "Option 2: Adopt the @supabase/server SDK" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-5-verify-nothing-uses-the-legacy-keys", - "title": "Step 5: Verify nothing uses the legacy keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-6-deactivate-the-legacy-keys", - "title": "Step 6: Deactivate the legacy keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#known-limitations", - "title": "Known limitations" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#next-steps", - "title": "Next steps" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets#default-secrets", - "title": "Default secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets#accessing-environment-variables", - "title": "Accessing environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets#local-secrets", - "title": "Local secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets#production-secrets", - "title": "Production secrets" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#project-setup", - "title": "Project setup" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#create-a-project", - "title": "Create a project" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#set-up-the-database-schema", - "title": "Set up the database schema" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#get-api-details", - "title": "Get API details" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#set-up-google-authentication", - "title": "Set up Google authentication" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#building-the-app", - "title": "Building the app" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#create-new-android-project", - "title": "Create new Android project" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#set-up-api-key-and-secret-securely", - "title": "Set up API key and secret securely" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#create-local-environment-secret", - "title": "Create local environment secret" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#read-and-set-value-to-buildconfig", - "title": "Read and set value to BuildConfig" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#use-value-from-buildconfig", - "title": "Use value from BuildConfig" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#set-up-supabase-dependencies", - "title": "Set up Supabase dependencies" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#set-up-hilt-for-dependency-injection", - "title": "Set up Hilt for dependency injection" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#provide-supabase-instances-with-hilt", - "title": "Provide Supabase instances with Hilt" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#create-a-data-transfer-object", - "title": "Create a data transfer object" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#implement-repositories", - "title": "Implement repositories" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#implement-screens", - "title": "Implement screens" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#implement-the-mainactivity", - "title": "Implement the MainActivity" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#create-the-success-screen", - "title": "Create the success screen" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#overview", - "title": "Overview" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#launching-projects", - "title": "Launching projects" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#nano-compute-instance", - "title": "Nano compute instance" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#recommended-api-keys", - "title": "Recommended API keys" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#changing-compute-sizes", - "title": "Changing compute sizes" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#configuration-changes", - "title": "Configuration changes" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#development-workflow", - "title": "Development workflow" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#creating-a-dev-branch", - "title": "Creating a DEV branch" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#make-database-changes", - "title": "Make database changes" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#create-a-restore-point", - "title": "Create a restore point" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#reverting-changes", - "title": "Reverting changes" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#add-seed-data", - "title": "Add seed data" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#deploying-edge-functions", - "title": "Deploying Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#merge-all-changes", - "title": "Merge all changes" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#security-checks-for-production", - "title": "Security checks for production" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#disaster-recovery-for-production", - "title": "Disaster recovery for production" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#claim-flow", - "title": "Claim flow" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#platform-kit", - "title": "Platform kit" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#debugging-projects", - "title": "Debugging projects" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/cloudflare-turnstile", - "title": "CAPTCHA support with Cloudflare Turnstile" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/cloudflare-turnstile#setup", - "title": "Setup" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/cloudflare-turnstile#code", - "title": "Code" + "url": "https://supabase.com/changelog.md" }, { - "url": "https://supabase.com/docs/guides/functions/examples/cloudflare-turnstile#deploy-the-server-side-validation-edge-functions", - "title": "Deploy the server-side validation Edge Functions" + "url": "https://supabase.com/docs/guides/functions/secrets.md" }, { - "url": "https://supabase.com/docs/guides/functions/examples/cloudflare-turnstile#invoke-the-function-from-your-site", - "title": "Invoke the function from your site" + "url": "https://supabase.com/docs/guides/functions/deploy.md" } ], - "resultChars": 209442 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase secrets set env file edge function\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets#default-secrets", - "title": "Default secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets#accessing-environment-variables", - "title": "Accessing environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets#local-secrets", - "title": "Local secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets#production-secrets", - "title": "Production secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", - "title": "Transcription Telegram Bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#introduction", - "title": "Introduction" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#requirements", - "title": "Requirements" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#setup", - "title": "Setup" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#register-a-telegram-bot", - "title": "Register a Telegram bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-supabase-project-locally", - "title": "Create a Supabase project locally" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-database-table-to-log-the-transcription-results", - "title": "Create a database table to log the transcription results" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-supabase-edge-function-to-handle-telegram-webhook-requests", - "title": "Create a Supabase Edge Function to handle Telegram webhook requests" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-up-the-environment-variables", - "title": "Set up the environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#dependencies", - "title": "Dependencies" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#code-the-telegram-bot", - "title": "Code the Telegram bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#deploy-to-supabase", - "title": "Deploy to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#apply-the-database-migrations", - "title": "Apply the database migrations" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-up-the-webhook", - "title": "Set up the webhook" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-the-function-secrets", - "title": "Set the function secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#test-the-bot", - "title": "Test the bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#introduction", - "title": "Introduction" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#requirements", - "title": "Requirements" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#setup", - "title": "Setup" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#create-a-supabase-project-locally", - "title": "Create a Supabase project locally" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#configure-the-storage-bucket", - "title": "Configure the storage bucket" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#configure-background-tasks-for-supabase-edge-functions", - "title": "Configure background tasks for Supabase Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#create-a-supabase-edge-function-for-speech-generation", - "title": "Create a Supabase Edge Function for speech generation" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#set-up-the-environment-variables", - "title": "Set up the environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#dependencies", - "title": "Dependencies" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#code-the-supabase-edge-function", - "title": "Code the Supabase Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#run-locally", - "title": "Run locally" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#try-it-out", - "title": "Try it out" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#deploy-to-supabase", - "title": "Deploy to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#set-the-function-secrets", - "title": "Set the function secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#test-the-function", - "title": "Test the function" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#invoke-the-default-function", - "title": "Invoke the default function" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#create-a-new-function", - "title": "Create a new function" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-1-add-a-new-function-directory-and-the-function-code", - "title": "Step 1: Add a new function directory and the function code" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-2-restart-the-functions-service-to-pick-up-the-new-function", - "title": "Step 2: Restart the functions service to pick up the new function" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-3-invoke-your-function", - "title": "Step 3: Invoke your function" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#custom-environment-variables", - "title": "Custom environment variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#using-an-env-file-recommended", - "title": "Using an env file (recommended)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#using-inline-environment-variables", - "title": "Using inline environment variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#accessing-variables-in-functions", - "title": "Accessing variables in functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#calling-supabase-services-from-functions", - "title": "Calling Supabase services from functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#internal-vs-external-urls", - "title": "Internal vs external URLs" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#managing-functions-via-dashboard", - "title": "Managing functions via dashboard" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#deploying-functions-to-a-remote-server", - "title": "Deploying functions to a remote server" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#copying-functions-from-supabase-platform", - "title": "Copying functions from Supabase platform" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#500-error-on-invocation", - "title": "500 error on invocation" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#changes-to-function-code-not-reflected-after-editing", - "title": "Changes to function code not reflected after editing" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#custom-env-vars-not-available-in-functions", - "title": "Custom env vars not available in functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#memory-or-timeout-errors", - "title": "Memory or timeout errors" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#before-you-start", - "title": "Before you start" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-1-create-the-new-api-keys", - "title": "Step 1: Create the new API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-2-swap-the-publishable-key-in-client-code", - "title": "Step 2: Swap the publishable key in client code" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-3-swap-the-secret-key-in-backend-code", - "title": "Step 3: Swap the secret key in backend code" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#database-webhooks-and-pg_net", - "title": "Database Webhooks and pg_net" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-4-update-edge-functions", - "title": "Step 4: Update Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-1-read-the-new-keys-from-the-environment", - "title": "Option 1: Read the new keys from the environment" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-2-adopt-the-supabaseserver-sdk", - "title": "Option 2: Adopt the @supabase/server SDK" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-5-verify-nothing-uses-the-legacy-keys", - "title": "Step 5: Verify nothing uses the legacy keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-6-deactivate-the-legacy-keys", - "title": "Step 6: Deactivate the legacy keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#known-limitations", - "title": "Known limitations" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#next-steps", - "title": "Next steps" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#400-missing-function-name-in-request" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#401-invalid-jwt" - } - ], - "resultChars": 119787 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf 'functions help:\\\\n'; supabase functions --help | sed -n '1,240p'; printf '\\\\n--- secrets help ---\\\\n'; supabase secrets --help | sed -n '1,220p'; printf '\\\\n--- changelog ---\\\\n'; curl -fsSL https://supabase.com/changelog.md | rg -n \\\"breaking-change|edge function|functions|secret\\\" -n | sed -n '1,120p'\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 10557 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase config.toml functions verify_jwt\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - }, - { - "url": "https://supabase.com/docs/guides/ai-tools/byo-mcp", - "title": "Deploy MCP servers" - } - ], - "resultChars": 31841 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"edge functions config.toml verify_jwt false\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/github-actions", - "title": "GitHub Actions" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - } - ], - "resultChars": 26184 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"functions invoke curl apikey edge function\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/quickstart", - "title": "Getting Started with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - } - ], - "resultChars": 58426 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"invoke edge function management api supabase\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/functions/architecture", - "title": "Edge Functions Architecture" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-functions", - "title": "Manage Supabase Edge functions" - } - ], - "resultChars": 63286 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"management api invoke edge function\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/functions/architecture", - "title": "Edge Functions Architecture" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-functions", - "title": "Manage Supabase Edge functions" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/reference/csharp/functions-invoke", - "title": "invoke()" - }, - { - "url": "https://supabase.com/docs/guides/ai-tools/mcp", - "title": "Supabase MCP Server" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - } - ], - "resultChars": 135265 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"edge function test request authorization token anon key\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - } - ], - "resultChars": 102695 + "resultChars": 10711 } ] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini/deploy-functions-001-edge-function-secrets/run-1/result.json" + "sourcePath": "codex-gpt-5.6/deploy-functions-001-edge-function-secrets/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "deploy-functions-001-edge-function-secrets", @@ -29963,7 +27799,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get." + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment via Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -29976,368 +27812,24 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query:\"Edge Function secrets env variable secrets set deploy function runtime environment variable\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", - "title": "Transcription Telegram Bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#introduction", - "title": "Introduction" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#requirements", - "title": "Requirements" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#setup", - "title": "Setup" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#register-a-telegram-bot", - "title": "Register a Telegram bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-supabase-project-locally", - "title": "Create a Supabase project locally" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-database-table-to-log-the-transcription-results", - "title": "Create a database table to log the transcription results" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-supabase-edge-function-to-handle-telegram-webhook-requests", - "title": "Create a Supabase Edge Function to handle Telegram webhook requests" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-up-the-environment-variables", - "title": "Set up the environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#dependencies", - "title": "Dependencies" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#code-the-telegram-bot", - "title": "Code the Telegram bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#deploy-to-supabase", - "title": "Deploy to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#apply-the-database-migrations", - "title": "Apply the database migrations" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-up-the-webhook", - "title": "Set up the webhook" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-the-function-secrets", - "title": "Set the function secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#test-the-bot", - "title": "Test the bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#introduction", - "title": "Introduction" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#requirements", - "title": "Requirements" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#setup", - "title": "Setup" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#create-a-supabase-project-locally", - "title": "Create a Supabase project locally" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#configure-the-storage-bucket", - "title": "Configure the storage bucket" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#configure-background-tasks-for-supabase-edge-functions", - "title": "Configure background tasks for Supabase Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#create-a-supabase-edge-function-for-speech-generation", - "title": "Create a Supabase Edge Function for speech generation" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#set-up-the-environment-variables", - "title": "Set up the environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#dependencies", - "title": "Dependencies" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#code-the-supabase-edge-function", - "title": "Code the Supabase Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#run-locally", - "title": "Run locally" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#try-it-out", - "title": "Try it out" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#deploy-to-supabase", - "title": "Deploy to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#set-the-function-secrets", - "title": "Set the function secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#test-the-function", - "title": "Test the function" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#understanding-the-challenge", - "title": "Understanding the challenge" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#understanding-the-architecture", - "title": "Understanding the architecture" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#implementation", - "title": "Implementation" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#step-1-enable-extensions", - "title": "Step 1: Enable extensions" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#step-2-create-utility-functions", - "title": "Step 2: Create utility functions" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#step-3-create-queue-and-triggers", - "title": "Step 3: Create queue and triggers" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#why-not-generate-all-embeddings-in-a-single-edge-function-request", - "title": "Why not generate all embeddings in a single Edge Function request?" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#why-not-one-request-per-row", - "title": "Why not one request per row?" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#why-queue-requests-instead-of-processing-them-immediately", - "title": "Why queue requests instead of processing them immediately?" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#how-do-visibility-timeouts-work", - "title": "How do visibility timeouts work?" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#how-do-we-handle-retries", - "title": "How do we handle retries?" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#is-10-seconds-a-good-interval-for-processing", - "title": "Is 10 seconds a good interval for processing?" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#step-4-create-the-edge-function", - "title": "Step 4: Create the Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#usage", - "title": "Usage" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#1-create-table-to-store-documents-with-embeddings", - "title": "1. Create table to store documents with embeddings" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#2-create-triggers-to-enqueue-embedding-jobs", - "title": "2. Create triggers to enqueue embedding jobs" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#optional-clearing-embeddings-on-update", - "title": "(Optional) Clearing embeddings on update" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#3-insert-and-update-documents", - "title": "3. Insert and update documents" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#conclusion", - "title": "Conclusion" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#see-also", - "title": "See also" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets#default-secrets", - "title": "Default secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets#accessing-environment-variables", - "title": "Accessing environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets#local-secrets", - "title": "Local secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets#production-secrets", - "title": "Production secrets" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration", - "title": "Configuration" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration#branch-configuration-with-remotes", - "title": "Branch configuration with remotes" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration#basic-configuration", - "title": "Basic configuration" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration#remote-specific-configuration", - "title": "Remote-specific configuration" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration#configuration-merging", - "title": "Configuration merging" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration#available-configuration-options", - "title": "Available configuration options" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration#managing-secrets-for-branches", - "title": "Managing secrets for branches" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration#using-dotenvx-for-git-based-workflow", - "title": "Using dotenvx for git-based workflow" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration#environment-file-structure", - "title": "Environment file structure" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration#setting-up-encrypted-secrets", - "title": "Setting up encrypted secrets" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration#using-with-preview-branches", - "title": "Using with preview branches" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration#configuration-examples", - "title": "Configuration examples" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration#multi-environment-setup", - "title": "Multi-environment setup" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration#feature-branch-configuration", - "title": "Feature branch configuration" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration#next-steps", - "title": "Next steps" - } - ], - "resultChars": 158096 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,120p' && printf '\\\\n---\\\\n' && supabase functions --help | sed -n '1,220p' && printf '\\\\n---\\\\n' && supabase secrets --help | sed -n '1,220p'\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3198 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query:\"Edge Functions URL format project not specified functions.supabase.co public endpoint\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/storage-caching", - "title": "Integrating with Supabase Storage" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" - } - ], - "resultChars": 42609 - } - ] + "calls": [] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini/deploy-functions-001-edge-function-secrets/run-2/result.json" + "sourcePath": "codex-gpt-5.6/deploy-functions-001-edge-function-secrets/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "deploy-functions-001-edge-function-secrets", @@ -30364,7 +27856,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment via Deno.env.get('WEATHER_API_KEY')." + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -30385,129 +27877,28 @@ "calls": [ { "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,200p'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 11521 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions environment variables deploy function secrets deno supabase functions\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on CLICommandReference { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } ... on ManagementApiReference { title href content } } totalCount } }", - "hasContent": true, + "query": "/bin/bash -lc \"printf 'Project ref: '; tr -d '\\\\n' < supabase/.temp/project-ref; printf '\\\\nEnvironment variables:\\\\n'; sed -nE 's/\"'^([A-Za-z_][A-Za-z0-9_]*)=.*/'\"\\\\1=/p' .env; printf '\\\\nGitignore:\\\\n'; sed -n '1,200p' .gitignore; printf '\\\\nProject files:\\\\n'; find . -maxdepth 3 -type f \\\\( -path './supabase/*' -o -not -path './.*' \\\\) -not -path './skills/*' -not -path './data/*' -print | sort; printf '\\\\nRemote functions:\\\\n'; supabase functions list -o json; printf '\\\\nRemote secrets:\\\\n'; supabase secrets list -o json; printf '\\\\nCORS docs excerpt:\\\\n'; curl -fsSL https://supabase.com/docs/guides/functions/cors.md | sed -n '1,220p'\"", "pages": [ { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" - } - ], - "resultChars": 62812 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions CORS browser Access-Control-Allow-Origin options preflight\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on CLICommandReference { title href content } } totalCount } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/cors", - "title": "CORS (Cross-Origin Resource Sharing) support for Invoking from the browser" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/unable-to-call-edge-function", - "title": "Unable to call Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - } - ], - "resultChars": 36787 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions verify JWT default no-verify-jwt deploy public function\", limit: 10) { nodes { __typename ... on Guide { title href content } ... on CLICommandReference { title href content } ... on TroubleshootingGuide { title href content } } totalCount } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/push-notifications", - "title": "Sending Push Notifications" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-hooks", - "title": "Auth Hooks" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/discord-bot", - "title": "Building a Discord Bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/ai/examples/openai", - "title": "Generating OpenAI GPT3 completions" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" + "url": "https://supabase.com/docs/guides/functions/cors.md" } ], - "resultChars": 97491 + "resultChars": 5247 } ] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini/deploy-functions-001-edge-function-secrets/run-3/result.json" + "sourcePath": "codex-gpt-5.6/deploy-functions-001-edge-function-secrets/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "deploy-self-hosting-001-docker-compose", @@ -30547,53 +27938,25 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"self-hosted docker compose environment variables Supabase\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting", - "title": "Self-Hosting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/remove-superuser-access", - "title": "Remove Superuser Access from Studio" - } - ], - "resultChars": 66813 - } - ] + "calls": [] }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini/deploy-self-hosting-001-docker-compose/run-1/result.json" + "sourcePath": "codex-gpt-5.6/deploy-self-hosting-001-docker-compose/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "deploy-self-hosting-001-docker-compose", @@ -30640,67 +28003,48 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"self-hosting docker compose environment variables JWT_SECRET ANON_KEY SERVICE_ROLE_KEY\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"git status --short --branch\nsupabase --version\ncurl -fsSL https://supabase.com/changelog.md | sed -n '1,180p'\ncurl -fsSL https://supabase.com/docs/guides/self-hosting/docker.md | sed -n '1,280p'\"", "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" + "url": "https://supabase.com/changelog.md" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/self-hosting/docker.md" } ], - "resultChars": 90702 + "resultChars": 22784 }, { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"Self-hosting with Docker docker setup guide environment variables generate-keys.sh quick start Linux\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog/48048-self-hosted-supabase-envoy-becomes-the-default-api-gateway-b.md | sed -n '1,220p'\ncurl -fsSL https://supabase.com/changelog/46084-self-hosted-supabase-making-analytics-and-vector-opt-in.md | sed -n '1,180p'\ncurl -fsSL https://supabase.com/changelog/46080-self-hosted-supabase-upgrading-from-pg-15-to-17-breaking-change.md | sed -n '1,180p'\ngit ls-remote --tags https://github.com/supabase/supabase.git 'refs/tags/self-hosted/*' | tail -n 12\"", "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" + "url": "https://supabase.com/changelog/48048-self-hosted-supabase-envoy-becomes-the-default-api-gateway-b.md" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/changelog/46084-self-hosted-supabase-making-analytics-and-vector-opt-in.md" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" + "url": "https://supabase.com/changelog/46080-self-hosted-supabase-upgrading-from-pg-15-to-17-breaking-change.md" } ], - "resultChars": 74184 + "resultChars": 13587 } ] }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini/deploy-self-hosting-001-docker-compose/run-2/result.json" + "sourcePath": "codex-gpt-5.6/deploy-self-hosting-001-docker-compose/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "deploy-self-hosting-001-docker-compose", @@ -30744,178 +28088,257 @@ "supabase-postgres-best-practices" ] }, + "docs": { + "calls": [] + }, + "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", + "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-5.6/deploy-self-hosting-001-docker-compose/run-3/result.json" + }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", + "product": [ + "auth" + ], + "topic": [ + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "victim session active before delete-account", + "passed": true + }, + { + "name": "delete_account flow ran for the victim", + "passed": true + }, + { + "name": "delete-account revokes the user's sessions", + "passed": true, + "notes": "sessions left: 0" + }, + { + "name": "deleted user's refresh token is rejected", + "passed": true + }, + { + "name": "deleted user cannot sign back in", + "passed": true + }, + { + "name": "other users keep their sessions and access", + "passed": true + }, + { + "name": "diagnosed and explained session revocation", + "passed": true, + "judgeNotes": "Correctly diagnoses the soft delete, hard-deletes the Auth user and sessions, adds session-aware RLS to close the current data-path window, accurately caveats stateless JWT validity elsewhere until expiry, and correctly distinguishes publishable versus server-only secret keys." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, "docs": { "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i -C 3 'breaking-change|auth|api key|publishable|secret|session|delete user' | sed -n '1,300p'\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 16935 + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"self-hosted docker compose supabase env docker\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on CLICommandReference { title href content } ... on TroubleshootingGuide { title href content } } totalCount } }", + "query": "query { searchDocs(query: \"publishable secret API keys RLS anon authenticated service_role bypass\", limit: 6) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" + } + ], + "resultChars": 91131 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"delete user access token session revoke sign out user deletion JWT remains valid\", limit: 6) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-admin-signout", + "title": "signOut()" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signout" + } + ], + "resultChars": 54461 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"delete user auth.users SQL admin deleteUser account deletion database function cascade sessions\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", + "title": "deleteUser()" }, { - "url": "https://supabase.com/docs/guides/self-hosting", - "title": "Self-Hosting" + "url": "https://supabase.com/docs/reference/kotlin/auth-admin-deleteuser", + "title": "deleteUser()" }, { - "url": "https://supabase.com/docs/guides/self-hosting/updating", - "title": "Update Your Self-Hosted Deployment" + "url": "https://supabase.com/docs/reference/javascript/auth-passkey-delete" } ], - "resultChars": 76655 - } - ] - }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "run": 3, - "sourcePath": "codex-gpt-5.4-mini/deploy-self-hosting-001-docker-compose/run-3/result.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", - "product": [ - "auth" - ], - "topic": [ - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": false, - "checks": [ - { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true - }, - { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" - }, - { - "name": "deleted user's refresh token is rejected", - "passed": true - }, - { - "name": "deleted user cannot sign back in", - "passed": true - }, - { - "name": "other users keep their sessions and access", - "passed": true - }, - { - "name": "diagnosed and explained session revocation", - "passed": false, - "judgeNotes": "It correctly diagnoses the soft delete, deletes auth users/sessions/refresh tokens, closes the Data API window with session-aware RLS, and explains key usage. However, it omits the required caveat that the stateless JWT itself remains locally valid until expiry, so getClaims() or custom signature/expiry-only middleware will still accept it." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ + "resultChars": 23487 + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"delete user invalidate sessions Supabase auth delete account RPC delete account\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ClientLibraryFunctionReference { title href methodName language content } } } }", + "query": "query { searchDocs(query: \"RLS check auth.sessions session_id JWT signed out immediately access token\", limit: 6) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" }, { - "url": "https://supabase.com/docs/guides/platform/sso", - "title": "Enable SSO for Your Organization" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" }, { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" }, { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" + "url": "https://supabase.com/docs/guides/auth/oauth-server/mcp-authentication", + "title": "Model Context Protocol (MCP) Authentication" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" } ], - "resultChars": 43976 + "resultChars": 63733 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"publishable key secret key anon service_role frontend RLS\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ClientLibraryFunctionReference { title href methodName language content } } } }", + "query": "query { searchDocs(query: \"API keys publishable secret RLS bypass service role Authorization header user JWT\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" } ], - "resultChars": 63084 + "resultChars": 42307 } ] }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini/investigate-auth-001-deleted-user-access/run-1/result.json" + "sourcePath": "codex-gpt-5.6/investigate-auth-001-deleted-user-access/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "investigate-auth-001-deleted-user-access", @@ -30959,7 +28382,7 @@ { "name": "diagnosed and explained session revocation", "passed": false, - "judgeNotes": "The fix correctly deletes the Auth user, cascades session/refresh-token removal, and closes the Data API window with session-aware RLS. Key guidance is correct. However, it does not explicitly clarify that stateless access JWTs cannot be recalled and remain accepted by purely local validation such as getClaims() or signature/expiry middleware until exp; “unless you explicitly revoke” is misleading for access tokens." + "judgeNotes": "The fix and API-key guidance are correct, but it omits that the deleted user’s stateless access JWT remains cryptographically valid until expiry for local validation such as getClaims() or custom signature/expiry middleware. Closing the Data API window via session-checking RLS does not invalidate the token itself everywhere." } ], "skills": { @@ -30984,79 +28407,189 @@ ] }, { - "source": "web_search", - "query": "site:supabase.com changelog breaking change delete user tokens publishable key anon service_role", - "pages": [] + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i -C 2 'breaking.change|session|sign.?out|delete.*user|api key|publishable|secret' | head -160\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 11504 }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"delete user does not invalidate access tokens revoke sessions signOut auth.sessions\", limit: 5) {\n nodes {\n ... on Guide {\n title\n href\n content\n }\n ... on TroubleshootingGuide {\n title\n href\n content\n }\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n }\n }\n}", + "query": "query { searchDocs(query: \"delete user access token session revoke sign out API keys publishable secret RLS bypass\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/getting-started", + "title": "Getting Started with OAuth 2.1 Server" + } + ], + "resultChars": 140852 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Auth delete user does not sign out JWT remains valid session_id auth.sessions revoke\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/guides/auth/sessions", "title": "User sessions" }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, { "url": "https://supabase.com/docs/reference/swift/auth-admin-signout", "title": "signOut()" }, + { + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" + }, { "url": "https://supabase.com/docs/reference/javascript/auth-signout" + } + ], + "resultChars": 62877 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"publishable secret API keys RLS bypass Authorization user JWT\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" }, { - "url": "https://supabase.com/docs/guides/auth/signout", - "title": "Signing out" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" } ], - "resultChars": 25116 + "resultChars": 54816 }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"publishable secret keys frontend RLS anon service_role\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", + "query": "query { searchDocs(query: \"delete auth.users database SQL trigger cascade user deletion\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" + "url": "https://supabase.com/docs/guides/troubleshooting/dashboard-errors-when-managing-users-N1ls4A", + "title": "Errors when creating / updating / deleting users" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" + "url": "https://supabase.com/docs/guides/observability/advisors?queryGroups=lint&lint=0002_auth_users_exposed", + "title": "Database Advisor: Lint 0002_auth_users_exposed" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/resolving-500-status-authentication-errors-7bU5U8", + "title": "Resolving 500 Status Authentication Errors" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-hooks", + "title": "Configure Auth Hooks" + } + ], + "resultChars": 54847 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Understanding API keys publishable secret keys RLS service_role bypass browser frontend\", limit: 4) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" }, { "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "title": "API keys" } ], - "resultChars": 91173 + "resultChars": 47606 } ] }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini/investigate-auth-001-deleted-user-access/run-2/result.json" + "sourcePath": "codex-gpt-5.6/investigate-auth-001-deleted-user-access/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "investigate-auth-001-deleted-user-access", @@ -31074,8 +28607,7 @@ "checks": [ { "name": "victim session active before delete-account", - "passed": false, - "notes": "permission denied for table users" + "passed": true }, { "name": "delete_account flow ran for the victim", @@ -31092,17 +28624,17 @@ }, { "name": "deleted user cannot sign back in", - "passed": true + "passed": false, + "notes": "deleted account can still sign in" }, { "name": "other users keep their sessions and access", - "passed": false, - "notes": "permission denied for table users" + "passed": true }, { "name": "diagnosed and explained session revocation", "passed": false, - "judgeNotes": "The fix deletes auth.users, cascades session/refresh-token removal, closes the shown Data API path via RLS, and correctly explains API keys. However, it does not explicitly state that the access JWT itself remains valid until expiry for purely local validation (for example getClaims() or signature/expiry-only middleware), which the rubric requires." + "judgeNotes": "The fix revokes current sessions and blocks data via RLS, but does not delete the auth user or identities. The user can sign in again and create a new session, so this remains a profile soft-delete rather than true account revocation." } ], "skills": { @@ -31119,126 +28651,208 @@ "calls": [ { "source": "search_docs", - "query": "query {\n searchDocs(query: \"delete user revoke sessions auth.admin.deleteUser auth.sessions publishable secret keys anon service_role frontend RLS\", limit: 5) {\n nodes {\n __typename\n ... on Guide { title href content }\n ... on TroubleshootingGuide { title href content }\n ... on CLICommandReference { title href content }\n ... on ManagementApiReference { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n }\n }\n}", + "query": "query { searchDocs(query: \"delete user access token sessions revoke refresh token publishable secret keys RLS\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" }, { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/storage/management/delete-objects", - "title": "Delete Objects" + "url": "https://supabase.com/docs/guides/storage/security/ownership", + "title": "Ownership" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/getting-started", + "title": "Getting Started with OAuth 2.1 Server" }, { "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", "title": "Why is my service role key client getting RLS errors or not returning data?" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" } ], - "resultChars": 51729 + "resultChars": 71362 + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"publishable secret keys frontend apikey RLS anon service_role\", limit: 8) {\n nodes {\n __typename\n ... on Guide { title href content }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", + "query": "query { searchDocs(query: \"security definer function delete current user auth.users\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/database/prisma/prisma-troubleshooting", + "title": "Troubleshooting prisma errors" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" }, { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" + "url": "https://supabase.com/docs/guides/troubleshooting/dashboard-errors-when-managing-users-N1ls4A", + "title": "Errors when creating / updating / deleting users" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" + } + ], + "resultChars": 26414 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"RLS check session_id auth.sessions revoked JWT\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" } ], - "resultChars": 102612 + "resultChars": 54544 }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"revoke sessions delete user database function auth.users admin delete user sessions\", limit: 10) {\n nodes {\n __typename\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", + "query": "query { searchDocs(query: \"Supabase API keys publishable secret keys RLS bypass\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", - "title": "Custom Claims & Role-based Access Control (RBAC)" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" }, { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" }, { - "url": "https://supabase.com/docs/reference/swift/auth-admin-signout", - "title": "signOut()" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + } + ], + "resultChars": 57690 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"deleting users JWT remains valid until expiry sign out revoke session\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", - "title": "deleteUser()" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" + "url": "https://supabase.com/docs/reference/swift/auth-admin-signout", + "title": "signOut()" }, { - "url": "https://supabase.com/docs/reference/javascript/oauth-server-revokegrant" + "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", + "title": "Should I set a shorter Max-Age parameter on the cookies?" + } + ], + "resultChars": 53833 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"publishable and secret API keys sb_publishable sb_secret RLS\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" }, { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + } + ], + "resultChars": 42665 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i -C 2 'breaking|api key|auth|session|jwt|rls' | sed -n '1,240p'\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 72739 + "resultChars": 15708 } ] }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini/investigate-auth-001-deleted-user-access/run-3/result.json" + "sourcePath": "codex-gpt-5.6/investigate-auth-001-deleted-user-access/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "investigate-realtime-001-subscribed-no-events", @@ -31278,7 +28892,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified the missing publication membership and added only public.orders to supabase_realtime, preserving courier_locations, RLS, and policies." + "judgeNotes": "Correctly identified orders missing from supabase_realtime, added only public.orders, and preserved courier_locations, RLS, and existing policies." } ], "skills": { @@ -31293,27 +28907,36 @@ }, "docs": { "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, { "source": "search_docs", - "query": "query { searchDocs(query:\"postgres_changes publication supabase_realtime table not receiving events\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ClientLibraryFunctionReference { title href content methodName language } } } }", + "query": "query { searchDocs(query: \"Realtime Postgres Changes add table supabase_realtime publication postgres_changes subscribed no events\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/realtime-postgres-changes-troubleshooting", - "title": "Realtime: Postgres Changes Troubleshooting" - }, { "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", "title": "Subscribing to Database Changes" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks", - "title": "Benchmarks" + "url": "https://supabase.com/docs/guides/troubleshooting/realtime-postgres-changes-troubleshooting", + "title": "Realtime: Postgres Changes Troubleshooting" }, { "url": "https://supabase.com/docs/guides/realtime/postgres-changes", "title": "Postgres Changes" }, + { + "url": "https://supabase.com/docs/guides/realtime/benchmarks", + "title": "Benchmarks" + }, { "url": "https://supabase.com/docs/guides/realtime/protocol", "title": "Realtime Protocol" @@ -31326,15 +28949,15 @@ "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini/investigate-realtime-001-subscribed-no-events/run-1/result.json" + "sourcePath": "codex-gpt-5.6/investigate-realtime-001-subscribed-no-events/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "investigate-realtime-001-subscribed-no-events", @@ -31374,7 +28997,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified the missing publication membership, added only public.orders to supabase_realtime, preserved courier_locations and RLS/policies, and verified both tables remain published." + "judgeNotes": "Correctly diagnosed orders as missing from supabase_realtime, added only public.orders to the existing publication, and preserved courier_locations, RLS, and policies." } ], "skills": { @@ -31389,48 +29012,62 @@ }, "docs": { "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/changelog Realtime postgres_changes publication breaking change", + "pages": [] + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"postgres_changes INSERT realtime publication orders table replica identity\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"Realtime postgres_changes publication add table supabase_realtime troubleshooting pg_publication_tables\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/database/replication/snowflake", - "title": "Snowflake destination" - }, { "url": "https://supabase.com/docs/guides/troubleshooting/realtime-postgres-changes-troubleshooting", "title": "Realtime: Postgres Changes Troubleshooting" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", + "title": "Subscribing to Database Changes" + }, + { + "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", + "title": "Pipelines FAQ" }, { "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", "title": "Migrate from Postgres to Supabase" }, { - "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", - "title": "Pipelines FAQ" + "url": "https://supabase.com/docs/guides/database/replication/pipelines", + "title": "Set up Pipelines" } ], - "resultChars": 117371 + "resultChars": 74237 } ] }, "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini/investigate-realtime-001-subscribed-no-events/run-2/result.json" + "sourcePath": "codex-gpt-5.6/investigate-realtime-001-subscribed-no-events/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "investigate-realtime-001-subscribed-no-events", @@ -31470,7 +29107,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified orders missing from supabase_realtime, added only public.orders, and verified courier_locations remained published with RLS unchanged." + "judgeNotes": "Correctly diagnosed orders as missing from supabase_realtime, added only public.orders to the existing publication, and preserved courier_locations, RLS, and policies." } ], "skills": { @@ -31485,48 +29122,57 @@ }, "docs": { "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"realtime postgres_changes publication table requires replica identity insert events\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", + "query": "query { searchDocs(query: \"Realtime Postgres Changes add table to supabase_realtime publication SUBSCRIBED no events\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" + "url": "https://supabase.com/docs/guides/troubleshooting/realtime-postgres-changes-troubleshooting", + "title": "Realtime: Postgres Changes Troubleshooting" }, { - "url": "https://supabase.com/docs/guides/database/replication/snowflake", - "title": "Snowflake destination" + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", + "title": "Subscribing to Database Changes" }, { - "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", - "title": "Pipelines FAQ" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" }, { - "url": "https://supabase.com/docs/guides/database/replication/bigquery", - "title": "BigQuery destination" + "url": "https://supabase.com/docs/guides/realtime/benchmarks", + "title": "Benchmarks" }, { - "url": "https://supabase.com/docs/guides/database/replication/ducklake", - "title": "DuckLake destination" + "url": "https://supabase.com/docs/guides/realtime/protocol", + "title": "Realtime Protocol" } ], - "resultChars": 110030 + "resultChars": 117138 } ] }, "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini/investigate-realtime-001-subscribed-no-events/run-3/result.json" + "sourcePath": "codex-gpt-5.6/investigate-realtime-001-subscribed-no-events/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "investigate-reliability-003-edge-function-5xx-correlation", @@ -31539,22 +29185,22 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform and listed all 8 recurring HTTP 503 gateway failures from 07:00Z through 12:00Z on 2026-04-28." + "judgeNotes": "Identified image-transform and the recurring eight HTTP 503 gateway failures from 07:00–12:00 UTC on 2026-04-28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "Although it correctly attributes image-transform 503s to the gateway/platform and cites missing runtime rows plus unchanged deployment, it recommends redeploying/rolling back image-transform, which the rubric explicitly marks as a failure." + "passed": true, + "judgeNotes": "Attributes the 503s to the Supabase gateway/platform layer, supported by absent runtime records, interleaved successes on unchanged deployment, and distinction from avatar-upload’s application-level 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Recommended concrete next steps: check Edge Functions incident/runtime health for a specific UTC window, verify or redeploy affected functions, add targeted downstream error logging, correlate request IDs/timestamps, and open a Supabase support case." + "judgeNotes": "Provides concrete next steps, including retry configuration and escalating to Supabase Support with specific gateway request IDs and the UTC time window." } ], "skills": { @@ -31566,21 +29212,80 @@ "supabase" ] }, - "docs": { - "calls": [] - }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,160p'\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 9005 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions 503 Service Unavailable request does not reach function logs gateway retry\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/status-codes", + "title": "Status codes" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", + "title": "Edge Function 504 error response" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", + "title": "Edge Function 503 error response" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-404-error-response", + "title": "Edge Function 404 error response" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-takes-too-long-to-respond", + "title": "Edge Function takes too long to respond" + } + ], + "resultChars": 31502 + }, + { + "source": "web_search", + "query": "site:status.supabase.com April 28 2026 Edge Functions incident 503 ...", + "pages": [] + }, + { + "source": "web_search", + "query": "Supabase Status \"Apr 28, 2026\" ...", + "pages": [] + } + ] + }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini/investigate-reliability-003-edge-function-5xx-correlation/run-1/result.json" + "sourcePath": "codex-gpt-5.6/investigate-reliability-003-edge-function-5xx-correlation/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "investigate-reliability-003-edge-function-5xx-correlation", @@ -31593,22 +29298,22 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform as the affected function and explicitly listed all eight recurring HTTP 503 failures from 07:00Z through 12:00Z on 2026-04-28." + "judgeNotes": "Identified `image-transform` and the recurring eight HTTP 503 gateway failures between 07:00Z and 12:00Z on 2026-04-28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "Correctly attributes 503s to the gateway and cites missing runtime rows, but explicitly recommends redeploying/rolling back `image-transform`, which the rubric defines as a failure." + "passed": true, + "judgeNotes": "Attributes the recurring 503s to the Supabase Edge gateway/platform layer and supports this with missing runtime/invocation metadata, successful interleaved calls, and distinction from avatar-upload’s runtime 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Recommended concrete actions: check platform health, redeploy or roll back the function, add retry/fallback handling, and open a support ticket with specific gateway request IDs and timestamps." + "judgeNotes": "Recommended opening a Supabase support ticket with the project ref, UTC window, and gateway request IDs, plus concrete retry and diagnostic-capture steps." } ], "skills": { @@ -31617,25 +29322,64 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,180p'\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 10168 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions 503 troubleshooting retry gateway error deployment runtime logs\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", + "title": "Edge Function 503 error response" + }, + { + "url": "https://supabase.com/docs/guides/functions/status-codes", + "title": "Status codes" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-404-error-response", + "title": "Edge Function 404 error response" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + } + ], + "resultChars": 38331 + } + ] }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini/investigate-reliability-003-edge-function-5xx-correlation/run-2/result.json" + "sourcePath": "codex-gpt-5.6/investigate-reliability-003-edge-function-5xx-correlation/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "investigate-reliability-003-edge-function-5xx-correlation", @@ -31648,22 +29392,22 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform as the affected function and clearly described repeated HTTP 503s from 07:00Z through 12:00Z, including all eight gateway failures." + "judgeNotes": "Identified `image-transform` and the recurring eight HTTP 503 gateway failures across 07:00–12:00 UTC on April 28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "Correctly attributes the 503s to the edge/gateway and cites valid evidence, but recommends redeploying/refreshing image-transform, which is an explicit failure condition." + "passed": true, + "judgeNotes": "Attributes the 503s to the gateway/platform layer and supports this with absent runtime invocations for failures, successful nearby executions on the same deployment, and distinction from the function-level avatar-upload 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Recommended concrete actions: redeploying the function, adding targeted error logging, and opening a Supabase support case with specific gateway request IDs and incident timing." + "judgeNotes": "Provides concrete escalation details: open a Supabase support ticket with the project ID, exact time window, and gateway request IDs, plus capture specific failure headers and avoid an unsupported rollback." } ], "skills": { @@ -31676,20 +29420,49 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions 503 gateway request did not reach function troubleshooting\", limit: 5) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/status-codes", + "title": "Status codes" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", + "title": "Edge Function 503 error response" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", + "title": "Edge Function 504 error response" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-404-error-response", + "title": "Edge Function 404 error response" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" + } + ], + "resultChars": 39097 + } + ] }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini/investigate-reliability-003-edge-function-5xx-correlation/run-3/result.json" + "sourcePath": "codex-gpt-5.6/investigate-reliability-003-edge-function-5xx-correlation/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "resolve-dataapi-001-empty-results", @@ -31705,12 +29478,36 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { - "name": "created auth sessions", - "passed": false, - "notes": "Internal server error" + "name": "RLS still enabled on bookmarks", + "passed": true + }, + { + "name": "user A reads own bookmarks", + "passed": true + }, + { + "name": "user B cannot read user A bookmarks", + "passed": true + }, + { + "name": "anon reads no bookmarks", + "passed": true + }, + { + "name": "user A can save a new bookmark", + "passed": true + }, + { + "name": "user B cannot insert a bookmark as user A", + "passed": true + }, + { + "name": "diagnosed RLS and added owner-scoped policies", + "passed": true, + "judgeNotes": "Correctly diagnosed deny-all RLS, kept RLS enabled, and created authenticated owner-scoped SELECT and INSERT policies using auth.uid() with USING and WITH CHECK." } ], "skills": { @@ -31726,103 +29523,56 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"Row Level Security Data API bookmarks select policy anon authenticated grants\", limit: 5) { nodes { title href ... on Guide { content } } } }", - "hasContent": true, + "source": "web_search", + "query": "https://supabase.com/changelog.md", "pages": [ { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/changelog.md" } - ], - "resultChars": 74686 + ] }, { "source": "search_docs", - "query": "query { searchDocs(query: \"row level security auth.uid current_setting request.jwt.claims\", limit: 5) { nodes { title href ... on Guide { content } } } }", + "query": "query { searchDocs(query: \"Row Level Security auth.uid authenticated select insert policy Data API grants\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/realtime/authorization", - "title": "Realtime Authorization" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", - "title": "Row Level Security performance" - }, - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, { "url": "https://supabase.com/docs/guides/api/securing-your-api", "title": "Securing your API" }, { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" - } - ], - "resultChars": 70326 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"auth.uid request.jwt.claims current_setting Supabase\", limit: 5) { nodes { title href ... on Guide { content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/realtime/authorization", - "title": "Realtime Authorization" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/rls-performance-and-best-practices-Z5Jjwv", - "title": "RLS Performance and Best Practices" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", - "title": "Row Level Security performance" + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" } ], - "resultChars": 53269 + "resultChars": 95779 } ] }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini/resolve-dataapi-001-empty-results/run-1/result.json" + "sourcePath": "codex-gpt-5.6/resolve-dataapi-001-empty-results/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "resolve-dataapi-001-empty-results", @@ -31867,7 +29617,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and created authenticated owner-scoped SELECT and INSERT policies using auth.uid(), USING, and WITH CHECK." + "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and created authenticated owner-scoped SELECT and INSERT policies using USING and WITH CHECK against auth.uid()." } ], "skills": { @@ -31884,68 +29634,57 @@ "calls": [ { "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,220p'\"", + "query": "/bin/bash -lc \"curl -LfsS https://supabase.com/changelog.md | rg -n -i 'breaking|rls|data api|row level|grant' | head -80\"", "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 12609 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/docs/guides/api/securing-your-api.md | sed -n '1,220p'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/securing-your-api.md" - } - ], - "resultChars": 11145 + "resultChars": 14816 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"exposing a table to the Data API grants RLS bookmarks auth.uid authenticated\", limit: 5) { nodes { __typename title href content } } }", + "query": "query {\n searchDocs(query: \"Row Level Security auth.uid ownership SELECT INSERT policy Data API grants authenticated\", limit: 5) {\n nodes { title href content }\n }\n }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0013_rls_disabled_in_public", - "title": "Database Advisor: Lint 0013_rls_disabled_in_public" + "url": "https://supabase.com/docs/guides/auth/auth-mfa", + "title": "Multi-Factor Authentication" }, { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0023_sensitive_columns_exposed", - "title": "Database Advisor: Lint 0023_sensitive_columns_exposed" + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" } ], - "resultChars": 46018 + "resultChars": 88445 } ] }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini/resolve-dataapi-001-empty-results/run-2/result.json" + "sourcePath": "codex-gpt-5.6/resolve-dataapi-001-empty-results/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "resolve-dataapi-001-empty-results", @@ -31990,7 +29729,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and created authenticated owner-scoped SELECT and INSERT policies using auth.uid(), plus secure UPDATE/DELETE policies." + "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and added authenticated owner-scoped SELECT and INSERT policies using auth.uid() with USING/WITH CHECK." } ], "skills": { @@ -32005,48 +29744,54 @@ }, "docs": { "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/guides/database/postgres/row-level-security Supabase RLS auth.uid insert select policy ...", + "pages": [] + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Data API exposed table RLS policy bookmarks authenticated anon\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"Row Level Security SELECT INSERT auth.uid ownership Data API grants\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, { "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" } ], - "resultChars": 75991 + "resultChars": 62707 } ] }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini/resolve-dataapi-001-empty-results/run-3/result.json" + "sourcePath": "codex-gpt-5.6/resolve-dataapi-001-empty-results/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "resolve-database-001-migration-history-mismatch", @@ -32059,27 +29804,23 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "the avatar_url column is applied on the hosted profiles table", - "passed": false, - "notes": "avatar_url not found on public.profiles" + "passed": true }, { "name": "migration 20240220000000 is recorded in the remote history", - "passed": false, - "notes": "remote history versions: [\"20240101000000\",\"20240115000000\"]" + "passed": true }, { "name": "remote migration history matches local migration files", - "passed": false, - "notes": "remote-only versions still unreconciled: [\"20240115000000\"] (remote: [\"20240101000000\",\"20240115000000\"], local: [\"20240101000000\",\"20240220000000\"])" + "passed": true }, { "name": "local migrations are a valid reconciled sequence", - "passed": false, - "notes": "expected 20240101000000_create_profiles → → 20240220000000_add_avatar_url, strictly ascending; got [20240101000000_create_profiles.sql, 20240220000000_add_avatar_url.sql]" + "passed": true }, { "name": "production profile data is intact (not reset)", @@ -32087,8 +29828,8 @@ }, { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": false, - "judgeNotes": "`npx supabase db push` (#96) reported the database already up to date and applied no migration. No `migration repair`, `db pull`, or push-based history reconciliation occurred. No mutation workaround was observed; the `psql` commands were read-only." + "passed": true, + "judgeNotes": "`supabase db push --db-url \"$task_db_url\" --yes` applied `20240220000000_add_avatar_url.sql` and finished successfully. History was reconciled by adding local `20240115000000_add_profile_bio.sql`, after which the push succeeded. The direct psql ALTER was rolled back; no prohibited workaround occurred." } ], "skills": { @@ -32102,20 +29843,34 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking.change|migration|db push' | sed -n '1,200p'; curl -fsSL https://supabase.com/docs/guides/deployment/database-migrations.md | sed -n '1,240p'\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations.md" + } + ], + "resultChars": 8383 + } + ] }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini/resolve-database-001-migration-history-mismatch/run-1/result.json" + "sourcePath": "codex-gpt-5.6/resolve-database-001-migration-history-mismatch/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "resolve-database-001-migration-history-mismatch", @@ -32128,17 +29883,15 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "the avatar_url column is applied on the hosted profiles table", - "passed": false, - "notes": "avatar_url not found on public.profiles" + "passed": true }, { "name": "migration 20240220000000 is recorded in the remote history", - "passed": false, - "notes": "remote history versions: [\"20240101000000\",\"20240115000000\"]" + "passed": true }, { "name": "remote migration history matches local migration files", @@ -32146,8 +29899,7 @@ }, { "name": "local migrations are a valid reconciled sequence", - "passed": false, - "notes": "expected 20240101000000_create_profiles → → 20240220000000_add_avatar_url, strictly ascending; got [20240101000000_create_profiles.sql, 20240115000000_add_profile_bio.sql]" + "passed": true }, { "name": "production profile data is intact (not reset)", @@ -32155,8 +29907,8 @@ }, { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": false, - "judgeNotes": "No remote `supabase db push` succeeded; attempts #31 and #33 failed with DNS errors. No successful CLI history reconciliation (`migration repair`, `db pull`, or push) occurred. The API calls shown were read-only inspections, not mutation workarounds." + "passed": true, + "judgeNotes": "`supabase db push --db-url ... --yes` (#13) applied `20240220000000_add_avatar_url.sql` and finished successfully. The orphan bio migration was fetched/added locally via `supabase migration fetch --db-url ... --yes` (#9), then reconciled during the successful push. Only read-only `psql` checks were used; no prohibited workaround was seen." } ], "skills": { @@ -32170,20 +29922,52 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,220p'; printf '\\\\n-- CLI --\\\\n'; supabase --version; supabase db --help | sed -n '1,220p'; printf '\\\\n-- git status --\\\\n'; git status --short; printf '\\\\n-- config --\\\\n'; sed -n '1,240p' supabase/config.toml; printf '\\\\n-- migrations --\\\\n'; for f in supabase/migrations/*.sql; do printf '\\\\nFILE %s\\\\n' \\\"\"'$f\"; sed -n '\"'1,260p' \\\"\"'$f\"; done; printf '\"'\\\\n-- hidden/link files --\\\\n'; find supabase -maxdepth 3 -type f -not -path '*/migrations/*' -print\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 2849 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"sed -n '1,240p' .claude/skills/supabase-postgres-best-practices/references/schema-data-types.md; sed -n '1,240p' .claude/skills/supabase-postgres-best-practices/references/schema-constraints.md; curl -fsSL https://supabase.com/docs/guides/deployment/database-migrations.md | sed -n '1,260p'\"", + "pages": [ + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations.md" + } + ], + "resultChars": 12298 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/docs/guides/deployment/database-migrations.md | sed -n '260,430p'\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations.md" + } + ], + "resultChars": 678 + } + ] }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini/resolve-database-001-migration-history-mismatch/run-2/result.json" + "sourcePath": "codex-gpt-5.6/resolve-database-001-migration-history-mismatch/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "resolve-database-001-migration-history-mismatch", @@ -32196,7 +29980,7 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "the avatar_url column is applied on the hosted profiles table", @@ -32204,18 +29988,15 @@ }, { "name": "migration 20240220000000 is recorded in the remote history", - "passed": false, - "notes": "remote history versions: [\"20240101000000\",\"20240115000000\",\"20260826132330\"]" + "passed": true }, { "name": "remote migration history matches local migration files", - "passed": false, - "notes": "remote-only versions still unreconciled: [\"20240115000000\"] (remote: [\"20240101000000\",\"20240115000000\",\"20260826132330\"], local: [\"20240101000000\",\"20260826132330\"])" + "passed": true }, { "name": "local migrations are a valid reconciled sequence", - "passed": false, - "notes": "expected 20240101000000_create_profiles → → 20240220000000_add_avatar_url, strictly ascending; got [20240101000000_create_profiles.sql, 20260826132330_add_avatar_url.sql]" + "passed": true }, { "name": "production profile data is intact (not reset)", @@ -32223,8 +30004,8 @@ }, { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": false, - "judgeNotes": "No real `supabase db push` succeeded; all attempts failed or were dry runs. The avatar migration was applied around the CLI via Management API POST in #64. Remote history was directly edited with an INSERT into `supabase_migrations.schema_migrations` in #76, while the orphan bio migration remained unreconciled. These are prohibited workarounds." + "passed": true, + "judgeNotes": "`supabase db push --db-url ... --yes` (#27) applied `20240220000000_add_avatar_url.sql` and finished successfully. The orphan bio history was reconciled by adding `20240115000000_add_profile_bio.sql` locally (#21), after which the push succeeded and migration lists matched (#30). No prohibited workaround was used." } ], "skills": { @@ -32238,20 +30019,37 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i -C 2 'breaking.change|migration|database|cli' | sed -n '1,220p'\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 14875 + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/guides/deployment/database-migrations migration history remote migration versions not found local migrations directory supabase migration fetch", + "pages": [] + } + ] }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini/resolve-database-001-migration-history-mismatch/run-3/result.json" + "sourcePath": "codex-gpt-5.6/resolve-database-001-migration-history-mismatch/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "resolve-performance-001-slow-query-cpu-spike", @@ -32282,7 +30080,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -32300,20 +30098,58 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Postgres query optimization indexes EXPLAIN pg_stat_statements\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detecting issues" + } + ], + "resultChars": 36452 + } + ] }, "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini/resolve-performance-001-slow-query-cpu-spike/run-1/result.json" + "sourcePath": "codex-gpt-5.6/resolve-performance-001-slow-query-cpu-spike/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "resolve-performance-001-slow-query-cpu-spike", @@ -32357,37 +30193,73 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", "supabase-postgres-best-practices" ] }, "docs": { "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, { "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | rg -n \"breaking-change|database|postgres|rls|mcp|cli\" -i | head -80'", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i -C 2 'breaking-change|postgres|index|query|performance' | sed -n '1,220p'\"", "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 14550 + "resultChars": 15320 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Postgres query optimization indexes EXPLAIN pg_stat_statements\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detecting issues" + } + ], + "resultChars": 36452 } ] }, "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini/resolve-performance-001-slow-query-cpu-spike/run-2/result.json" + "sourcePath": "codex-gpt-5.6/resolve-performance-001-slow-query-cpu-spike/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "resolve-performance-001-slow-query-cpu-spike", @@ -32418,7 +30290,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -32436,20 +30308,69 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"database query optimization indexes EXPLAIN pg_stat_statements\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detecting issues" + } + ], + "resultChars": 36452 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i -C 2 'breaking-change|postgres|index|query|performance' | sed -n '1,220p'\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 15320 + } + ] }, "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini/resolve-performance-001-slow-query-cpu-spike/run-3/result.json" + "sourcePath": "codex-gpt-5.6/resolve-performance-001-slow-query-cpu-spike/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "resolve-security-002-rls-cross-tenant-leak", @@ -32516,36 +30437,28 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security policy auth.uid select using with check org_id workspace Supabase\", limit: 5) { nodes { title href content } } }", - "hasContent": true, + "source": "web_search", + "query": "https://supabase.com/changelog.md", "pages": [ { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/firebase-auth", - "title": "Firebase Auth" - }, + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|row.level|rls|policy' | head -80\"", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 45755 + "resultChars": 8361 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"enable row level security create policy select own rows Supabase\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"row level security multi tenant team membership policies auth uid using with check\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -32553,38 +30466,30 @@ "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/auth/auth-mfa", + "title": "Multi-Factor Authentication" } ], - "resultChars": 73521 + "resultChars": 51634 } ] }, "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini/resolve-security-002-rls-cross-tenant-leak/run-1/result.json" + "sourcePath": "codex-gpt-5.6/resolve-security-002-rls-cross-tenant-leak/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "resolve-security-002-rls-cross-tenant-leak", @@ -32651,47 +30556,67 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security workspace notes multi-tenant policy\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ClientLibraryFunctionReference { title href methodName language content href } ... on ManagementApiReference { title href content } } } }", + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking.change|row.level|rls|policy' | head -80\"", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/enterprise-sso/auth-sso-saml", - "title": "Single Sign-On with SAML 2.0 for Projects" - }, + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 13759 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Row Level Security workspace membership policy auth.uid tenant isolation\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { "url": "https://supabase.com/docs/guides/resources/glossary", "title": "Glossary" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", + "title": "Column Level Security" + }, + { + "url": "https://supabase.com/docs/guides/realtime/getting_started", + "title": "Getting Started with Realtime" } ], - "resultChars": 88668 + "resultChars": 75044 } ] }, "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini/resolve-security-002-rls-cross-tenant-leak/run-2/result.json" + "sourcePath": "codex-gpt-5.6/resolve-security-002-rls-cross-tenant-leak/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini", + "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, "eval": "resolve-security-002-rls-cross-tenant-leak", @@ -32759,7 +30684,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"row level security membership table policy using auth.uid org_id\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Row Level Security workspace membership multi tenant policy auth.uid()\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -32767,38 +30692,58 @@ "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" }, { "url": "https://supabase.com/docs/guides/auth/auth-mfa", "title": "Multi-Factor Authentication" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" + "url": "https://supabase.com/docs/guides/observability/advisors?queryGroups=lint&lint=0003_auth_rls_initplan", + "title": "Database Advisor: Lint 0003_auth_rls_initplan" } ], - "resultChars": 77604 + "resultChars": 74697 + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|row level|rls|policy' | sed -n '1,120p'\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 8361 } ] }, "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini/resolve-security-002-rls-cross-tenant-leak/run-3/result.json" + "sourcePath": "codex-gpt-5.6/resolve-security-002-rls-cross-tenant-leak/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-auth-001-email-password-flow", @@ -32814,7 +30759,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "auth module loads and the driver completes", @@ -32824,12 +30769,12 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user f0095e33-e42e-4427-8c45-54cc1dda0aa3, signUp returned {\"userId\":\"f0095e33-e42e-4427-8c45-54cc1dda0aa3\"}" + "notes": "db user 8bd83422-bc6a-4011-b9f0-d06518753515, signUp returned {\"userId\":\"8bd83422-bc6a-4011-b9f0-d06518753515\"}" }, { "name": "signup metadata reaches the profile (display name)", - "passed": false, - "notes": "profiles.display_name = \"alex-mta43wde\"" + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" }, { "name": "wrong password is rejected gracefully (no throw, no session)", @@ -32839,12 +30784,12 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"f0095e33-e42e-4427-8c45-54cc1dda0aa3\"}" + "notes": "{\"userId\":\"8bd83422-bc6a-4011-b9f0-d06518753515\"}" }, { "name": "getMyProfile returns the signed-in user's profile", - "passed": false, - "notes": "{\"displayName\":\"alex-mta43wde\",\"plan\":\"free\"}" + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" }, { "name": "app code does not use the secret / service-role key", @@ -32853,29 +30798,67 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "passed": true, + "notes": "imports found in: app/src/auth.mjs" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '---PACKAGE---'; sed -n '1,220p' app/package.json; printf '%s\\\\n' '---SCHEMA---'; sed -n '1,260p' supabase/migrations/0000_profiles_schema.sql; printf '%s\\\\n' '---CONFIG---'; sed -n '1,260p' supabase/config.toml; printf '%s\\\\n' '---TREE---'; find . -maxdepth 3 -type f -not -path './.git/*' -print | sort; printf '%s\\\\n' '---CLI---'; supabase --version; printf '%s\\\\n' '---CHANGELOG HEAD---'; curl -fsSL https://supabase.com/changelog.md | sed -n '1,100p'\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5471 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '---AUTH DOC---'; curl -fsSL https://supabase.com/docs/reference/javascript/auth-signup.md | sed -n '1,180p'; printf '%s\\\\n' '---SIGNIN DOC---'; curl -fsSL https://supabase.com/docs/reference/javascript/auth-signinwithpassword.md | sed -n '1,180p'; printf '%s\\\\n' '---CLIENT DOC---'; curl -fsSL https://supabase.com/docs/reference/javascript/initializing.md | sed -n '1,180p'; printf '%s\\\\n' '---MODULE CHECK---'; node -e \\\"try { console.log(require.resolve('@supabase/supabase-js', {paths:['app']})) } catch (e) { console.log('not installed') }\\\"; printf '%s\\\\n' '---HELP---'; supabase start --help | sed -n '1,180p'\"", + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-signup.md" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword.md" + }, + { + "url": "https://supabase.com/docs/reference/javascript/initializing.md" + } + ], + "resultChars": 2384 + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/reference/javascript auth signUp supabase-js ...", + "pages": [] + } + ] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-auth-001-email-password-flow/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-auth-001-email-password-flow/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-auth-001-email-password-flow", @@ -32891,7 +30874,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "auth module loads and the driver completes", @@ -32901,7 +30884,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 163cc6dc-2885-41cf-accc-06825b428315, signUp returned {\"userId\":\"163cc6dc-2885-41cf-accc-06825b428315\"}" + "notes": "db user d38abda1-6989-425e-9d74-98fb56fbcd51, signUp returned {\"userId\":\"d38abda1-6989-425e-9d74-98fb56fbcd51\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -32916,7 +30899,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"163cc6dc-2885-41cf-accc-06825b428315\"}" + "notes": "{\"userId\":\"d38abda1-6989-425e-9d74-98fb56fbcd51\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -32930,29 +30913,51 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "passed": true, + "notes": "imports found in: app/src/auth.mjs" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"supabase --version; supabase status; printf '\\\\n--- env candidates ---\\\\n'; env | rg '\"'^SUPABASE|''^VITE|''^NEXT_PUBLIC'\"' || true; printf '\\\\n--- changelog head ---\\\\n'; curl -L --max-time 10 -s https://supabase.com/changelog.md | head -80\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5909 + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/reference/javascript auth signUp signInWithPassword getUser select profile supabase-js", + "pages": [] + } + ] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-auth-001-email-password-flow/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-auth-001-email-password-flow/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-auth-001-email-password-flow", @@ -32968,7 +30973,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "auth module loads and the driver completes", @@ -32978,7 +30983,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user ed82a446-7771-465f-a408-9da0f393eafa, signUp returned {\"userId\":\"ed82a446-7771-465f-a408-9da0f393eafa\"}" + "notes": "db user 1ee8fda4-b7ce-47c9-a972-c03f5615918a, signUp returned {\"userId\":\"1ee8fda4-b7ce-47c9-a972-c03f5615918a\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -32993,7 +30998,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"ed82a446-7771-465f-a408-9da0f393eafa\"}" + "notes": "{\"userId\":\"1ee8fda4-b7ce-47c9-a972-c03f5615918a\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -33007,29 +31012,59 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "passed": true, + "notes": "imports found in: app/src/auth.mjs" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"find . -maxdepth 3 -type f -not -path './.git/*' | sort; printf '\\\\n---NODE MODULE CHECK---\\\\n'; test -d app/node_modules/@supabase/supabase-js && echo installed || true; printf '\\\\n---CLI---\\\\n'; supabase --version; printf '\\\\n---DOC CHANGELOG---\\\\n'; curl -fsSL https://supabase.com/changelog.md | head -40; printf '\\\\n---AUTH DOC---\\\\n'; curl -fsSL https://supabase.com/docs/reference/javascript/auth-signup.md | head -80\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signup.md" + } + ], + "resultChars": 2296 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"supabase status; printf '\\\\n---HELP AUTH DOC SEARCH---\\\\n'; curl -fsSL https://supabase.com/docs/reference/javascript/auth-signup | rg -o 'https?[\"'^\" ]+'\"' | head -5 || true; printf '\\\\n---NPM---\\\\n'; npm --version; node --version; find app -maxdepth 2 -type f -print\"", + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-signup" + } + ], + "resultChars": 1861 + } + ] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-auth-001-email-password-flow/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-auth-001-email-password-flow/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-cli-001-bootstrap-app", @@ -33079,24 +31114,41 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'command -v supabase; supabase --version; echo exit:$?; supabase --help | head -60; echo '\"'--- git ---'; git status --short --branch; echo '--- changelog ---'; curl -fsSL --max-time 20 https://supabase.com/changelog.md | head -80\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 4435 + } + ] }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-001-bootstrap-app/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-cli-001-bootstrap-app/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-cli-001-bootstrap-app", @@ -33124,7 +31176,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 3 rows" + "notes": "found 2 rows" }, { "name": "row level security is enabled on todos", @@ -33137,33 +31189,50 @@ { "name": "REST API returns no todos to anonymous requests", "passed": true, - "notes": "0 rows" + "notes": "error 42501: permission denied for table todos" }, { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "3 rows" + "notes": "2 rows" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | head -80; printf '%s\\\\n' '--- init help ---'; supabase init --help; printf '%s\\\\n' '--- migration help ---'; supabase migration --help; printf '%s\\\\n' '--- db help ---'; supabase db --help\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 3761 + } + ] }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-001-bootstrap-app/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-cli-001-bootstrap-app/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-cli-001-bootstrap-app", @@ -33213,24 +31282,41 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | head -80; printf '%s\\\\n' '--- cli help ---'; supabase migration --help; supabase start --help; supabase init --help\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 4766 + } + ] }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-001-bootstrap-app/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-cli-001-bootstrap-app/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-cli-002-declarative-schema", @@ -33244,11 +31330,11 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "supabase db diff used to generate the migration", - "passed": false + "passed": true }, { "name": "schema file updated to include description column", @@ -33256,8 +31342,7 @@ }, { "name": "a new migration was generated for the change", - "passed": false, - "notes": "found 1 migration file(s)" + "passed": true }, { "name": "description column exists in the live database", @@ -33265,24 +31350,41 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\n' '--- config ---' && sed -n '1,220p' supabase/config.toml && printf '%s\n' '--- schema ---' && sed -n '1,220p' supabase/schemas/products.sql && printf '%s\n' '--- migration ---' && sed -n '1,220p' supabase/migrations/20260101000000_create_products.sql && printf '%s\n' '--- cli ---' && supabase --version && printf '%s\n' '--- changelog status ---' && curl -L --max-time 10 -sS https://supabase.com/changelog.md | head -40 && printf '%s\n' '--- schema data types guidance ---' && sed -n '1,220p' skills/supabase-postgres-best-practices/references/schema-data-types.md\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 4660 + } + ] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-002-declarative-schema/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-cli-002-declarative-schema/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-cli-002-declarative-schema", @@ -33296,11 +31398,11 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "supabase db diff used to generate the migration", - "passed": false + "passed": true }, { "name": "schema file updated to include description column", @@ -33316,24 +31418,41 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- config ---' && sed -n '1,220p' supabase/config.toml && printf '%s\\\\n' '--- schema ---' && sed -n '1,220p' supabase/schemas/products.sql && printf '%s\\\\n' '--- migration ---' && sed -n '1,220p' supabase/migrations/20260101000000_create_products.sql && printf '%s\\\\n' '--- relevant best-practice references ---' && sed -n '1,220p' skills/supabase-postgres-best-practices/references/schema-data-types.md && sed -n '1,180p' skills/supabase-postgres-best-practices/references/schema-constraints.md && printf '%s\\\\n' '--- changelog check ---' && curl -fsSL https://supabase.com/changelog.md | sed -n '1,100p'\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 9541 + } + ] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-002-declarative-schema/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-cli-002-declarative-schema/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-cli-002-declarative-schema", @@ -33347,11 +31466,11 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "supabase db diff used to generate the migration", - "passed": false + "passed": true }, { "name": "schema file updated to include description column", @@ -33359,33 +31478,59 @@ }, { "name": "a new migration was generated for the change", - "passed": false, - "notes": "found 1 migration file(s)" + "passed": true }, { "name": "description column exists in the live database", - "passed": false + "passed": true } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- Supabase changelog scan ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -m 20 'breaking-change|database|cli' || true; printf '%s\\\\n' '--- local status ---'; supabase status 2>&1 || true; printf '%s\\\\n' '--- migration help ---'; supabase migration --help 2>&1 || true; printf '%s\\\\n' '--- db diff help ---'; supabase db diff --help 2>&1 || true; printf '%s\\\\n' '--- db reset help ---'; supabase db reset --help 2>&1 || true\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5470 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"psql 'postgresql://postgres:postgres@127.0.0.1:54322/postgres' -X -Atc \\\"select column_name || '|' || data_type || '|' || coalesce(is_nullable,'') from information_schema.columns where table_schema='public' and table_name='products' order by ordinal_position;\\\" && printf '%s\\\\n' '--- declarative docs ---' && curl -fsSL https://supabase.com/docs/guides/local-development/declarative-database-schemas.md | sed -n '1,220p'\"", + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas.md" + } + ], + "resultChars": 8751 + } + ] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-002-declarative-schema/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-cli-002-declarative-schema/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-cli-003-pg-cron-queue-workflow", @@ -33402,43 +31547,60 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": false, - "notes": "job not found in cron.job" + "passed": true, + "notes": "schedule='* * * * *', active=true" }, { "name": "cron command enqueues to the 'tasks' queue", - "passed": false, - "notes": "job not found, so its command can't run" + "passed": true, + "notes": "queue depth 0 -> 1" }, { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 2) from the queue" + "notes": "function removed the seeded message (id 4) from the queue" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- version/help ---' && supabase --version && supabase functions --help && printf '%s\\\\n' '--- config ---' && sed -n '1,240p' supabase/config.toml && printf '%s\\\\n' '--- existing migration ---' && sed -n '1,240p' supabase/migrations/20260101000000_enable_cron_and_queues.sql && printf '%s\\\\n' '--- changelog ---' && curl -L --max-time 20 -s https://supabase.com/changelog.md | head -120 && printf '%s\\\\n' '--- dirs ---' && find supabase -maxdepth 3 -type f -print\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 6850 + } + ] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-003-pg-cron-queue-workflow/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-cli-003-pg-cron-queue-workflow/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-cli-003-pg-cron-queue-workflow", @@ -33470,28 +31632,61 @@ { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 3) from the queue" + "notes": "function removed the seeded message (id 5) from the queue" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- config ---' && sed -n '1,260p' supabase/config.toml && printf '%s\\\\n' '--- migration ---' && sed -n '1,260p' supabase/migrations/20260101000000_enable_cron_and_queues.sql && printf '%s\\\\n' '--- cli ---' && supabase --version && printf '%s\\\\n' '--- help snippets ---' && supabase migration new --help && supabase functions --help && printf '%s\\\\n' '--- changelog ---' && curl -L --fail --silent https://supabase.com/changelog.md | rg -i -C 2 'cron|queue|edge function' | head -100 && printf '%s\\\\n' '--- docs ---' && curl -L --fail --silent https://supabase.com/docs/guides/queues/quickstart.md | head -120\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart.md" + } + ], + "resultChars": 13526 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- cron docs ---' && curl -L --fail --silent https://supabase.com/docs/guides/cron/quickstart.md | rg -n -C 3 'schedule|job|sql|pg_cron' | head -160 && printf '%s\\\\n' '--- queue api ---' && curl -L --fail --silent https://supabase.com/docs/guides/queues/api.md | rg -n -C 3 'create|read|pop|delete|send' | head -220 && printf '%s\\\\n' '--- local status ---' && supabase status || true && printf '%s\\\\n' '--- directory detail ---' && find supabase -maxdepth 3 -type f -print | sort\"", + "pages": [ + { + "url": "https://supabase.com/docs/guides/cron/quickstart.md" + }, + { + "url": "https://supabase.com/docs/guides/queues/api.md" + } + ], + "resultChars": 4091 + } + ] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-003-pg-cron-queue-workflow/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-cli-003-pg-cron-queue-workflow/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-cli-003-pg-cron-queue-workflow", @@ -33518,33 +31713,66 @@ { "name": "cron command enqueues to the 'tasks' queue", "passed": true, - "notes": "queue depth 1 -> 2" + "notes": "queue depth 0 -> 1" }, { "name": "process-tasks function drains the queue", "passed": false, - "notes": "HTTP 404: Function not found" + "notes": "function returned 200 but message 3 is still queued, so it was read but never removed" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | head -120; printf '\\\\n--- cron docs ---\\\\n'; curl -fsSL https://supabase.com/docs/guides/cron/quickstart.md | head -220; printf '\\\\n--- queues docs ---\\\\n'; curl -fsSL https://supabase.com/docs/guides/queues/quickstart.md | head -260; printf '\\\\n--- functions local files ---\\\\n'; find supabase -maxdepth 3 -type f -print\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart.md" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart.md" + } + ], + "resultChars": 15128 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/docs/guides/queues/api.md | rg -n -C 3 'create|send|pop|delete|queue' | head -220; printf '\\\\n--- pgmq installed functions ---\\\\n'; psql 'postgresql://postgres:postgres@127.0.0.1:54322/postgres' -Atc \\\"select extname, extversion from pg_extension where extname in ('pgmq','pg_cron','pg_net'); select n.nspname, p.proname, pg_get_function_identity_arguments(p.oid) from pg_proc p join pg_namespace n on n.oid=p.pronamespace where n.nspname='pgmq' and p.proname in ('create','send','pop','read','delete') order by p.proname,2;\\\"\"", + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/api.md" + } + ], + "resultChars": 612 + } + ] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-003-pg-cron-queue-workflow/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-cli-003-pg-cron-queue-workflow/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-dataapi-001-relational-report", @@ -33588,24 +31816,41 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '---REPORT---'; sed -n '1,260p' app/report.mjs; printf '%s\\\\n' '---PACKAGE---'; sed -n '1,200p' app/package.json; printf '%s\\\\n' '---MIGRATION---'; sed -n '1,300p' supabase/migrations/0000_orders_schema.sql; printf '%s\\\\n' '---CONFIG---'; sed -n '1,220p' supabase/config.toml; printf '%s\\\\n' '---GIT---'; git status --short; printf '%s\\\\n' '---SUPABASE VERSION---'; supabase --version; printf '%s\\\\n' '---CHANGELOG HEAD---'; curl -L --max-time 10 -s https://supabase.com/changelog.md | head -80\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 4415 + } + ] }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-dataapi-001-relational-report/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-dataapi-001-relational-report/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-dataapi-001-relational-report", @@ -33649,8 +31894,14 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [] @@ -33658,15 +31909,15 @@ "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-dataapi-001-relational-report/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-dataapi-001-relational-report/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-dataapi-001-relational-report", @@ -33710,24 +31961,42 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -80'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 4435 + } + ] }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-dataapi-001-relational-report/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-dataapi-001-relational-report/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-dataapi-002-restock-alert-report", @@ -33771,8 +32040,14 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [] @@ -33780,15 +32055,15 @@ "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-dataapi-002-restock-alert-report/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-dataapi-002-restock-alert-report/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-dataapi-002-restock-alert-report", @@ -33832,24 +32107,41 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"sed -n '1,240p' app/restock.mjs; printf '\\\\n--- package ---\\\\n'; sed -n '1,200p' app/package.json; printf '\\\\n--- migration ---\\\\n'; sed -n '1,260p' supabase/migrations/0000_inventory_schema.sql; printf '\\\\n--- status ---\\\\n'; supabase status; printf '\\\\n--- cli ---\\\\n'; supabase --version; printf '\\\\n--- changelog check ---\\\\n'; curl -fsSL --max-time 10 https://supabase.com/changelog.md | head -40\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 3755 + } + ] }, "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-dataapi-002-restock-alert-report/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-dataapi-002-restock-alert-report/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-dataapi-002-restock-alert-report", @@ -33893,24 +32185,42 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,80p'\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 4386 + } + ] }, "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-dataapi-002-restock-alert-report/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-dataapi-002-restock-alert-report/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-database-001-migrate-postgres-to-supabase", @@ -33947,24 +32257,41 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"echo '--- current postgres tools ---'; command -v pg_restore; find /usr /opt -type f -name pg_restore 2>/dev/null | head -20; echo '--- docker images ---'; docker images --format '{{.Repository}}:{{.Tag}}' | head -30; echo '--- changelog check ---'; curl -L --max-time 15 -s https://supabase.com/changelog.md | head -40\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 2330 + } + ] }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-database-001-migrate-postgres-to-supabase/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-database-001-migrate-postgres-to-supabase/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-database-001-migrate-postgres-to-supabase", @@ -34001,8 +32328,14 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [] @@ -34010,15 +32343,15 @@ "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-database-001-migrate-postgres-to-supabase/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-database-001-migrate-postgres-to-supabase/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-database-001-migrate-postgres-to-supabase", @@ -34055,24 +32388,41 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | head -80; printf '\\\\n---CLI---\\\\n'; supabase --version; supabase --help | sed -n '1,180p'; printf '\\\\n---DUMP CONTENTS---\\\\n'; pg_restore --list source.dump; printf '\\\\n---PG RESTORE HELP---\\\\n'; pg_restore --help | sed -n '1,160p'\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 7267 + } + ] }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-database-001-migrate-postgres-to-supabase/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-database-001-migrate-postgres-to-supabase/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-functions-004-service-role-bypass", @@ -34118,24 +32468,60 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Edge Functions verify JWT Authorization header Supabase\", limit: 3) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + } + ], + "resultChars": 18979 + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + } + ] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-004-service-role-bypass/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-functions-004-service-role-bypass/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-functions-004-service-role-bypass", @@ -34152,7 +32538,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "rejects missing auth", @@ -34166,39 +32552,85 @@ }, { "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" + "passed": false, + "notes": "bearer_tokens=2, all_match=false" }, { "name": "user A cannot force-read user B note", "passed": true, - "notes": "status=403" + "notes": "status=200" }, { "name": "user B cannot force-read user A note", "passed": true, - "notes": "status=403" + "notes": "status=200" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL --max-time 10 https://supabase.com/changelog.md | sed -n '1,80p'\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 4386 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"Edge Functions authenticate user Authorization header supabase.auth.getUser service role\", limit: 5) {\n nodes { title href content }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/recursive-functions", + "title": "Recursive / Nested Function Calls" + }, + { + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + } + ], + "resultChars": 41397 + } + ] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-004-service-role-bypass/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-functions-004-service-role-bypass/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-functions-004-service-role-bypass", @@ -34235,62 +32667,67 @@ { "name": "user A cannot force-read user B note", "passed": true, - "notes": "status=403" + "notes": "status=200" }, { "name": "user B cannot force-read user A note", "passed": true, - "notes": "status=403" + "notes": "status=200" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Function getEdgeFunction deploy_edge_function verify_jwt Authorization header Deno.serve\", limit: 5) { nodes { title href ... on Guide { content } ... on TroubleshootingGuide { content } } } }", + "query": "query { searchDocs(query: \"Edge Functions authenticate user Authorization header supabase-js createClient auth.getUser\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content methodName language } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" + "url": "https://supabase.com/docs/reference/javascript/auth-getuser" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" + "url": "https://supabase.com/docs/guides/functions/websockets", + "title": "Handling WebSockets" }, { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" } ], - "resultChars": 40523 + "resultChars": 37177 } ] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-004-service-role-bypass/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-functions-004-service-role-bypass/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-functions-005-dual-auth-user-secret", @@ -34318,206 +32755,104 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"error\": \"missing user access token\"}" + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { "name": "user with JWT reads only their own rows", "passed": false, - "notes": "status 401: {\"error\": \"invalid user access token\"}" + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { "name": "user cannot read another user's rows by passing user_id", "passed": false, - "notes": "status 401: {\"error\": \"invalid user access token\"}" + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\": \"f17dea0a-c74a-40bd-b493-d2ed715cad72\", \"metric\": \"steps_b_mta4ku8w\", \"value\": 222}]" + "passed": false, + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"error\": \"missing user access token\"}" + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"error\": \"invalid user access token\"}" + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"error\": \"missing user access token\"}" + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { "name": "implementation uses @supabase/server", - "passed": false, - "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" + "passed": true, + "notes": "imports @supabase/server / withSupabase" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"Edge Function apikey header service role verify_jwt auth.getUser\", limit: 10) {\n nodes {\n ... on Guide {\n title\n href\n content\n }\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n ... on TroubleshootingGuide {\n title\n href\n content\n }\n ... on CLICommandReference {\n title\n href\n content\n }\n }\n totalCount\n }\n}", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | head -80; printf '\\\\n--- function docs ---\\\\n'; curl -fsSL https://supabase.com/docs/guides/functions/auth | sed -n '1,220p'\"", "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getclaims" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-listusers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/changelog.md" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-createuser" + "url": "https://supabase.com/docs/guides/functions/auth" } ], - "resultChars": 74254 + "resultChars": 364954 }, { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"supabase functions index.ts main.ts entrypoint\", limit: 10) {\n nodes {\n ... on Guide { title href content }\n ... on CLICommandReference { title href content }\n }\n totalCount\n }\n}", + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/docs/guides/functions/auth.md | rg -n -C 4 'withSupabase|auth: \\\\[|supabaseAdmin|secret' | head -160\"", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/api/rest/generating-types", - "title": "Generating TypeScript Types" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-environment", - "title": "Development Environment" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart", - "title": "Getting Started with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/upstash-redis", - "title": "Upstash Redis" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/sentry-monitoring", - "title": "Monitoring with Sentry" - }, - { - "url": "https://supabase.com/docs/guides/ai/langchain", - "title": "LangChain" + "url": "https://supabase.com/docs/guides/functions/auth.md" } ], - "resultChars": 70758 + "resultChars": 6476 }, { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"auth publishable user claims withSupabase supabase server\", limit: 10) {\n nodes {\n ... on Guide { title href content }\n }\n totalCount\n }\n}", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/docs/guides/functions/auth.md | sed -n '18,42p'; curl -fsSL https://supabase.com/docs/guides/functions/auth.md | sed -n '120,180p'; printf '\\\\n--- db checks ---\\\\n'; PGPASSWORD=postgres psql -h 127.0.0.1 -p 54322 -U postgres -d postgres -c '\\\\d+ public.user_stats' -c 'select polname, polroles, polcmd, polqual from pg_policies where schemaname = '\\\\''public'\\\\'' and tablename = '\\\\''user_stats'\\\\'';' -c 'select indexname, indexdef from pg_indexes where schemaname = '\\\\''public'\\\\'' and tablename = '\\\\''user_stats'\\\\'';'\"", "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/auth0", - "title": "Auth0" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/aws-cognito", - "title": "Amazon Cognito (Amplify)" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server", - "title": "OAuth 2.1 Server" + "url": "https://supabase.com/docs/guides/functions/auth.md" } ], - "resultChars": 75028 - }, - { - "source": "web_search", - "query": "\"failed to determine entrypoint\" \"supabase functions serve\"", - "pages": [] - }, - { - "source": "web_search", - "query": "site:github.com/supabase/cli \"functions/main/index.ts\" \"failed to determine entrypoint\"", - "pages": [] + "resultChars": 4130 } ] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-005-dual-auth-user-secret/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-functions-005-dual-auth-user-secret/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-functions-005-dual-auth-user-secret", @@ -34538,96 +32873,76 @@ "passed": false, "checks": [ { - "name": "read stack config from `supabase status`", + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" + }, + { + "name": "rejects request with no credentials", + "passed": true, + "notes": "status 503: {\"message\":\"name resolution failed\"}" + }, + { + "name": "user with JWT reads only their own rows", + "passed": false, + "notes": "status 503: {\"message\":\"name resolution failed\"}" + }, + { + "name": "user cannot read another user's rows by passing user_id", + "passed": false, + "notes": "status 503: {\"message\":\"name resolution failed\"}" + }, + { + "name": "service key bypasses RLS to read the target user's rows", "passed": false, - "notes": "missing API_URL/SECRET_KEY/PUBLISHABLE_KEY — new API keys are required for @supabase/server; is the stack running on a new-enough CLI? got keys: ANON_KEY, DB_URL, INBUCKET_URL, JWT_SECRET, MAILPIT_URL, PUBLISHABLE_KEY, S3_PROTOCOL_ACCESS_KEY_ID, S3_PROTOCOL_ACCESS_KEY_SECRET, S3_PROTOCOL_REGION, SECRET_KEY, SERVICE_ROLE_KEY, STORAGE_S3_URL, STUDIO_URL" + "notes": "status 503: {\"message\":\"name resolution failed\"}" + }, + { + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 503: {\"message\":\"name resolution failed\"}" + }, + { + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 503: {\"message\":\"name resolution failed\"}" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 503: {\"message\":\"name resolution failed\"}" + }, + { + "name": "implementation uses @supabase/server", + "passed": true, + "notes": "imports @supabase/server / withSupabase" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions verify JWT service role apikey header authorization supabase-js user stats\", limit: 5) { edges { node { ... on Guide { title href content } ... on CLICommandReference { title href content } ... on ManagementApiReference { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" - } - ], - "resultChars": 50544 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"withSupabase multiple auth modes same function user secret\", limit: 10) { edges { node { ... on Guide { title href content } ... on CLICommandReference { title href content } ... on ManagementApiReference { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } } }", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\n' '--- changelog check ---' && curl -fsSL https://supabase.com/changelog.md | head -80 && printf '%s\n' '--- function help ---' && supabase functions new --help && printf '%s\n' '--- local env candidates ---' && env | rg '\"'^(SUPABASE|JWT|ANON|SERVICE|PROJECT)'\"' | sed -E 's/=.*/=/' || true && printf '%s\n' '--- supabase dirs ---' && find supabase -maxdepth 3 -type f -print\"", "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/overview", - "title": "Third-party auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/enterprise-sso", - "title": "Enterprise Single Sign-On" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/firebase-auth", - "title": "Firebase Auth" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/migrating-auth-users-between-projects", - "title": "Migrating Auth Users Between Supabase Projects" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 107966 + "resultChars": 2322 }, { "source": "web_search", - "query": "Supabase CLI failed to determine entrypoint hyphen function name functions serve", + "query": "site:supabase.com/docs/guides/functions @supabase/server withSupabase auth none ctx supabase", "pages": [] } ] @@ -34635,15 +32950,15 @@ "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-005-dual-auth-user-secret/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-functions-005-dual-auth-user-secret/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-functions-005-dual-auth-user-secret", @@ -34664,30 +32979,87 @@ "passed": false, "checks": [ { - "name": "read stack config from `supabase status`", + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" + }, + { + "name": "rejects request with no credentials", + "passed": true, + "notes": "status 503: {\"message\":\"name resolution failed\"}" + }, + { + "name": "user with JWT reads only their own rows", + "passed": false, + "notes": "status 503: {\"message\":\"name resolution failed\"}" + }, + { + "name": "user cannot read another user's rows by passing user_id", + "passed": false, + "notes": "status 503: {\"message\":\"name resolution failed\"}" + }, + { + "name": "service key bypasses RLS to read the target user's rows", + "passed": false, + "notes": "status 503: {\"message\":\"name resolution failed\"}" + }, + { + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 503: {\"message\":\"name resolution failed\"}" + }, + { + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 503: {\"message\":\"name resolution failed\"}" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 503: {\"message\":\"name resolution failed\"}" + }, + { + "name": "implementation uses @supabase/server", "passed": false, - "notes": "missing API_URL/SECRET_KEY/PUBLISHABLE_KEY — new API keys are required for @supabase/server; is the stack running on a new-enough CLI? got keys: ANON_KEY, DB_URL, JWT_SECRET, PUBLISHABLE_KEY, SECRET_KEY, SERVICE_ROLE_KEY" + "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- migration ---' && sed -n '1,240p' supabase/migrations/0000_stats_schema.sql && printf '%s\\\\n' '--- config ---' && sed -n '1,220p' supabase/config.toml && printf '%s\\\\n' '--- versions/help ---' && supabase --version && supabase functions --help | sed -n '1,220p' && printf '%s\\\\n' '--- changelog ---' && curl -fsSL https://supabase.com/changelog.md | head -80\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 6734 + } + ] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-005-dual-auth-user-secret/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-functions-005-dual-auth-user-secret/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-rls-003-org-roles-permissions", @@ -34702,7 +33074,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "RLS enabled on documents", @@ -34710,80 +33082,107 @@ }, { "name": "viewer sees only documents in their org", - "passed": false + "passed": true }, { "name": "viewer cannot insert", - "passed": false + "passed": true }, { "name": "editor can insert own org document", - "passed": false + "passed": true }, { "name": "editor can update own document", - "passed": false + "passed": true }, { "name": "editor cannot update another user's document", - "passed": false + "passed": true }, { "name": "editor cannot delete another user's document", - "passed": false + "passed": true }, { "name": "admin can update any document in their org", - "passed": false + "passed": true }, { "name": "admin can delete any document in their org", - "passed": false + "passed": true }, { "name": "admin cannot affect another org", - "passed": false + "passed": true }, { "name": "WITH CHECK blocks editor from moving document to another org", - "passed": false + "passed": true }, { "name": "editor can still see their own org's roster", - "passed": false + "passed": true }, { "name": "cannot see another org's membership roster", - "passed": false + "passed": true }, { "name": "a viewer role in one org doesn't grant admin power in another org", - "passed": false + "passed": true }, { "name": "multi-org user can act as admin in the org where they hold that role", - "passed": false + "passed": true } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Row Level Security policies security definer function auth.uid Supabase\", limit: 3) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" + } + ], + "resultChars": 32899 + } + ] }, "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-rls-003-org-roles-permissions/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-rls-003-org-roles-permissions/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-rls-003-org-roles-permissions", @@ -34862,24 +33261,57 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{\"query\":\"query Search($q: String!) { searchDocs(query: $q, limit: 3) { nodes { title href content } } }\",\"variables\":{\"q\":\"Supabase Row Level Security policies security definer function membership lookup\"}}", + "hasContent": false, + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase Row Level Security policies security definer function membership lookup\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" + } + ], + "resultChars": 35747 + } + ] }, "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-rls-003-org-roles-permissions/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-rls-003-org-roles-permissions/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-rls-003-org-roles-permissions", @@ -34958,8 +33390,14 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [] @@ -34967,15 +33405,15 @@ "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-rls-003-org-roles-permissions/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-rls-003-org-roles-permissions/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-storage-001-private-bucket-access", @@ -34990,122 +33428,63 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 01a03e35-0c5d-712c-bb53-e651ba7f07e3/receipt-alpha.pdf, 01a03e35-0c5d-712c-bb53-e651ba7f07e3/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Private user-files bucket, authenticated owner-scoped SELECT/INSERT policies with RLS retained, and expiring createSignedUrl sharing code are all provided." + "passed": false, + "notes": "no row in storage.buckets with id or name 'user-files'" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"storage.objects policy authenticated user own path bucket upload download private bucket\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", - "hasContent": true, + "source": "web_search", + "query": "https://supabase.com/changelog.md", "pages": [ { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", - "title": "Why can't I upload/list/etc my public bucket?" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", - "title": "Build a User Management App with Angular" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-angular", - "title": "Build a User Management App with Ionic Angular" + "url": "https://supabase.com/changelog.md" } - ], - "resultChars": 71588 + ] }, { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"createSignedUrl supabase-js storage signed url private bucket\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", - "hasContent": true, + "source": "web_search", + "query": "Supabase JavaScript createSignedUrl storage docs ...", + "pages": [] + }, + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/storage/schema/design", "pages": [ { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", - "title": "Manage Storage Image Transformations usage" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", - "title": "Storage Image Transformations" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" - }, - { - "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads", - "title": "Resumable Uploads" + "url": "https://supabase.com/docs/guides/storage/schema/design" } - ], - "resultChars": 46845 + ] } ] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-storage-001-private-bucket-access/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-storage-001-private-bucket-access/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-storage-001-private-bucket-access", @@ -35137,7 +33516,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 01a03e34-da59-73cf-923f-7eba33f5ea95/receipt-alpha.pdf, 01a03e34-da59-73cf-923f-7eba33f5ea95/receipt-beta.pdf" + "notes": "saw: 01a0803c-8c98-74bf-8830-334fe819cccf/receipt-alpha.pdf, 01a0803c-8c98-74bf-8830-334fe819cccf/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -35158,57 +33537,74 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Private bucket created, RLS remains enabled, authenticated SELECT/INSERT policies enforce UID-prefixed ownership, and createSignedUrl uses a 10-minute expiry." + "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT policies, enabled RLS, and expiring createSignedUrl usage are all present." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"storage.objects RLS policy auth.uid path prefix\", limit: 5) {\n nodes {\n __typename\n ... on Guide { title href content }\n ... on TroubleshootingGuide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n }\n }\n}", + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|storage|signed' | head -80\"", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 6201 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Supabase Storage access control RLS policies foldername signed URLs createSignedUrl\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", "title": "Storage Helper Functions" }, { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" }, { - "url": "https://supabase.com/docs/guides/storage/debugging/error-codes", - "title": "Error Codes" + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" }, { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/storage/s3/authentication", + "title": "S3 Authentication" } ], - "resultChars": 27578 + "resultChars": 25624 } ] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-storage-001-private-bucket-access/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-storage-001-private-bucket-access/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-storage-001-private-bucket-access", @@ -35223,7 +33619,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "bucket user-files exists", @@ -35240,7 +33636,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 01a03e34-ce1a-738e-8f63-7da3e0412d5f/receipt-alpha.pdf, 01a03e34-ce1a-738e-8f63-7da3e0412d5f/receipt-beta.pdf" + "notes": "saw: 01a0803c-536e-7467-85b0-37aba9165421/receipt-alpha.pdf, 01a0803c-536e-7467-85b0-37aba9165421/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -35260,96 +33656,25 @@ }, { "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT policies with RLS retained, and expiring createSignedUrl code are all present." + "passed": false, + "judgeNotes": "Bucket and authenticated owner-scoped policies pass, but no visible supabase-js code calling createSignedUrl/createSignedUrls with an expiry was provided; the response only claims such an example exists in a file." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase storage bucket policies create bucket SQL storage.objects policy user id path\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ClientLibraryFunctionReference { title href language methodName content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", - "title": "Custom Roles" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter", - "title": "Build a User Management App with Flutter" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/storage/quickstart", - "title": "Storage Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - } - ], - "resultChars": 94846 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createSignedUrl storage temporary link expires\", limit: 5) { nodes { __typename ... on ClientLibraryFunctionReference { title href language methodName content } ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", - "title": "Storage Image Transformations" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" - }, - { - "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" - } - ], - "resultChars": 26956 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"storage signurl signed URL javascript reference\", limit: 5) { nodes { __typename ... on ClientLibraryFunctionReference { title href language methodName content } ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-uploadtosignedurl" - } - ], - "resultChars": 7453 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"storage.objects policy authenticated users own objects foldername auth.uid private bucket\", limit: 10) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href language methodName content } } } }", + "query": "{ searchDocs(query: \"Storage RLS policies bucket_id owner path auth.uid signed URL createSignedUrl\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { @@ -35357,98 +33682,38 @@ "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/storage/quickstart", - "title": "Storage Quickstart" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter", + "title": "Build a User Management App with Flutter" }, { "url": "https://supabase.com/docs/guides/storage/security/access-control", "title": "Storage Access Control" }, - { - "url": "https://supabase.com/docs/guides/storage/cdn/fundamentals", - "title": "Storage CDN" - }, - { - "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", - "title": "Copy Objects" - }, { "url": "https://supabase.com/docs/guides/security/product-security", "title": "Secure configuration of Supabase products" }, - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/design", - "title": "The Storage Schema" - } - ], - "resultChars": 40670 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"storage policy allow users own files path starts with auth.uid foldername\", limit: 10) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href language methodName content } } } }", - "hasContent": true, - "pages": [ { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", "title": "Configure S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/storage", - "title": "Storage" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - }, - { - "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", - "title": "Copy Objects" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/design", - "title": "The Storage Schema" - }, - { - "url": "https://supabase.com/docs/guides/storage/management/download-objects", - "title": "Download Objects" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" } ], - "resultChars": 66759 + "resultChars": 56761 } ] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-storage-001-private-bucket-access/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-storage-001-private-bucket-access/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-tests-001-rls-tenant-isolation", @@ -35467,67 +33732,52 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" + "notes": "1 file(s): supabase/tests/tenant_isolation.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "4 passed, 0 failed" + "notes": "13 passed, 0 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies `posts` as the broken tenant-isolation policy, explains the cross-organization read flaw, distinguishes `notes` as correctly scoped, and reports passing pgTAP regression results after the fix." + "judgeNotes": "Correctly identifies `posts` as having the cross-tenant read leak and reports passing pgTAP verification after fixing the policies." } ], "skills": { - "available": [], + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], "loaded": [] }, "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"database tests pgTAP supabase test db\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on CLICommandReference { title href content } } } }", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- config ---'; sed -n '1,260p' supabase/config.toml; printf '%s\\\\n' '--- migration ---'; sed -n '1,320p' supabase/migrations/20240101000000_init.sql; printf '%s\\\\n' '--- RLS guidance ---'; sed -n '1,260p' skills/supabase-postgres-best-practices/references/security-rls-basics.md; sed -n '1,260p' skills/supabase-postgres-best-practices/references/security-rls-performance.md; printf '%s\\\\n' '--- CLI ---'; supabase --version; supabase test db --help; printf '%s\\\\n' '--- changelog head ---'; curl -fsSL https://supabase.com/changelog.md | sed -n '1,100p'\"", "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-test-db", - "title": "Tests local database with pgTAP" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, - { - "url": "https://supabase.com/docs/guides/database/testing", - "title": "Testing Your Database" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 43238 + "resultChars": 5442 } ] }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-tests-001-rls-tenant-isolation/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-tests-001-rls-tenant-isolation/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-tests-001-rls-tenant-isolation", @@ -35546,38 +33796,55 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation.sql" + "notes": "1 file(s): supabase/tests/database/tenant_isolation_test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "4 passed, 0 failed" + "notes": "9 passed, 0 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies `posts` as having broken tenant isolation, cites pgTAP failures showing cross-org reads, and distinguishes `notes` as correctly isolated." + "judgeNotes": "Correctly identifies `posts` as allowing cross-organization reads, fixes the policy, and validates tenant isolation with passing pgTAP tests. It does not blame `notes` or dismiss test results." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"sed -n '1,260p' supabase/migrations/20240101000000_init.sql && printf '\\\\n--- config ---\\\\n' && sed -n '1,220p' supabase/config.toml && printf '\\\\n--- RLS guidance ---\\\\n' && sed -n '1,260p' skills/supabase-postgres-best-practices/references/security-rls-basics.md && sed -n '1,240p' skills/supabase-postgres-best-practices/references/security-rls-performance.md && printf '\\\\n--- changelog check ---\\\\n' && curl -L --max-time 10 -s https://supabase.com/changelog.md | head -80\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 12226 + } + ] }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-tests-001-rls-tenant-isolation/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-tests-001-rls-tenant-isolation/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-tests-001-rls-tenant-isolation", @@ -35596,38 +33863,55 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" + "notes": "1 file(s): supabase/tests/database/tenant_isolation.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "4 passed, 0 failed" + "notes": "16 passed, 0 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies `posts` as having the cross-organization RLS flaw and reports pgTAP passing after the policy fix." + "judgeNotes": "Correctly identifies `posts` as the tenant-isolation flaw, explains the missing `org_id` match, and validates the fix with passing pgTAP results." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL --max-time 15 https://supabase.com/changelog.md | rg -i -m 5 'breaking-change|row level|rls' || true; mkdir -p supabase/tests/database; supabase start --exclude studio --exclude edge-runtime --exclude analytics --exclude vector --exclude imgproxy --exclude logflare --exclude supavisor\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 2627 + } + ] }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-tests-001-rls-tenant-isolation/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-tests-001-rls-tenant-isolation/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-vectors-001-rag-with-permissions", @@ -35652,12 +33936,12 @@ { "name": "HNSW index on the embedding column", "passed": true, - "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" + "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" }, { "name": "index operator class matches the search operator", "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" + "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" }, { "name": "user A search returns only own sections, best match first", @@ -35677,40 +33961,59 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ { - "source": "web_search", - "query": "site:supabase.com docs edge functions invoke project not specified functions.supabase.co", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com \"Project not specified\" \"Edge Functions\"", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com Supabase.ai.Session gte-small embedding API", - "pages": [] + "source": "search_docs", + "query": "query {\n searchDocs(query: \"pgvector semantic search match_documents RPC HNSW vector extension row level security auth.uid\", limit: 5) {\n nodes { title href content }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + }, + { + "url": "https://supabase.com/docs/guides/ai/hybrid-search", + "title": "Hybrid search" + }, + { + "url": "https://supabase.com/docs/guides/ai/vector-indexes", + "title": "Vector indexes" + }, + { + "url": "https://supabase.com/docs/guides/ai/vector-indexes/ivf-indexes", + "title": "IVFFlat indexes" + } + ], + "resultChars": 47589 } ] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-vectors-001-rag-with-permissions/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-vectors-001-rag-with-permissions/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-vectors-001-rag-with-permissions", @@ -35725,32 +34028,30 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "document_sections.embedding is vector(384)", - "passed": false, - "notes": "vector(1536)" + "passed": true, + "notes": "vector(384)" }, { "name": "HNSW index on the embedding column", "passed": true, - "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" + "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { "name": "index operator class matches the search operator", "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { "name": "user A search returns only own sections, best match first", - "passed": false, - "notes": "different vector dimensions 1536 and 384" + "passed": true }, { "name": "user B search returns only own sections, best match first", - "passed": false, - "notes": "different vector dimensions 1536 and 384" + "passed": true }, { "name": "user A reads only own sections through the API", @@ -35762,24 +34063,78 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Supabase pgvector semantic search match_documents RPC HNSW RLS\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + }, + { + "url": "https://supabase.com/docs/guides/ai/hybrid-search", + "title": "Hybrid search" + }, + { + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" + }, + { + "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors", + "title": "Querying Vectors" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + } + ], + "resultChars": 95476 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"gte-small embedding dimensions 384 Supabase Edge Functions\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + }, + { + "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", + "title": "Choosing your Compute Add-on" + }, + { + "url": "https://supabase.com/docs/guides/ai/concepts", + "title": "Concepts" + } + ], + "resultChars": 38298 + } + ] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-vectors-001-rag-with-permissions/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-vectors-001-rag-with-permissions/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-vectors-001-rag-with-permissions", @@ -35794,49 +34149,57 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" + "passed": false, + "notes": "no embedding column" }, { "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + "passed": false, + "notes": "no index on embedding column" }, { "name": "index operator class matches the search operator", - "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + "passed": false, + "notes": "match_document_sections not found" }, { "name": "user A search returns only own sections, best match first", - "passed": true + "passed": false, + "notes": "Could not find the function public.match_document_sections(match_count, query_embedding) in the schema cache" }, { "name": "user B search returns only own sections, best match first", - "passed": true + "passed": false, + "notes": "Could not find the function public.match_document_sections(match_count, query_embedding) in the schema cache" }, { "name": "user A reads only own sections through the API", - "passed": true + "passed": false }, { "name": "user A reads only own documents through the API", - "passed": true + "passed": false } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ { "source": "web_search", - "query": "site:supabase.com docs gte-small embedding dimension 384 Supabase.ai.Session gte-small", + "query": "site:supabase.com/docs pgvector semantic search match_documents RLS Supabase ...", "pages": [] } ] @@ -35844,15 +34207,15 @@ "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-vectors-001-rag-with-permissions/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna/build-vectors-001-rag-with-permissions/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "deploy-database-001-prometheus-metrics", @@ -35873,79 +34236,56 @@ }, { "name": "configured the Supabase Metrics API scrape correctly", - "passed": false, - "judgeNotes": "Supabase basic_auth uses an inline rendered password instead of password_file, and docker-compose does not mount the password file via a volume or Compose secret." + "passed": true, + "judgeNotes": "HTTPS Supabase metrics scrape is correctly configured with the required path, Basic Auth password_file, matching read-only secret mount, project target, and preserved app job." }, { "name": "documented live deployment and verification steps", "passed": false, - "judgeNotes": "README lacks instructions to create the Secret API key, does not place/use a matching secret file, and provides no concrete verification through Prometheus targets, PromQL, or Grafana." + "judgeNotes": "Prometheus Basic Auth is misconfigured: the Supabase metrics endpoint requires username `service_role`, not literal `username`. This would prevent verification from succeeding despite otherwise adequate secret, restart, and target-check instructions." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"Prometheus metrics Supabase project observability endpoint\", limit: 10) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on CLICommandReference { title href content } } } }", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- files ---' && find observability -maxdepth 4 -type f -print -exec sed -n '1,240p' {} \\\\; && printf '%s\\\\n' '--- root metadata ---' && find . -maxdepth 2 -type f -not -path './.git/*' -not -path './observability/*' -print -exec sed -n '1,160p' {} \\\\; && printf '%s\\\\n' '--- cli ---' && supabase --version && printf '%s\\\\n' '--- changelog check ---' && curl -fsSL --max-time 15 https://supabase.com/changelog.md | head -80\"", "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/database/connection-management", - "title": "Connection management" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", - "title": "How to View Database Metrics" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports", - "title": "Reports" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 92242 + "resultChars": 4435 + }, + { + "source": "web_search", + "query": "site:supabase.com/docs metrics Prometheus Supabase project metrics endpoint observability ...", + "pages": [] } ] }, "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-database-001-prometheus-metrics/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna/deploy-database-001-prometheus-metrics/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "deploy-database-001-prometheus-metrics", @@ -35962,53 +34302,60 @@ "checks": [ { "name": "preserved existing app scrape job", - "passed": false + "passed": true }, { "name": "configured the Supabase Metrics API scrape correctly", - "passed": false, - "judgeNotes": "prometheus.yml is empty: no app job or Supabase scrape. docker-compose.yml does not mount a password_file or Compose secret and instead passes the API key via environment variable." + "passed": true, + "judgeNotes": "HTTPS Supabase Metrics API scrape is correctly configured with the required path, Basic Auth password_file, preserved app job, and matching read-only secret volume mount." }, { "name": "documented live deployment and verification steps", "passed": false, - "judgeNotes": "README lacks secret API key creation and secret-file placement steps, provides no concrete verification, and documents likely incorrect Basic Auth username (`username`)." + "judgeNotes": "README does not explain where/how to create the Secret API key, and the setup appears mismatched: it specifies an `sb_secret_...` key with an `analytics_logs_read` scope while Prometheus uses Basic Auth username `username`. It also lacks an explicit Compose restart/recreate step after configuration changes." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ { - "source": "web_search", - "query": "site:supabase.com/docs metrics Prometheus Supabase project metrics", - "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- files ---' && rg --files -g '\"'!node_modules'\"' -g '\"'!dist'\"' | sort && printf '%s\\\\n' '--- supabase references ---' && rg -n -i 'supabase|observab|metrics|prometheus|grafana|otel|opentelemetry|datadog|honeycomb|loki|tempo' -g '\"'!node_modules'\"' -g '\"'!dist'\"' . || true && printf '%s\\\\n' '--- changelog scan ---' && curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|observab|metric' | head -80 || true\"", "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted" + "url": "https://supabase.com/changelog.md" } - ] + ], + "resultChars": 74147 + }, + { + "source": "web_search", + "query": "site:supabase.com/docs metrics Prometheus project metrics endpoint ...", + "pages": [] } ] }, "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-database-001-prometheus-metrics/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna/deploy-database-001-prometheus-metrics/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "deploy-database-001-prometheus-metrics", @@ -36030,68 +34377,29 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": false, - "judgeNotes": "Fails: basic_auth uses an environment-rendered hardcoded password value, not password_file, and docker-compose.yml does not mount the required password file via a volume or Compose secret." + "judgeNotes": "Fails: basic_auth uses a hardcoded/environment-expanded password field instead of password_file, and docker-compose.yml does not mount the password file via a volume or Compose secret." }, { "name": "documented live deployment and verification steps", "passed": false, - "judgeNotes": "README lacks steps to create a Supabase Secret API key, place it in a matching secret file, and verify the scrape via Prometheus targets or PromQL/Grafana. It only documents environment variables and container recreation." + "judgeNotes": "README uses an environment variable instead of requiring and placing a matching secret file. Additionally, Prometheus does not automatically substitute ${SUPABASE_*} variables in prometheus.yml, so recreating the Compose services will not make this configuration live." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"project metrics prometheus observability\", limit: 10) {\n edges {\n node {\n title\n href\n content\n }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", - "title": "How to View Database Metrics" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", - "title": "Scrape a project's metrics" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", - "title": "Deleting data and dropping objects safely" - } - ], - "resultChars": 38153 - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/guides/monitoring-and-debugging/metrics Metrics API Supabase service_role sb_secret", - "pages": [] - }, { "source": "web_search", - "query": "site:supabase.com/docs/guides/monitoring-and-debugging/metrics \"service_role\" \"sb_secret\"", + "query": "Supabase Prometheus metrics endpoint project metrics official docs ...", "pages": [] } ] @@ -36099,15 +34407,15 @@ "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-database-001-prometheus-metrics/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna/deploy-database-001-prometheus-metrics/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "deploy-functions-001-edge-function-secrets", @@ -36134,7 +34442,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment via Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -36142,24 +34450,51 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- status ---' && git status --short && printf '%s\\\\n' '--- all project files ---' && rg --files -g '.*' -g '\"'!skills/**'\"' -g '\"'!data/**'\"' | sort && printf '%s\\\\n' '--- cli ---' && supabase --version && supabase functions --help && printf '%s\\\\n' '--- env names ---' && if test -f .env; then sed -E 's/\"'^([A-Za-z_][A-Za-z0-9_]*)=.*/'\"\\\\1=/' .env; else echo 'no .env'; fi && printf '%s\\\\n' '--- config ---' && if test -f supabase/config.toml; then sed -n '1,240p' supabase/config.toml; fi && printf '%s\\\\n' '--- changelog ---' && curl -fsSL https://supabase.com/changelog.md | rg -i -m 20 'edge function|function|secret|cli' || true\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 157 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- all project files ---'; rg --files -g '.*' -g '\"'!skills/**'\"' -g '\"'!data/**'\"' | sort; printf '%s\\\\n' '--- cli ---'; supabase --version; supabase functions --help; printf '%s\\\\n' '--- env names ---'; if test -f .env; then sed -E 's/\"'^([A-Za-z_][A-Za-z0-9_]*)=.*/'\"\\\\1=/' .env; else echo 'no .env'; fi; printf '%s\\\\n' '--- config ---'; if test -f supabase/config.toml; then sed -n '1,240p' supabase/config.toml; fi; printf '%s\\\\n' '--- changelog ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -m 20 'edge function|function|secret|cli' || true\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 3786 + } + ] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-functions-001-edge-function-secrets/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna/deploy-functions-001-edge-function-secrets/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "deploy-functions-001-edge-function-secrets", @@ -36186,7 +34521,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment via Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -36194,30 +34529,43 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] }, "docs": { "calls": [ { - "source": "web_search", - "query": "evalshostedprojectxy supabase weather function", - "pages": [] + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | head -120 && printf '\\\\n--- edge docs ---\\\\n' && curl -fsSL https://supabase.com/docs/guides/functions/quickstart.md | head -100\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart.md" + } + ], + "resultChars": 4243 } ] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-functions-001-edge-function-secrets/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna/deploy-functions-001-edge-function-secrets/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "deploy-functions-001-edge-function-secrets", @@ -36244,7 +34592,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get." + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -36252,54 +34600,41 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ { - "source": "web_search", - "query": "site:supabase.com/docs edge function URL project ref functions supabase.co project not specified", - "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/functions/quickstart", + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- project metadata ---' && for f in supabase/.temp/project-ref supabase/.temp/cli-latest; do echo \\\"\"'$f:\"; sed -n '\"'1,3p' \\\"\"'$f\"; done && printf '\"'%s\\\\n' '--- cli functions help ---' && supabase functions --help && printf '%s\\\\n' '--- cli secrets help ---' && supabase secrets set --help && printf '%s\\\\n' '--- docs ---' && curl -fsSL https://supabase.com/docs/guides/functions/secrets.md | sed -n '1,220p'\"", "pages": [ { - "url": "https://supabase.com/docs/guides/functions/quickstart" + "url": "https://supabase.com/docs/guides/functions/secrets.md" } - ] - }, - { - "source": "web_search", - "query": "evalshostedprojectxy.supabase.co", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs \"Project not specified\" \"functions.supabase.co\"", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs edge function logs cli", - "pages": [] + ], + "resultChars": 5106 } ] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-functions-001-edge-function-secrets/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna/deploy-functions-001-edge-function-secrets/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "deploy-self-hosting-001-docker-compose", @@ -36334,14 +34669,20 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ { "source": "web_search", - "query": "site:github.com supabase docker compose self-hosted docker-compose.yml official", + "query": "site:supabase.com/docs/guides/self-hosting/docker Supabase self-hosting Docker official ...", "pages": [] } ] @@ -36349,15 +34690,15 @@ "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-self-hosting-001-docker-compose/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna/deploy-self-hosting-001-docker-compose/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "deploy-self-hosting-001-docker-compose", @@ -36392,30 +34733,29 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [ - { - "source": "web_search", - "query": "site:supabase.com/docs self-hosting docker compose Supabase official", - "pages": [] - } - ] + "calls": [] }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-self-hosting-001-docker-compose/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna/deploy-self-hosting-001-docker-compose/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "deploy-self-hosting-001-docker-compose", @@ -36450,113 +34790,203 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"self-hosting docker compose local development secrets JWT anon service_role site_url dashboard\", limit: 10) {\n nodes {\n __typename\n ... on Guide {\n title\n href\n content\n }\n ... on CLICommandReference {\n title\n href\n content\n }\n ... on TroubleshootingGuide {\n title\n href\n content\n }\n }\n }\n}", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"set -o pipefail\nprintf '%s\\\\n' '--- supabase cli ---'\nsupabase --version\nprintf '%s\\\\n' '--- upstream repo listing ---'\ncurl -fsSL https://api.github.com/repos/supabase/supabase-docker/contents/ | sed -n '1,220p'\nprintf '%s\\\\n' '--- changelog head ---'\ncurl -fsSL https://supabase.com/changelog.md | sed -n '1,100p'\nprintf '%s\\\\n' '--- self-hosting docs ---'\ncurl -fsSL https://supabase.com/docs/guides/self-hosting/docker | sed -n '1,100p'\"", "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" + "url": "https://supabase.com/changelog.md" }, { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - }, + "url": "https://supabase.com/docs/guides/self-hosting/docker" + } + ], + "resultChars": 113364 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"set -o pipefail\ncurl -fsSL https://supabase.com/docs/guides/self-hosting/docker.md | sed -n '1,260p'\nprintf '\\\\n--- repo candidates ---\\\\n'\nfor u in https://api.github.com/repos/supabase-community/supabase-docker/contents/ https://api.github.com/repos/supabase/supabase/contents/docker; do echo \\\"\"'$u\"; curl -fsSL \"$u\" | sed -n '\"'1,80p'; done\"", + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" - }, + "url": "https://supabase.com/docs/guides/self-hosting/docker.md" + } + ], + "resultChars": 16079 + } + ] + }, + "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", + "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-5.6-luna/deploy-self-hosting-001-docker-compose/run-3/result.json" + }, + { + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", + "product": [ + "auth" + ], + "topic": [ + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "victim session active before delete-account", + "passed": true + }, + { + "name": "delete_account flow ran for the victim", + "passed": true + }, + { + "name": "delete-account revokes the user's sessions", + "passed": true, + "notes": "sessions left: 0" + }, + { + "name": "deleted user's refresh token is rejected", + "passed": true + }, + { + "name": "deleted user cannot sign back in", + "passed": true + }, + { + "name": "other users keep their sessions and access", + "passed": true + }, + { + "name": "diagnosed and explained session revocation", + "passed": true, + "judgeNotes": "Correctly diagnoses the soft delete, implements Auth-user/session removal plus RLS defense against stale JWTs, consistently notes JWT validity until expiry while closing the data path, and accurately distinguishes publishable from server-only secret keys." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query Search(\"delete user invalidate sessions access tokens Supabase Auth\": String!) { searchDocs(query: $q, limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content } } } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "query Search(\"publishable keys secret keys anon service_role RLS Supabase\": String!) { searchDocs(query: $q, limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content } } } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"delete user invalidate sessions access tokens Supabase Auth\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content } } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates", - "title": "Custom Email Templates" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", + "title": "OAuth 2.1 Flows" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/auth/users", + "title": "Users" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" + "url": "https://supabase.com/docs/guides/realtime/protocol", + "title": "Realtime Protocol" } ], - "resultChars": 165210 + "resultChars": 106884 }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"Self-hosting with Docker quick start Linux generate-keys.sh docker-compose.yml .env.example API_EXTERNAL_URL DB_PASSWORD JWT_SECRET ANON_KEY SERVICE_ROLE_KEY\", limit: 10) {\n nodes {\n __typename\n ... on Guide { title href content }\n ... on CLICommandReference { title href content }\n }\n }\n}", + "query": "query { searchDocs(query: \"publishable keys secret keys anon service_role RLS Supabase\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" + "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", + "title": "Creating a Supabase client for SSR" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", - "title": "Configure Reverse Proxy and HTTPS" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" } ], - "resultChars": 165410 + "resultChars": 83013 + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] } ] }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "run": 3, - "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-self-hosting-001-docker-compose/run-3/result.json" + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "run": 1, + "sourcePath": "codex-gpt-5.6-luna/investigate-auth-001-deleted-user-access/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "investigate-auth-001-deleted-user-access", @@ -36591,7 +35021,8 @@ }, { "name": "deleted user cannot sign back in", - "passed": true + "passed": false, + "notes": "deleted account can still sign in" }, { "name": "other users keep their sessions and access", @@ -36600,24 +35031,35 @@ { "name": "diagnosed and explained session revocation", "passed": false, - "judgeNotes": "The fix and key guidance are correct, but the JWT explanation is incomplete: it does not explicitly state that stateless/local validation such as getClaims() or signature/expiry middleware will continue accepting the deleted user’s access token until expiry." + "judgeNotes": "The fix deletes sessions but not the auth user or identities, so the user can sign in again. It also omits that the old JWT remains valid until expiry for purely local validation such as getClaims() or signature/expiry middleware." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"delete user revoke sessions refresh tokens access token auth admin deleteUser Supabase\", limit: 5) {\n nodes {\n __typename\n ... on Guide { title href content }\n ... on TroubleshootingGuide { title href content }\n ... on ClientLibraryFunctionReference { title href methodName language content }\n ... on ManagementApiReference { title href content }\n }\n }\n}", + "query": "query { searchDocs(query: \"delete user existing access tokens revoke sessions auth.sessions JWT invalidation\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content } } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, { "url": "https://supabase.com/docs/guides/auth/sessions", "title": "User sessions" @@ -36627,77 +35069,66 @@ "title": "OAuth 2.1 Flows" }, { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", - "title": "Build a Supabase Integration" + "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", + "title": "Multiple SSO Providers" } ], - "resultChars": 68972 + "resultChars": 107892 }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"banned_until auth.users deleted_at revoke refresh tokens immediately sign out delete user session_id current_user in RLS\", limit: 10) {\n nodes {\n __typename\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href methodName language content }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", + "query": "query { searchDocs(query: \"publishable keys secret keys frontend RLS anon service_role\", limit: 6) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/management/delete-objects", - "title": "Delete Objects" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" }, { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" }, { - "url": "https://supabase.com/docs/guides/realtime/authorization", - "title": "Realtime Authorization" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0012_auth_allow_anonymous_sign_ins", - "title": "Database Advisor: Lint 0012_auth_allow_anonymous_sign_ins" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" } ], - "resultChars": 91594 + "resultChars": 104127 } ] }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "run": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-auth-001-deleted-user-access/run-1/result.json" + "run": 2, + "sourcePath": "codex-gpt-5.6-luna/investigate-auth-001-deleted-user-access/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "investigate-auth-001-deleted-user-access", @@ -36711,7 +35142,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "victim session active before delete-account", @@ -36740,387 +35171,25 @@ }, { "name": "diagnosed and explained session revocation", - "passed": false, - "judgeNotes": "The fix and key guidance are correct, but the JWT explanation is incomplete/inconsistent. It says the only residual window is an in-flight request, without clarifying that getClaims() or custom local signature/expiry validation will continue accepting the stateless JWT until exp. It also misleadingly suggests session revocation itself makes access tokens invalid." + "passed": true, + "judgeNotes": "Correctly diagnoses the soft-delete-only RPC, implements Auth user/session deletion, closes current Data API access via active-profile RLS while acknowledging JWT validity until expiry, and accurately distinguishes publishable frontend keys from RLS-bypassing server-only secret keys." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query {\n searchDocs(query: \"delete user revoke sessions access token still valid after deleting auth.users Supabase\", limit: 5) {\n nodes {\n title\n href\n content\n ... on Guide {\n subsections {\n nodes {\n title\n href\n content\n }\n }\n }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#how-to-create-mint-jwts-if-access-to-the-private-key-or-shared-secret-is-not-possible", - "title": "How to create (mint) JWTs if access to the private key or shared secret is not possible?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#why-is-a-5-minute-wait-imposed-when-changing-signing-key-states", - "title": "Why is a 5 minute wait imposed when changing signing key states?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#why-is-deleting-the-legacy-jwt-secret-disallowed", - "title": "Why is deleting the legacy JWT secret disallowed?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#why-does-revoking-the-legacy-jwt-secret-require-disabling-of-anon-and-service_role-api-keys", - "title": "Why does revoking the legacy JWT secret require disabling of anon and service_role API keys?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#using-jwt-based-anon-key-in-a-mobile-desktop-or-cli-application-and-need-to-rotate-a-service_role-jwt-secret", - "title": "Using JWT-based anon key in a mobile, desktop, or CLI application and need to rotate a service_role JWT secret?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#overview", - "title": "Overview" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#benefits-of-the-signing-keys-system", - "title": "Benefits of the signing keys system" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#getting-started", - "title": "Getting started" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#rotating-and-revoking-keys", - "title": "Rotating and revoking keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#lifetime-of-a-signing-key", - "title": "Lifetime of a signing key" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#public-key-discovery-and-caching", - "title": "Public key discovery and caching" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#choosing-the-right-signing-algorithm", - "title": "Choosing the right signing algorithm" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#frequently-asked-questions", - "title": "Frequently asked questions" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#why-is-it-not-possible-to-extract-the-private-key-or-shared-secret-from-supabase", - "title": "Why is it not possible to extract the private key or shared secret from Supabase?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#why-are-anon-and-service_role-jwt-based-keys-no-longer-recommended", - "title": "Why are anon and service_role JWT-based keys no longer recommended?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#can-you-still-use-an-old-anon-and-service-role-api-keys-after-enabling-the-publishable-and-secret-keys", - "title": "Can you still use an old anon and service-role API keys after enabling the publishable and secret keys?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#how-to-deactivate-the-anon-and-service_role-jwt-based-api-keys-after-moving-to-publishable-and-secret-keys", - "title": "How to deactivate the anon and service_role JWT-based API keys after moving to publishable and secret keys?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#how-are-publishable-and-secret-keys-implemented-on-the-hosted-platform", - "title": "How are publishable and secret keys implemented on the hosted platform?" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data#accessing-user-data-via-api", - "title": "Accessing user data via API" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data#adding-and-retrieving-user-metadata", - "title": "Adding and retrieving user metadata" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data#deleting-users", - "title": "Deleting users" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data#exporting-users", - "title": "Exporting users" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#supported-grant-types", - "title": "Supported grant types" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#authorization-code-flow-with-pkce", - "title": "Authorization code flow with PKCE" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#how-it-works", - "title": "How it works" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#flow-diagram", - "title": "Flow diagram" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#step-1-generate-pkce-parameters", - "title": "Step 1: Generate PKCE parameters" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#step-2-authorization-request", - "title": "Step 2: Authorization request" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#required-parameters", - "title": "Required parameters" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#optional-parameters", - "title": "Optional parameters" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#step-3-user-authentication-and-consent", - "title": "Step 3: User authentication and consent" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#step-4-authorization-code-issued", - "title": "Step 4: Authorization code issued" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#step-5-token-exchange", - "title": "Step 5: Token exchange" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#public-clients-token_endpoint_auth_method-none", - "title": "Public clients (token_endpoint_auth_method: none)" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#confidential-clients-token_endpoint_auth_method-client_secret_basic", - "title": "Confidential clients (token_endpoint_auth_method: client_secret_basic)" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#confidential-clients-token_endpoint_auth_method-client_secret_post", - "title": "Confidential clients (token_endpoint_auth_method: client_secret_post)" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#example-in-javascript", - "title": "Example in JavaScript" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#step-6-token-response", - "title": "Step 6: Token response" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#access-token-structure", - "title": "Access token structure" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#oauth-specific-claims", - "title": "OAuth-specific claims" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#available-scopes", - "title": "Available scopes" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#refresh-token-flow", - "title": "Refresh token flow" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#when-to-refresh", - "title": "When to refresh" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#refresh-request", - "title": "Refresh request" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#public-clients-token_endpoint_auth_method-none-1", - "title": "Public clients (token_endpoint_auth_method: none)" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#confidential-clients-token_endpoint_auth_method-client_secret_basic-1", - "title": "Confidential clients (token_endpoint_auth_method: client_secret_basic)" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#confidential-clients-token_endpoint_auth_method-client_secret_post-1", - "title": "Confidential clients (token_endpoint_auth_method: client_secret_post)" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#example-in-javascript-1", - "title": "Example in JavaScript" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#refresh-response", - "title": "Refresh response" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#openid-connect-oidc", - "title": "OpenID Connect (OIDC)" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#id-tokens", - "title": "ID tokens" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#example-id-token", - "title": "Example ID token" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#standard-oidc-claims", - "title": "Standard OIDC claims" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#userinfo-endpoint", - "title": "UserInfo endpoint" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#oidc-discovery", - "title": "OIDC discovery" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#token-validation", - "title": "Token validation" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#jwks-endpoint", - "title": "JWKS endpoint" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#validating-tokens", - "title": "Validating tokens" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#what-to-validate", - "title": "What to validate" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#managing-user-grants", - "title": "Managing user grants" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#viewing-authorized-applications", - "title": "Viewing authorized applications" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#revoking-access", - "title": "Revoking access" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#next-steps", - "title": "Next steps" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions#what-is-a-session", - "title": "What is a session?" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions#access-token-jwt-claims", - "title": "Access token (JWT) claims" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions#initiating-a-session", - "title": "Initiating a session" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions#limiting-session-lifetime-and-number-of-allowed-sessions-per-user", - "title": "Limiting session lifetime and number of allowed sessions per user" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions#frequently-asked-questions", - "title": "Frequently asked questions" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions#what-are-recommended-values-for-access-token-jwt-expiration", - "title": "What are recommended values for access token (JWT) expiration?" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions#what-is-refresh-token-reuse-detection-and-what-does-it-protect-from", - "title": "What is refresh token reuse detection and what does it protect from?" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions#what-are-the-benefits-of-using-access-and-refresh-tokens-instead-of-traditional-sessions", - "title": "What are the benefits of using access and refresh tokens instead of traditional sessions?" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions#how-to-ensure-an-access-token-jwt-cannot-be-used-after-a-user-signs-out", - "title": "How to ensure an access token (JWT) cannot be used after a user signs out" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions#using-http-only-cookies-to-store-access-and-refresh-tokens", - "title": "Using HTTP-only cookies to store access and refresh tokens" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso", - "title": "Enable SSO for Your Organization" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso#supported-providers", - "title": "Supported providers" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso#choosing-your-login-flow", - "title": "Choosing your login flow" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso#idp-initiated-login-recommended", - "title": "IdP-initiated login (recommended)" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso#sp-initiated-login", - "title": "SP-initiated login" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso#need-help-choosing", - "title": "Need help choosing?" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso#key-configuration-options", - "title": "Key configuration options" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso#how-sso-works-in-supabase", - "title": "How SSO works in Supabase" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso#enabling-sso-for-an-organization", - "title": "Enabling SSO for an organization" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso#disabling-sso-for-an-organization", - "title": "Disabling SSO for an organization" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso#removing-an-individual-sso-users-access", - "title": "Removing an individual SSO user's access" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso#testing-and-best-practices", - "title": "Testing and best practices" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso#advanced-scenarios", - "title": "Advanced scenarios" - } - ], - "resultChars": 239695 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"Supabase delete user revoke refresh tokens sign out all sessions access token expiry remains valid\", limit: 10) {\n nodes {\n title\n href\n content\n }\n }\n}", + "query": "{ searchDocs(query: \"Supabase delete user existing access tokens sessions invalidate access token JWT\", limit: 5) { nodes { ... on Guide { title href content } ... on ManagementApiReference { title href content } ... on ClientLibraryFunctionReference { title href content } } } }", "hasContent": true, "pages": [ { @@ -37128,47 +35197,27 @@ "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/auth/jwts", + "title": "JSON Web Token (JWT)" }, { "url": "https://supabase.com/docs/guides/auth/managing-user-data", "title": "User Management" }, - { - "url": "https://supabase.com/docs/guides/auth/signout", - "title": "Signing out" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" - }, { "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", "title": "OAuth 2.1 Flows" }, { - "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", - "title": "Build a Supabase Integration" - }, - { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server", - "title": "OAuth 2.1 Server" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" } ], - "resultChars": 136202 + "resultChars": 102612 }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"publishable secret API keys frontend backend RLS Supabase\", limit: 10) {\n nodes {\n title\n href\n content\n }\n }\n}", + "query": "{ searchDocs(query: \"Supabase publishable keys secret keys frontend RLS anon service_role\", limit: 5) { nodes { ... on Guide { title href content } ... on ManagementApiReference { title href content } ... on ClientLibraryFunctionReference { title href content } } } }", "hasContent": true, "pages": [ { @@ -37180,173 +35229,34 @@ "title": "Build a User Management App with RedwoodJS" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "title": "API keys" }, { "url": "https://supabase.com/docs/guides/auth/signing-keys", "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-react", - "title": "Build a User Management App with Ionic React" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", - "title": "Build a User Management App with React" - }, - { - "url": "https://supabase.com/docs/guides/api/creating-routes", - "title": "Creating API Routes" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - } - ], - "resultChars": 210411 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"delete user supabase auth.admin.deleteUser revoke sessions\", limit: 10) {\n nodes {\n title\n href\n content\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso", - "title": "Enable SSO for Your Organization" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/signout", - "title": "Signing out" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", - "title": "deleteUser()" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-admin-signout", - "title": "signOut()" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" } ], - "resultChars": 82657 - } - ] - }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "run": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-auth-001-deleted-user-access/run-2/result.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", - "product": [ - "auth" - ], - "topic": [ - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": false, - "checks": [ - { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true - }, - { - "name": "delete-account revokes the user's sessions", - "passed": false, - "notes": "sessions left: 1" - }, - { - "name": "deleted user's refresh token is rejected", - "passed": false, - "notes": "refresh token still produces a session" - }, - { - "name": "deleted user cannot sign back in", - "passed": false, - "notes": "deleted account can still sign in" - }, - { - "name": "other users keep their sessions and access", - "passed": true - }, - { - "name": "diagnosed and explained session revocation", - "passed": false, - "judgeNotes": "The fix remains a soft delete and never deletes the auth user or revokes sessions/refresh tokens. It also omits that existing JWT access tokens remain valid for local validation until expiry, and incorrectly suggests RLS can apply to secret-key requests rather than clearly stating secret keys are server-only and bypass RLS." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "web_search", - "query": "site:supabase.com docs publishable secret keys anon service_role frontend RLS", - "pages": [] + "resultChars": 91518 } ] }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-auth-001-deleted-user-access/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna/investigate-auth-001-deleted-user-access/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "investigate-realtime-001-subscribed-no-events", @@ -37386,28 +35296,74 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly diagnosed missing publication membership and added only public.orders to supabase_realtime, preserving courier_locations, RLS, and policies." + "judgeNotes": "Correctly diagnosed orders as missing from supabase_realtime, added only public.orders, and preserved courier_locations, RLS, and existing policies." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | rg -n -i \"breaking-change|realtime|postgres_changes\" | head -40'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 7890 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Postgres Changes realtime publication supabase_realtime table INSERT\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ClientLibraryFunctionReference { title href content methodName language } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", + "title": "Subscribing to Database Changes" + }, + { + "url": "https://supabase.com/docs/guides/realtime/architecture", + "title": "Realtime Architecture" + }, + { + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" + }, + { + "url": "https://supabase.com/docs/guides/realtime/benchmarks", + "title": "Benchmarks" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/realtime-postgres-changes-troubleshooting", + "title": "Realtime: Postgres Changes Troubleshooting" + } + ], + "resultChars": 82022 + } + ] }, "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-realtime-001-subscribed-no-events/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna/investigate-realtime-001-subscribed-no-events/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "investigate-realtime-001-subscribed-no-events", @@ -37447,28 +35403,63 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified and fixed the missing orders table in supabase_realtime while preserving courier_locations and RLS." + "judgeNotes": "Correctly identified and fixed the missing orders publication membership while preserving courier_locations, RLS, and existing policies." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Realtime postgres_changes enable table publication supabase_realtime\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", + "title": "Subscribing to Database Changes" + }, + { + "url": "https://supabase.com/docs/guides/realtime/benchmarks", + "title": "Benchmarks" + }, + { + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/realtime-postgres-changes-troubleshooting", + "title": "Realtime: Postgres Changes Troubleshooting" + }, + { + "url": "https://supabase.com/docs/guides/realtime", + "title": "Realtime" + } + ], + "resultChars": 80760 + } + ] }, "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-realtime-001-subscribed-no-events/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna/investigate-realtime-001-subscribed-no-events/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "investigate-realtime-001-subscribed-no-events", @@ -37508,28 +35499,91 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified orders missing from supabase_realtime, added only public.orders, verified both feeds remain published, and preserved RLS and policies." + "judgeNotes": "Correctly diagnosed missing publication membership, added only public.orders, preserved courier_locations and RLS/policies, and verified both tables remain published." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Realtime postgres_changes tables publication supabase_realtime\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", + "title": "Subscribing to Database Changes" + }, + { + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" + }, + { + "url": "https://supabase.com/docs/guides/realtime/benchmarks", + "title": "Benchmarks" + }, + { + "url": "https://supabase.com/docs/guides/realtime/architecture", + "title": "Realtime Architecture" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/realtime-postgres-changes-troubleshooting", + "title": "Realtime: Postgres Changes Troubleshooting" + } + ], + "resultChars": 82022 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Realtime postgres_changes add table to supabase_realtime publication\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", + "title": "Subscribing to Database Changes" + }, + { + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/realtime-postgres-changes-troubleshooting", + "title": "Realtime: Postgres Changes Troubleshooting" + }, + { + "url": "https://supabase.com/docs/guides/realtime/benchmarks", + "title": "Benchmarks" + }, + { + "url": "https://supabase.com/docs/guides/realtime/getting_started", + "title": "Getting Started with Realtime" + } + ], + "resultChars": 97269 + } + ] }, "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-realtime-001-subscribed-no-events/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna/investigate-realtime-001-subscribed-no-events/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "investigate-reliability-003-edge-function-5xx-correlation", @@ -37542,27 +35596,33 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform and the recurring eight HTTP 503 failures from 07:00Z through 12:00Z on 2026-04-28." + "judgeNotes": "Identified image-transform as the affected function and described recurring gateway-level 503s across the morning of 2026-04-28, listing 7 of the 8 failures." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "It correctly attributes the 503s to the gateway/platform layer using missing invocation/runtime logs, but fails the rubric by recommending redeploying/restarting the functions as remediation." + "passed": true, + "judgeNotes": "Attributes image-transform 503s to the gateway/platform layer and supports this with missing deployment/version metadata, quick recovery of nearby requests, and distinction from avatar-upload’s execution-level 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Provides multiple concrete next steps: check Supabase Edge platform health, redeploy affected functions, add retry/backoff and alerting, and compile request IDs for escalation." + "judgeNotes": "The assistant provided multiple concrete actions, including checking Edge Function platform incidents for the specific window, reviewing the affected deployment and dependency, adding retries, and capturing correlation IDs." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [] @@ -37570,15 +35630,15 @@ "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-reliability-003-edge-function-5xx-correlation/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna/investigate-reliability-003-edge-function-5xx-correlation/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "investigate-reliability-003-edge-function-5xx-correlation", @@ -37591,27 +35651,33 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform as the main affected function and explicitly listed all eight recurring HTTP 503 failures from 07:00Z through 12:00Z on 2026-04-28." + "judgeNotes": "Identified image-transform as the affected function and explicitly reported 8 recurring HTTP 503 failures across 07:00–12:00Z on 2026-04-28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "Correctly attributes the 503s to the gateway/platform and cites missing function invocation logs plus nearby successes. However, it recommends rolling back or redeploying image-transform, which the rubric explicitly treats as a failure." + "passed": true, + "judgeNotes": "Attributes the recurring 503s to the Edge gateway/routing layer before invocation, supported by missing deployment IDs, durations, and execution logs while nearby executions succeeded on the same deployment. It also distinguishes the separate function-level avatar-upload 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Provides multiple concrete actions: inspect deployment/routing changes, roll back or redeploy, open a Supabase support case with request IDs, and add retry/fallback handling." + "judgeNotes": "Provides concrete next steps, including opening a Supabase support ticket with project, region, timestamps, and gateway request IDs to investigate Edge gateway/runtime routing." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [] @@ -37619,15 +35685,15 @@ "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-reliability-003-edge-function-5xx-correlation/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna/investigate-reliability-003-edge-function-5xx-correlation/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "investigate-reliability-003-edge-function-5xx-correlation", @@ -37645,22 +35711,28 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified `image-transform` as affected and recognized repeated HTTP 503s throughout the relevant morning window, covering 7 of the 8 gateway failures." + "judgeNotes": "Identified image-transform and all eight recurring HTTP 503 gateway failures from 07:00Z through 12:00Z on 2026-04-28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Attributes image-transform 503s to the gateway/ingress layer, supported by their presence only in edge logs with no matching runtime 503s while nearby function executions succeeded. It also correctly distinguishes avatar-upload’s matched function-level 500." + "judgeNotes": "Attributes the recurring image-transform 503s to the Edge gateway/platform layer, supported by their presence only in edge logs with no corresponding function executions. It also correctly distinguishes avatar-upload’s application-level 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Provides multiple concrete next steps, including treating the 503s as a gateway incident, checking Supabase infrastructure health for the exact time window, and investigating the correlated avatar-upload request and code." + "judgeNotes": "Provides concrete actions, including checking Edge Function platform incidents using specific gateway request IDs and time windows, adding retries, and separately investigating the avatar-upload error." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [] @@ -37668,15 +35740,15 @@ "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-reliability-003-edge-function-5xx-correlation/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna/investigate-reliability-003-edge-function-5xx-correlation/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-dataapi-001-empty-results", @@ -37721,12 +35793,18 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and created authenticated owner-scoped SELECT and INSERT policies using auth.uid()." + "judgeNotes": "Correctly diagnosed deny-all RLS with zero policies, added authenticated owner-scoped SELECT and INSERT policies using auth.uid(), and kept RLS enabled." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [] @@ -37734,15 +35812,15 @@ "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-dataapi-001-empty-results/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna/resolve-dataapi-001-empty-results/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-dataapi-001-empty-results", @@ -37787,28 +35865,55 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and added authenticated owner-scoped SELECT and INSERT policies using auth.uid() with USING and WITH CHECK." + "judgeNotes": "Correctly diagnosed deny-all RLS with no policies and added authenticated-only SELECT USING and INSERT WITH CHECK policies scoped to auth.uid(), while keeping RLS enabled." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Supabase Row Level Security policies auth.uid select insert authenticated\", limit: 3) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/auth/users", + "title": "Users" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/storage-error-403-forbidden-new-row-violates-row-level-security-policy-on-upload-a94384", + "title": "Storage error: 403 Forbidden: 'new row violates row-level security policy' on upload" + } + ], + "resultChars": 40256 + } + ] }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-dataapi-001-empty-results/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna/resolve-dataapi-001-empty-results/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-dataapi-001-empty-results", @@ -37824,57 +35929,56 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true - }, - { - "name": "user B cannot insert a bookmark as user A", - "passed": true - }, - { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and created authenticated owner-scoped SELECT and INSERT policies using auth.uid() with USING and WITH CHECK." + "name": "created auth sessions", + "passed": false, + "notes": "Internal server error" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Row Level Security authenticated select insert policy auth.uid Supabase\", limit: 3) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + } + ], + "resultChars": 32001 + } + ] }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-dataapi-001-empty-results/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna/resolve-dataapi-001-empty-results/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-database-001-migration-history-mismatch", @@ -37916,49 +36020,46 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": false, - "judgeNotes": "No successful remote `supabase db push` occurred, and no `supabase migration repair` or `supabase db pull` reconciled history. Only local `supabase db reset --local` commands applied migrations locally. Management API workarounds were attempted but failed; no recorded remote mutation succeeded." + "judgeNotes": "`supabase db push` (#23) only reported the database was up to date; it did not apply a migration. The other push (#21) was dry-run. No `supabase migration repair`, `supabase db pull`, or push-based history reconciliation occurred. No prohibited mutation workaround was shown." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ { - "source": "web_search", - "query": "site:supabase.com/docs database/query Supabase management API", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/reference/api database/query write query management api", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/reference/api database/postgres management api postgres config", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/reference/api upsert migration management api body version sql", - "pages": [] + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -120'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 6742 } ] }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-database-001-migration-history-mismatch/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna/resolve-database-001-migration-history-mismatch/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-database-001-migration-history-mismatch", @@ -37980,11 +36081,13 @@ }, { "name": "migration 20240220000000 is recorded in the remote history", - "passed": true + "passed": false, + "notes": "remote history versions: [\"20240101000000\",\"20240115000000\"]" }, { "name": "remote migration history matches local migration files", - "passed": true + "passed": false, + "notes": "remote-only versions still unreconciled: [\"20240115000000\"] (remote: [\"20240101000000\",\"20240115000000\"], local: [\"20240101000000\",\"20240220000000\"])" }, { "name": "local migrations are a valid reconciled sequence", @@ -37998,28 +36101,45 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": false, - "judgeNotes": "History was reconciled via `supabase migration repair --status reverted 20240115000000`. However, the avatar migration was only marked applied via repair; the subsequent `supabase db push` said “Remote database is up to date” and did not apply it. The earlier applying output was from local `db reset`, not a remote push." + "judgeNotes": "`supabase db push --db-url ... --yes` was invoked, but the recorded output does not show “Applying migration” or “Finished supabase db push.” No `supabase migration repair`, `supabase db pull`, or local bio migration addition is recorded to reconcile the orphan history. The visible `psql` usage is read-only; no explicit workaround is shown." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- files ---'; sed -n '1,240p' supabase/config.toml; for f in supabase/migrations/*.sql; do echo \\\"### \"'$f\"; sed -n '\"'1,240p' \\\"\"'$f\"; done; printf '\"'%s\\\\n' '--- cli ---'; supabase --version; supabase db --help; printf '%s\\\\n' '--- changelog check ---'; curl -fsSL https://supabase.com/changelog.md | head -80\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 6184 + } + ] }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-database-001-migration-history-mismatch/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna/resolve-database-001-migration-history-mismatch/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-database-001-migration-history-mismatch", @@ -38032,27 +36152,23 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "the avatar_url column is applied on the hosted profiles table", - "passed": false, - "notes": "avatar_url not found on public.profiles" + "passed": true }, { "name": "migration 20240220000000 is recorded in the remote history", - "passed": false, - "notes": "remote history versions: [\"20240101000000\",\"20240115000000\"]" + "passed": true }, { "name": "remote migration history matches local migration files", - "passed": false, - "notes": "remote-only versions still unreconciled: [\"20240115000000\"] (remote: [\"20240101000000\",\"20240115000000\"], local: [\"20240101000000\",\"20240220000000\"])" + "passed": true }, { "name": "local migrations are a valid reconciled sequence", - "passed": false, - "notes": "expected 20240101000000_create_profiles → → 20240220000000_add_avatar_url, strictly ascending; got [20240101000000_create_profiles.sql, 20240220000000_add_avatar_url.sql]" + "passed": true }, { "name": "production profile data is intact (not reset)", @@ -38060,29 +36176,45 @@ }, { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": false, - "judgeNotes": "No successful remote `supabase db push` occurred; all remote attempts failed, while `db push --local` and `db reset --local` do not count. No `supabase migration repair`, `db pull`, or successful push reconciled the orphan bio migration. Local SQL edits adding `IF NOT EXISTS` were not a valid remote-history reconciliation." + "passed": true, + "judgeNotes": "#20 used a real `supabase db push --db-url \"$remote_url\"` to apply `20240220000000_add_avatar_url.sql`; the remote columns then included `avatar_url`. History was reconciled by adding local `20240115000000_add_profile_bio.sql` and pushing. Only read-only psql inspection was used; no prohibited workaround appeared." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -L --max-time 15 -s https://supabase.com/changelog.md | sed -n '1,120p'\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 6693 + } + ] }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-database-001-migration-history-mismatch/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna/resolve-database-001-migration-history-mismatch/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-performance-001-slow-query-cpu-spike", @@ -38096,7 +36228,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "inspected pg_stat_statements for query performance", @@ -38104,7 +36236,7 @@ }, { "name": "ran EXPLAIN on the expensive query", - "passed": true + "passed": false }, { "name": "created index covering user_id and created_at", @@ -38113,7 +36245,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -38121,8 +36253,14 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [] @@ -38130,15 +36268,15 @@ "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-performance-001-slow-query-cpu-spike/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna/resolve-performance-001-slow-query-cpu-spike/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-performance-001-slow-query-cpu-spike", @@ -38177,8 +36315,14 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [] @@ -38186,15 +36330,15 @@ "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-performance-001-slow-query-cpu-spike/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna/resolve-performance-001-slow-query-cpu-spike/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-performance-001-slow-query-cpu-spike", @@ -38208,7 +36352,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "inspected pg_stat_statements for query performance", @@ -38216,7 +36360,7 @@ }, { "name": "ran EXPLAIN on the expensive query", - "passed": false + "passed": true }, { "name": "created index covering user_id and created_at", @@ -38233,24 +36377,58 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Postgres query optimization indexes EXPLAIN ANALYZE Supabase\", limit: 3) { nodes { ... on Guide { title href content } ... on ManagementApiReference { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + } + ], + "resultChars": 8794 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL --max-time 10 https://supabase.com/changelog.md | head -40'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 2202 + } + ] }, "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-performance-001-slow-query-cpu-spike/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna/resolve-performance-001-slow-query-cpu-spike/run-3/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-security-002-rls-cross-tenant-leak", @@ -38305,24 +36483,60 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"row level security policy auth.uid memberships multi tenant\", limit: 3) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + } + ], + "resultChars": 54073 + } + ] }, "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-security-002-rls-cross-tenant-leak/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna/resolve-security-002-rls-cross-tenant-leak/run-1/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-security-002-rls-cross-tenant-leak", @@ -38377,24 +36591,62 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -80'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 4435 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Row Level Security policies membership organization tenant isolation\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + } + ], + "resultChars": 49074 + } + ] }, "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-security-002-rls-cross-tenant-leak/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna/resolve-security-002-rls-cross-tenant-leak/run-2/result.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6-luna", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-security-002-rls-cross-tenant-leak", @@ -38449,24 +36701,61 @@ } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- all files ---' && rg --files -g '\"'!*node_modules*'\"' | sort && printf '%s\\\\n' '--- supabase dirs ---' && find . -maxdepth 4 -type f | sort && printf '%s\\\\n' '--- relevant RLS guidance ---' && sed -n '1,260p' skills/supabase-postgres-best-practices/references/security-rls-basics.md && sed -n '1,260p' skills/supabase-postgres-best-practices/references/security-rls-performance.md && printf '%s\\\\n' '--- CLI ---' && (supabase --version || true) && printf '%s\\\\n' '--- changelog status ---' && curl -L --max-time 10 -s https://supabase.com/changelog.md | head -80\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 26078 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Postgres row level security policies multi tenant workspace auth.uid\", limit: 3) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + } + ], + "resultChars": 54073 + } + ] }, "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-security-002-rls-cross-tenant-leak/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna/resolve-security-002-rls-cross-tenant-leak/run-3/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-auth-001-email-password-flow", @@ -38492,7 +36781,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 2b2f966a-1549-42be-ab6f-6b4bf87fd881, signUp returned {\"userId\":\"2b2f966a-1549-42be-ab6f-6b4bf87fd881\"}" + "notes": "db user 0b440bcb-55a5-416c-861a-4ccb797668ba, signUp returned {\"userId\":\"0b440bcb-55a5-416c-861a-4ccb797668ba\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -38507,7 +36796,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"2b2f966a-1549-42be-ab6f-6b4bf87fd881\"}" + "notes": "{\"userId\":\"0b440bcb-55a5-416c-861a-4ccb797668ba\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -38526,96 +36815,24 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js signUp email password options data display name signInWithPassword getUser profile select single\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" - } - ], - "resultChars": 101691 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript signUp options data signInWithPassword auth getUser select maybeSingle profiles\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", - "title": "signInWithPassword()" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/reference/csharp/auth-signinwithpassword", - "title": "SignIn(email, password)" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - } - ], - "resultChars": 78765 - } - ] + "calls": [] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.6/build-auth-001-email-password-flow/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-auth-001-email-password-flow/run-1/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-auth-001-email-password-flow", @@ -38631,7 +36848,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "auth module loads and the driver completes", @@ -38641,7 +36858,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 9cc02734-74a4-4420-b33a-4c4d9fca806a, signUp returned {\"userId\":\"9cc02734-74a4-4420-b33a-4c4d9fca806a\"}" + "notes": "db user 5d4d5541-8ae4-4f44-b21f-07c83a323783, signUp returned {\"userId\":\"5d4d5541-8ae4-4f44-b21f-07c83a323783\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -38656,7 +36873,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"9cc02734-74a4-4420-b33a-4c4d9fca806a\"}" + "notes": "{\"userId\":\"5d4d5541-8ae4-4f44-b21f-07c83a323783\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -38670,131 +36887,29 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/src/auth.mjs" + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js auth signUp email password options data user metadata signInWithPassword getUser profile table select single\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" - }, - { - "url": "https://supabase.com/docs/guides/auth/enterprise-sso", - "title": "Enterprise Single Sign-On" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" - } - ], - "resultChars": 111937 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -i -n -m 20 'breaking|auth|supabase-js' || true\nprintf '\\\\n---FILES---\\\\n'\nrg --files -g 'AGENTS.md' -g '\"'!node_modules'\"' -g '\"'!supabase/.temp'\"' && printf '\\\\n' || true\nrg --files app supabase | sort\nprintf '\\\\n---AUTH---\\\\n'\nsed -n '1,240p' app/src/auth.mjs\nprintf '\\\\n---PACKAGE---\\\\n'\ncat app/package.json\nprintf '\\\\n---CONFIG/MIGRATIONS---\\\\n'\nfind supabase -maxdepth 3 -type f -not -path '*/.temp/*' -print -exec sh -c 'echo \\\"--- \"'$1\"; sed -n \"1,240p\" \"$1\"'\"' _ {} \\\\;\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 9478 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript signUp email password options data signInWithPassword getUser select maybeSingle createClient\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", - "title": "signInWithPassword()" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" - }, - { - "url": "https://supabase.com/docs/reference/csharp/auth-signinwithpassword", - "title": "SignIn(email, password)" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-identity-linking", - "title": "Identity Linking" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - } - ], - "resultChars": 102378 - } - ] + "calls": [] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.6/build-auth-001-email-password-flow/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-auth-001-email-password-flow/run-2/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-auth-001-email-password-flow", @@ -38820,7 +36935,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user edb0c702-5151-4681-b7fe-883745add07e, signUp returned {\"userId\":\"edb0c702-5151-4681-b7fe-883745add07e\"}" + "notes": "db user 2c2e6080-3406-486a-9398-06086ad5fba0, signUp returned {\"userId\":\"2c2e6080-3406-486a-9398-06086ad5fba0\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -38835,7 +36950,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"edb0c702-5151-4681-b7fe-883745add07e\"}" + "notes": "{\"userId\":\"2c2e6080-3406-486a-9398-06086ad5fba0\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -38854,59 +36969,24 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js signUp email password options data user metadata signInWithPassword getUser profile select single\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/auth/enterprise-sso", - "title": "Enterprise Single Sign-On" - } - ], - "resultChars": 71550 - } - ] + "calls": [] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.6/build-auth-001-email-password-flow/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-auth-001-email-password-flow/run-3/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-cli-001-bootstrap-app", @@ -38956,69 +37036,24 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Row Level Security policy authenticated role REST API grants local development migrations seed data\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - } - ], - "resultChars": 74686 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i \\\"breaking-change|rls|data api|migration|seed\\\" | head -80; printf '\\\\n--- PROJECT FILES ---\\\\n'; find . -name AGENTS.md -o -path '*/supabase/*' -maxdepth 4 | sort; printf '\\\\n--- ROOT ---\\\\n'; ls -la; printf '\\\\n--- CONFIG/MIGRATIONS ---\\\\n'; if [ -f supabase/config.toml ]; then sed -n '1,240p' supabase/config.toml; fi; find supabase -maxdepth 3 -type f -print 2>/dev/null | sort\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 21972 - } - ] + "calls": [] }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.6/build-cli-001-bootstrap-app/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-cli-001-bootstrap-app/run-1/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-cli-001-bootstrap-app", @@ -39068,79 +37103,24 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Row Level Security policy authenticated role SELECT grant Data API tables migrations local development seed data\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - } - ], - "resultChars": 89651 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- changelog breaking changes ---'; curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|breaking change' | head -40 || true; printf '%s\\\\n' '--- files ---'; rg --files -g 'AGENTS.md' -g 'supabase/**' -g '\"'!supabase/.temp/**'\"' -g '\"'!supabase/.branches/**'\"' | sort; printf '%s\\\\n' '--- root ---'; ls -la; printf '%s\\\\n' '--- relevant rules ---'; for f in .agents/skills/supabase-postgres-best-practices/references/security-rls*.md .agents/skills/supabase-postgres-best-practices/references/schema-*.md; do [ -f \\\"\"'$f\" ] && { echo \"### $f\"; sed -n '\"'1,220p' \\\"\"'$f\"; }; done'", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 25056 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- root ---'; ls -la; printf '%s\\\\n' '--- project files ---'; rg --files -g 'AGENTS.md' -g 'supabase/**' -g '\"'!supabase/.temp/**'\"' -g '\"'!supabase/.branches/**'\"' | sort; printf '%s\\\\n' '--- config ---'; [ -f supabase/config.toml ] && sed -n '1,260p' supabase/config.toml || true; printf '%s\\\\n' '--- migrations ---'; for f in supabase/migrations/*.sql; do [ -f \\\"\"'$f\" ] && { echo \"### $f\"; cat \"$f\"; }; done; printf '\"'%s\\\\n' '--- seed ---'; [ -f supabase/seed.sql ] && cat supabase/seed.sql || true; printf '%s\\\\n' '--- git ---'; git status --short; printf '%s\\\\n' '--- cli ---'; supabase --version; printf '%s\\\\n' '--- relevant changelog ---'; curl -fsSL https://supabase.com/changelog/45329-breaking-change-tables-not-exposed-to-data-and-graphql-api-automatically | sed -n '1,180p' | rg -i -C 3 'grant|Data API|authenticated|anon|table' | head -120 || true\"", - "pages": [ - { - "url": "https://supabase.com/changelog/45329-breaking-change-tables-not-exposed-to-data-and-graphql-api-automatically" - } - ], - "resultChars": 157995 - } - ] + "calls": [] }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.6/build-cli-001-bootstrap-app/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-cli-001-bootstrap-app/run-2/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-cli-001-bootstrap-app", @@ -39190,79 +37170,24 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security policy authenticated role select Data API grants migrations local development seed.sql\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles", - "title": "Postgres Roles" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" - } - ], - "resultChars": 85517 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i \\\"breaking-change|rls|migration|seed|auth\\\" | head -80; printf '\\\\n---CLI---\\\\n'; supabase --version; printf '\\\\n---INIT---\\\\n'; supabase init --yes; printf '\\\\n---CONFIG---\\\\n'; sed -n '1,240p' supabase/config.toml\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 9512 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '---RLS RULES---'; cat .agents/skills/supabase-postgres-best-practices/references/security-rls-*.md 2>/dev/null || true; printf '%s\\\\n' '---SCHEMA RULES---'; for f in .agents/skills/supabase-postgres-best-practices/references/schema-primary-keys.md .agents/skills/supabase-postgres-best-practices/references/schema-data-types.md; do [ -f \\\"\"'$f\" ] && cat \"$f\"; done; printf '\"'%s\\\\n' '---CHANGELOG DETAIL---'; curl -fsSL https://supabase.com/changelog/45329-breaking-change-tables-not-exposed-to-data-and-graphql-api-automatically | rg -n -i -C 3 \\\"grant|exposed|default privileges|authenticated|anon\\\" | head -100; printf '%s\\\\n' '---START---'; supabase start\"", - "pages": [ - { - "url": "https://supabase.com/changelog/45329-breaking-change-tables-not-exposed-to-data-and-graphql-api-automatically" - } - ], - "resultChars": 183324 - } - ] + "calls": [] }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.6/build-cli-001-bootstrap-app/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-cli-001-bootstrap-app/run-3/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-cli-002-declarative-schema", @@ -39276,11 +37201,11 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { "name": "supabase db diff used to generate the migration", - "passed": true + "passed": false }, { "name": "schema file updated to include description column", @@ -39296,59 +37221,24 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"local development declarative schemas generate migration db diff schema_paths\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", - "title": "Declarative database schemas" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments", - "title": "Managing Environments" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-diff", - "title": "Diffs the local database for schema changes" - } - ], - "resultChars": 72112 - } - ] + "calls": [] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.6/build-cli-002-declarative-schema/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-cli-002-declarative-schema/run-1/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-cli-002-declarative-schema", @@ -39382,59 +37272,24 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"local development declarative database schemas schema_paths db diff migrations\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", - "title": "Declarative database schemas" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-diff", - "title": "Diffs the local database for schema changes" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - } - ], - "resultChars": 71255 - } - ] + "calls": [] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.6/build-cli-002-declarative-schema/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-cli-002-declarative-schema/run-2/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-cli-002-declarative-schema", @@ -39448,11 +37303,11 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "supabase db diff used to generate the migration", - "passed": false + "passed": true }, { "name": "schema file updated to include description column", @@ -39468,59 +37323,24 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI local database migrations declarative schemas db reset\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-reset", - "title": "Resets the local database to current migrations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", - "title": "Declarative database schemas" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" - } - ], - "resultChars": 79345 - } - ] + "calls": [] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.6/build-cli-002-declarative-schema/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-cli-002-declarative-schema/run-3/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-cli-003-pg-cron-queue-workflow", @@ -39556,177 +37376,24 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- CONFIG ---'; sed -n '1,260p' supabase/config.toml; printf '%s\\\\n' '--- MIGRATION ---'; cat supabase/migrations/20260101000000_enable_cron_and_queues.sql; printf '%s\\\\n' '--- SUPABASE SKILL ---'; sed -n '1,260p' .claude/skills/supabase/SKILL.md; printf '%s\\\\n' '--- REFERENCE FILES ---'; find .claude/skills/supabase-postgres-best-practices/references -maxdepth 1 -type f | sort | sed -n '1,200p'; printf '%s\\\\n' '--- CHANGELOG ---'; curl -fsSL https://supabase.com/changelog.md | sed -n '1,220p'\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 34359 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Queues pgmq create queue send read delete Edge Function pgmq_public\", limit: 8) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", - "title": "Expose Queues for local and self-hosted Supabase" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgmq", - "title": "pgmq: Queues" - } - ], - "resultChars": 54101 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Cron schedule SQL cron.schedule every minute pgmq send queue\", limit: 8) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_cron", - "title": "pg_cron: Schedule Recurring Jobs with Cron Syntax in Postgres" - } - ], - "resultChars": 51838 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Cron quickstart schedule database function every minute cron.schedule Supabase Cron\", limit: 6) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_cron", - "title": "pg_cron: Schedule Recurring Jobs with Cron Syntax in Postgres" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" - } - ], - "resultChars": 28388 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions createClient SUPABASE_SERVICE_ROLE_KEY npm @supabase/supabase-js Deno.serve\", limit: 5) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - } - ], - "resultChars": 43958 - } - ] + "calls": [] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.6/build-cli-003-pg-cron-queue-workflow/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-cli-003-pg-cron-queue-workflow/run-1/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-cli-003-pg-cron-queue-workflow", @@ -39758,121 +37425,28 @@ { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 4) from the queue" + "notes": "function removed the seeded message (id 3) from the queue" } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Queues pgmq create queue send read archive delete messages cron schedule every minute Edge Function queue worker\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgmq", - "title": "pgmq: Queues" - } - ], - "resultChars": 47633 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Consuming Supabase Queue Messages with Edge Functions example code read delete queue messages pgmq_public\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - } - ], - "resultChars": 26971 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- CHANGELOG MATCHES ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -n -C 2 'breaking-change|queue|cron|edge function' | head -120 || true; printf '%s\\\\n' '--- CONFIG ---'; sed -n '1,240p' supabase/config.toml; printf '%s\\\\n' '--- MIGRATION ---'; cat supabase/migrations/20260101000000_enable_cron_and_queues.sql; printf '%s\\\\n' '--- SKILL REFERENCES ---'; find .agents/skills/supabase-postgres-best-practices/references -maxdepth 1 -type f -printf '%f\\\\n' | sort | rg 'security|schema|lock|query|advanced' | head -80; printf '%s\\\\n' '--- STATUS ---'; git status --short; printf '%s\\\\n' '--- CLI ---'; supabase --version\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 226 - } - ] + "calls": [] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.6/build-cli-003-pg-cron-queue-workflow/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-cli-003-pg-cron-queue-workflow/run-2/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-cli-003-pg-cron-queue-workflow", @@ -39908,109 +37482,24 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- migration ---'; cat supabase/migrations/20260101000000_enable_cron_and_queues.sql; printf '%s\\\\n' '--- config ---'; cat supabase/config.toml; printf '%s\\\\n' '--- git ---'; git status --short; printf '%s\\\\n' '--- relevant postgres refs ---'; find .agents/skills/supabase-postgres-best-practices/references -maxdepth 1 -type f | sort | rg '(queue|cron|security|function|schema)' || true; printf '%s\\\\n' '--- changelog tags ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|queue|cron' | head -120\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 15223 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Queues pgmq create queue send read delete Edge Function service role pgmq_public cron schedule every minute local\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", - "title": "Expose Queues for local and self-hosted Supabase" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - } - ], - "resultChars": 39739 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"pgmq.create queue SQL Supabase Queues create basic queue\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - } - ], - "resultChars": 41186 - } - ] + "calls": [] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.6/build-cli-003-pg-cron-queue-workflow/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-cli-003-pg-cron-queue-workflow/run-3/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-dataapi-001-relational-report", @@ -40054,97 +37543,24 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase javascript select nested relationships foreign key ordering service role backend\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/sql-to-api", - "title": "Converting SQL to JavaScript API" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - }, - { - "url": "https://supabase.com/docs/guides/api/rest/generating-types", - "title": "Generating TypeScript Types" - }, - { - "url": "https://supabase.com/docs/guides/integrations", - "title": "Integrations" - } - ], - "resultChars": 33635 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- files ---'; rg --files -g 'AGENTS.md' -g '\"'!node_modules'\"' -g '\"'!supabase/.temp'\"' | sort; printf '%s\\\\n' '--- status ---'; git status --short; printf '%s\\\\n' '--- report ---'; sed -n '1,240p' app/report.mjs; printf '%s\\\\n' '--- package ---'; cat package.json; printf '%s\\\\n' '--- supabase files ---'; find supabase -maxdepth 3 -type f -not -path '*/.temp/*' -print | sort; printf '%s\\\\n' '--- changelog tags ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -m 12 'breaking-change|postgrest|supabase-js|api key' || true\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3700 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"secret API key REST apikey Authorization header backend Supabase\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", - "title": "Supabase CLI" - }, - { - "url": "https://supabase.com/docs/guides/api/creating-routes", - "title": "Creating API Routes" - } - ], - "resultChars": 57476 - } - ] + "calls": [] }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-dataapi-001-relational-report/run-1/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-dataapi-001-relational-report", @@ -40159,7 +37575,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "report runs and prints JSON", @@ -40178,74 +37594,34 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/report.mjs" + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" }, { "name": "report queries via the Data API, not raw SQL", "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- files ---'; rg --files -g 'AGENTS.md' -g 'app/**' -g 'supabase/**' -g 'package*.json' -g '.env*' | sort; printf '%s\\\\n' '--- report ---'; sed -n '1,240p' app/report.mjs; printf '%s\\\\n' '--- package ---'; for f in package.json app/package.json; do [ -f \\\"\"'$f\" ] && { echo \"### $f\"; cat \"$f\"; }; done; printf '\"'%s\\\\n' '--- config/migrations ---'; [ -f supabase/config.toml ] && sed -n '1,220p' supabase/config.toml; for f in supabase/migrations/*.sql supabase/seed.sql; do [ -f \\\"\"'$f\" ] && { echo \"### $f\"; sed -n '\"'1,260p' \\\"\"'$f\"; }; done; printf '\"'%s\\\\n' '--- changelog tags ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -m 20 'breaking-change|postgrest|supabase-js|node' || true\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 10423 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js Node createClient secret key select nested relationships foreign tables order data API\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/api/creating-routes", - "title": "Creating API Routes" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - }, - { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" - } - ], - "resultChars": 19475 - } - ] + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-dataapi-001-relational-report/run-2/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-dataapi-001-relational-report", @@ -40289,59 +37665,24 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js select nested relationships service role server backend\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/auth0", - "title": "Auth0" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles", - "title": "Postgres Roles" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/firebase-auth", - "title": "Firebase Auth" - } - ], - "resultChars": 47882 - } - ] + "calls": [] }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-dataapi-001-relational-report/run-3/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-dataapi-002-restock-alert-report", @@ -40385,69 +37726,24 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase javascript select joins foreign tables service role backend\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - }, - { - "url": "https://supabase.com/docs/guides/api/sql-to-api", - "title": "Converting SQL to JavaScript API" - }, - { - "url": "https://supabase.com/docs/guides/integrations", - "title": "Integrations" - }, - { - "url": "https://supabase.com/docs/guides/database/joins-and-nesting", - "title": "Querying Joins and Nested tables" - }, - { - "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", - "title": "Engineering for Scale" - } - ], - "resultChars": 37732 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,160p'; printf '\\\\n--- restock ---\\\\n'; sed -n '1,240p' app/restock.mjs; printf '\\\\n--- package ---\\\\n'; cat app/package.json; printf '\\\\n--- schema ---\\\\n'; sed -n '1,280p' supabase/migrations/0000_inventory_schema.sql; printf '\\\\n--- status ---\\\\n'; supabase status -o env\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 1450 - } - ] + "calls": [] }, "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.6/build-dataapi-002-restock-alert-report/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-dataapi-002-restock-alert-report/run-1/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-dataapi-002-restock-alert-report", @@ -40491,97 +37787,24 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- app/restock.mjs ---' && sed -n '1,240p' app/restock.mjs && printf '%s\\\\n' '--- app/package.json ---' && cat app/package.json && printf '%s\\\\n' '--- migration ---' && sed -n '1,280p' supabase/migrations/0000_inventory_schema.sql && printf '%s\\\\n' '--- config ---' && sed -n '1,220p' supabase/config.toml && printf '%s\\\\n' '--- git status ---' && git status --short && printf '%s\\\\n' '--- changelog relevant scan ---' && curl -fsSL https://supabase.com/changelog.md | rg -i -m 20 'breaking-change|postgrest|javascript|supabase-js|data api' || true\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 6727 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Data API JavaScript nested tables select foreign key relationships service_role secret key REST\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/api/sql-to-api", - "title": "Converting SQL to JavaScript API" - }, - { - "url": "https://supabase.com/docs/guides/api/creating-routes", - "title": "Creating API Routes" - }, - { - "url": "https://supabase.com/docs/guides/api/rest/client-libs", - "title": "Client Libraries" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - } - ], - "resultChars": 27783 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Data API REST embedded resources nested foreign tables select PostgREST Supabase\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api", - "title": "Data REST API" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/api/sql-to-api", - "title": "Converting SQL to JavaScript API" - }, - { - "url": "https://supabase.com/docs/guides/api/sql-to-rest", - "title": "SQL to REST API Translator" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - } - ], - "resultChars": 22664 - } - ] + "calls": [] }, "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.6/build-dataapi-002-restock-alert-report/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-dataapi-002-restock-alert-report/run-2/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-dataapi-002-restock-alert-report", @@ -40625,69 +37848,24 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Data API REST JavaScript fetch apikey Authorization secret key embedded relationships foreign key select\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/creating-routes", - "title": "Creating API Routes" - }, - { - "url": "https://supabase.com/docs/guides/api/sql-to-api", - "title": "Converting SQL to JavaScript API" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - }, - { - "url": "https://supabase.com/docs/guides/api", - "title": "Data REST API" - }, - { - "url": "https://supabase.com/docs/reference/swift/select", - "title": "Fetch data: select()" - } - ], - "resultChars": 67737 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|api key|postgrest|rest api' | sed -n '1,120p'; printf '\\\\n--- direct REST probe ---\\\\n'; eval \\\"\"'$(supabase status -o env 2>/dev/null | sed '\"'s/\"'^/export /'\"')\\\"; curl -sS -D /tmp/restock-headers -H \\\"apikey: \"'$SECRET_KEY\" -H \"Authorization: Bearer $SECRET_KEY\" --get \"$API_URL/rest/v1/inventory\" --data-urlencode '\"'select=quantity,warehouses(name),products(name,reorder_threshold,suppliers(email))' -o /tmp/restock-response; cat /tmp/restock-headers; cat /tmp/restock-response\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 1666 - } - ] + "calls": [] }, "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.6/build-dataapi-002-restock-alert-report/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-dataapi-002-restock-alert-report/run-3/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-database-001-migrate-postgres-to-supabase", @@ -40724,59 +37902,24 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"local development restore pg_restore existing database dump Supabase CLI\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", - "title": "Migrate from Vercel Postgres to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/neon", - "title": "Migrate from Neon to Supabase" - } - ], - "resultChars": 42818 - } - ] + "calls": [] }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.6/build-database-001-migrate-postgres-to-supabase/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-database-001-migrate-postgres-to-supabase/run-1/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-database-001-migrate-postgres-to-supabase", @@ -40813,407 +37956,78 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"restore pg_restore database dump local development Supabase CLI\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-reset", - "title": "Resets the local database to current migrations" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/heroku", - "title": "Migrate from Heroku to Supabase" - } - ], - "resultChars": 39873 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"set -e\nprintf '%s\\\\n' '--- relevant changelog scan ---'\ncurl -fsSL https://supabase.com/changelog.md | rg -i -m 20 'breaking-change|migration|restore|postgres' || true\nprintf '%s\\\\n' '--- dump table of contents ---'\npg_restore --list source.dump | sed -n '1,160p'\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3775 - } - ] + "calls": [] }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.6/build-database-001-migrate-postgres-to-supabase/run-2/result.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-database-001-migrate-postgres-to-supabase", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", - "passed": true - }, - { - "name": "tasks_team_status_idx index survived the restore", - "passed": true - }, - { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"restore existing Postgres database dump local Supabase pg_restore roles extensions\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" - }, - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/transferring-from-cloud-to-self-host-in-supabase-2oWNvW", - "title": "Transferring from platform to self-hosted Supabase" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-postgres-extensions", - "title": "Custom Postgres Extensions" - } - ], - "resultChars": 50510 - } - ] - }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "run": 3, - "sourcePath": "codex-gpt-5.6/build-database-001-migrate-postgres-to-supabase/run-3/result.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "rls", - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" - }, - { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" - }, - { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" - }, - { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=200" - }, - { - "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=200" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions authenticate user Authorization header createClient global headers RLS service role\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - } - ], - "resultChars": 38318 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | rg -n -i \"breaking-change|edge function|jwt|auth\" | head -80'", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 15092 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions default secrets SUPABASE_ANON_KEY SUPABASE_PUBLISHABLE_KEY environment variables\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - } - ], - "resultChars": 50039 - } - ] - }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "run": 1, - "sourcePath": "codex-gpt-5.6/build-functions-004-service-role-bypass/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-database-001-migrate-postgres-to-supabase/run-2/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, - "eval": "build-functions-004-service-role-bypass", + "eval": "build-database-001-migrate-postgres-to-supabase", "stage": "build", "product": [ - "edge-functions", - "auth", "database" ], "topic": [ - "rls", - "security", - "sdk" + "migrations" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true }, { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true }, { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" + "name": "foreign key constraints survived the restore", + "passed": true }, { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=403" + "name": "tasks_team_status_idx index survived the restore", + "passed": true }, { - "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=403" + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- relevant security rules ---'; find .agents/skills/supabase-postgres-best-practices/references -maxdepth 1 -type f | sort | rg 'security|rls' | xargs -r -n1 sh -c 'echo \\\"### \"'$0\"; cat \"$0\"'\"'; printf '%s\\\\n' '--- changelog tags ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|edge function|jwt|auth' | head -n 120\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 13243 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions authenticate users Authorization header getUser createClient service role RLS verify_jwt\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" - } - ], - "resultChars": 51597 - } - ] + "calls": [] }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "run": 2, - "sourcePath": "codex-gpt-5.6/build-functions-004-service-role-bypass/run-2/result.json" + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-database-001-migrate-postgres-to-supabase/run-3/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-functions-004-service-role-bypass", @@ -41250,81 +38064,36 @@ { "name": "user A cannot force-read user B note", "passed": true, - "notes": "status=200" + "notes": "status=403" }, { "name": "user B cannot force-read user A note", "passed": true, - "notes": "status=200" + "notes": "status=403" } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- relevant rule files ---'; rg --files .agents/skills/supabase-postgres-best-practices/references | rg 'security|rls|index' | sort; printf '%s\\\\n' '--- changelog security/function entries ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'edge function|jwt|rls|breaking-change' | head -n 160\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 11619 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions authorization JWT createClient Authorization header user getUser RLS service role\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - } - ], - "resultChars": 63849 - } - ] + "calls": [] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "run": 3, - "sourcePath": "codex-gpt-5.6/build-functions-004-service-role-bypass/run-3/result.json" + "run": 1, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-functions-004-service-role-bypass/run-1/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, - "eval": "build-functions-005-dual-auth-user-secret", + "eval": "build-functions-004-service-role-bypass", "stage": "build", "product": [ "edge-functions", @@ -41332,196 +38101,62 @@ "database" ], "topic": [ - "sdk", "rls", - "security" + "security", + "sdk" ], "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", + "interface": "mcp", "passed": false, "checks": [ { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" - }, - { - "name": "user with JWT reads only their own rows", + "name": "rejects missing auth", "passed": true, - "notes": "status 200: [{\"user_id\":\"e97313f5-6cc6-47ee-a81e-ec178bcce7e6\",\"metric\":\"steps_a_mta4aozk\",\"value\":111}]" + "notes": "status=401" }, { - "name": "user cannot read another user's rows by passing user_id", + "name": "user A reads own note", "passed": true, - "notes": "status 200: [{\"user_id\":\"e97313f5-6cc6-47ee-a81e-ec178bcce7e6\",\"metric\":\"steps_a_mta4aozk\",\"value\":111}]" + "notes": "status=200" }, { - "name": "service key bypasses RLS to read the target user's rows", + "name": "reads only with the caller's JWT", "passed": false, - "notes": "status 401: {\"error\":\"Unauthorized\"}" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" + "notes": "bearer_tokens=2, all_match=false" }, { - "name": "a user token in the apikey slot is not treated as the service key", + "name": "user A cannot force-read user B note", "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" + "notes": "status=403" }, { - "name": "implementation uses @supabase/server", + "name": "user B cannot force-read user A note", "passed": true, - "notes": "imports @supabase/server / withSupabase" + "notes": "status=403" } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions JWT verification Authorization apikey service_role key Deno serve createClient getUser\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/http-methods", - "title": "Routing" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - } - ], - "resultChars": 72313 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- CHANGELOG RELEVANT ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|edge function|jwt|apikey' | head -n 160 || true; printf '%s\\\\n' '--- AGENTS ---'; find .. -name AGENTS.md -print; printf '%s\\\\n' '--- ROOT FILES ---'; find . -maxdepth 4 -type f | sort | sed -n '1,260p'; printf '%s\\\\n' '--- SUPABASE SKILL REST ---'; sed -n '1,260p' .claude/skills/supabase/SKILL.md; printf '%s\\\\n' '--- RLS RULES ---'; find .claude/skills/supabase-postgres-best-practices/references -maxdepth 1 -type f -name 'security-*' -print | sort\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 35611 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"config.toml functions verify_jwt false Edge Functions per-function configuration\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - } - ], - "resultChars": 32331 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"@supabase/server withSupabase auth ['user', 'secret'] ctx supabaseAdmin userClaims authMode Edge Function\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - } - ], - "resultChars": 39346 - } - ] + "calls": [] }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "run": 1, - "sourcePath": "codex-gpt-5.6/build-functions-005-dual-auth-user-secret/run-1/result.json" + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-functions-004-service-role-bypass/run-2/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, - "eval": "build-functions-005-dual-auth-user-secret", + "eval": "build-functions-004-service-role-bypass", "stage": "build", "product": [ "edge-functions", @@ -41529,228 +38164,59 @@ "database" ], "topic": [ - "sdk", "rls", - "security" + "security", + "sdk" ], "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, + "interface": "mcp", + "passed": false, "checks": [ { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"a9467f56-aff1-4878-b576-123c8ec49fc6\",\"metric\":\"steps_a_mta4bdaw\",\"value\":111}]" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: [{\"user_id\":\"a9467f56-aff1-4878-b576-123c8ec49fc6\",\"metric\":\"steps_a_mta4bdaw\",\"value\":111}]" - }, - { - "name": "service key bypasses RLS to read the target user's rows", + "name": "rejects missing auth", "passed": true, - "notes": "status 200: [{\"user_id\":\"15e075e2-eb2d-4c1a-bc31-77c40be5a3c3\",\"metric\":\"steps_b_mta4bdaw\",\"value\":222}]" + "notes": "status=401" }, { - "name": "non-service key is not granted service access", + "name": "user A reads own note", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status=200" }, { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "name": "reads only with the caller's JWT", + "passed": false, + "notes": "bearer_tokens=2, all_match=false" }, { - "name": "a user token in the apikey slot is not treated as the service key", + "name": "user A cannot force-read user B note", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status=200" }, { - "name": "implementation uses @supabase/server", + "name": "user B cannot force-read user A note", "passed": true, - "notes": "imports @supabase/server / withSupabase" + "notes": "status=200" } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '---AGENTS---'; find .. -name AGENTS.md -print; printf '%s\\\\n' '---ROOT---'; ls -la; printf '%s\\\\n' '---FILES---'; rg --files -g '\"'!node_modules'\"' -g '\"'!**/.git/**'\"' | sort | sed -n '1,240p'; printf '%s\\\\n' '---CONFIG---'; sed -n '1,260p' supabase/config.toml 2>/dev/null || true; printf '%s\\\\n' '---MIGRATIONS---'; for f in supabase/migrations/*.sql; do [ -e \\\"\"'$f\" ] || continue; echo \"### $f\"; sed -n '\"'1,260p' \\\"\"'$f\"; done; printf '\"'%s\\\\n' '---CHANGELOG MATCHES---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|edge function|jwt|api key' | head -n 160\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 23290 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions authenticate JWT Authorization header getUser service role apikey header config verify_jwt secret key\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - } - ], - "resultChars": 87511 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Securing Edge Functions secret key apikey header authorize in code SUPABASE_SERVICE_ROLE_KEY\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - } - ], - "resultChars": 136249 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"@supabase/server Edge Functions userClaims apiKeyClaims SUPABASE_SECRET_KEYS authorize apikey\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getclaims" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", - "title": "Build a User Management App with React" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - } - ], - "resultChars": 145649 - } - ] + ], + "skills": { + "available": [], + "loaded": [] }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "run": 2, - "sourcePath": "codex-gpt-5.6/build-functions-005-dual-auth-user-secret/run-2/result.json" + "docs": { + "calls": [] + }, + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-functions-004-service-role-bypass/run-3/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-functions-005-dual-auth-user-secret", @@ -41768,7 +38234,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "seed rows present", @@ -41778,337 +38244,231 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"08e350fd-e341-417d-9609-6f09e167bb0e\",\"metric\":\"steps_a_mta4bxru\",\"value\":111}]" + "passed": false, + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: [{\"user_id\":\"08e350fd-e341-417d-9609-6f09e167bb0e\",\"metric\":\"steps_a_mta4bxru\",\"value\":111}]" + "passed": false, + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"a519da38-e12f-4bdf-aaf9-012109ab5b1e\",\"metric\":\"steps_b_mta4bxru\",\"value\":222}]" + "passed": false, + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" + "passed": false, + "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions authentication verify_jwt service_role apikey Authorization header getUser\", limit: 8) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" - } - ], - "resultChars": 86872 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- changelog relevant ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|edge function|api key|secret key|verify_jwt' | head -160 || true; printf '%s\\\\n' '--- config ---'; cat supabase/config.toml; printf '%s\\\\n' '--- migration ---'; cat supabase/migrations/0000_stats_schema.sql; printf '%s\\\\n' '--- git ---'; git status --short; printf '%s\\\\n' '--- cli ---'; supabase --version; printf '%s\\\\n' '--- security refs ---'; find .agents/skills/supabase-postgres-best-practices/references -maxdepth 1 -type f | sort | rg 'security|index' | head -40\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 875 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Securing Edge Functions auth user secret multiple auth modes one endpoint withSupabase\", limit: 5) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - } - ], - "resultChars": 33324 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"withSupabase auth array user secret same function auth: ['user', 'secret']\", limit: 10) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/migrating-auth-users-between-projects", - "title": "Migrating Auth Users Between Supabase Projects" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/overview", - "title": "Third-party auth" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" - }, - { - "url": "https://supabase.com/docs/guides/auth/enterprise-sso", - "title": "Enterprise Single Sign-On" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - } - ], - "resultChars": 114944 - } - ] + "calls": [] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "run": 3, - "sourcePath": "codex-gpt-5.6/build-functions-005-dual-auth-user-secret/run-3/result.json" + "run": 1, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-functions-005-dual-auth-user-secret/run-1/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, - "eval": "build-rls-003-org-roles-permissions", + "eval": "build-functions-005-dual-auth-user-secret", "stage": "build", "product": [ - "database", - "auth" + "edge-functions", + "auth", + "database" ], "topic": [ + "sdk", "rls", "security" ], "suite": "benchmark", - "interface": "mcp", - "passed": true, + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, "checks": [ { - "name": "RLS enabled on documents", - "passed": true + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" }, { - "name": "viewer sees only documents in their org", - "passed": true + "name": "rejects request with no credentials", + "passed": true, + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { - "name": "viewer cannot insert", - "passed": true + "name": "user with JWT reads only their own rows", + "passed": false, + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { - "name": "editor can insert own org document", - "passed": true + "name": "user cannot read another user's rows by passing user_id", + "passed": false, + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { - "name": "editor can update own document", - "passed": true + "name": "service key bypasses RLS to read the target user's rows", + "passed": false, + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { - "name": "editor cannot update another user's document", - "passed": true + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { - "name": "editor cannot delete another user's document", - "passed": true + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { - "name": "admin can update any document in their org", - "passed": true + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { - "name": "admin can delete any document in their org", - "passed": true + "name": "implementation uses @supabase/server", + "passed": false, + "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-functions-005-dual-auth-user-secret/run-2/result.json" + }, + { + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-luna", + "reasoningEffort": "medium" + }, + "eval": "build-functions-005-dual-auth-user-secret", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "sdk", + "rls", + "security" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" }, { - "name": "admin cannot affect another org", - "passed": true + "name": "rejects request with no credentials", + "passed": true, + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { - "name": "WITH CHECK blocks editor from moving document to another org", - "passed": true + "name": "user with JWT reads only their own rows", + "passed": false, + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { - "name": "editor can still see their own org's roster", - "passed": true + "name": "user cannot read another user's rows by passing user_id", + "passed": false, + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { - "name": "cannot see another org's membership roster", - "passed": true + "name": "service key bypasses RLS to read the target user's rows", + "passed": false, + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { - "name": "a viewer role in one org doesn't grant admin power in another org", - "passed": true + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { - "name": "multi-org user can act as admin in the org where they hold that role", - "passed": true + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 503: {\"message\":\"name resolution failed\"}" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 503: {\"message\":\"name resolution failed\"}" + }, + { + "name": "implementation uses @supabase/server", + "passed": false, + "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|row level|rls|policy' | head -120\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 9427 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security policies auth.uid organizations memberships security definer\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - } - ], - "resultChars": 79622 - } - ] + "calls": [] }, - "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", - "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", - "run": 1, - "sourcePath": "codex-gpt-5.6/build-rls-003-org-roles-permissions/run-1/result.json" + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-functions-005-dual-auth-user-secret/run-3/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-rls-003-org-roles-permissions", @@ -42187,79 +38547,24 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security policies organization membership authenticated role update select policy performance auth uid security definer\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles", - "title": "Postgres Roles" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - } - ], - "resultChars": 61783 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i -C 2 'breaking-change|row level|rls|policy|postgres' | sed -n '1,220p'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 15638 - } - ] + "calls": [] }, "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", - "run": 2, - "sourcePath": "codex-gpt-5.6/build-rls-003-org-roles-permissions/run-2/result.json" + "run": 1, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-rls-003-org-roles-permissions/run-1/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-rls-003-org-roles-permissions", @@ -42309,356 +38614,149 @@ "passed": true }, { - "name": "admin can delete any document in their org", - "passed": true - }, - { - "name": "admin cannot affect another org", - "passed": true - }, - { - "name": "WITH CHECK blocks editor from moving document to another org", - "passed": true - }, - { - "name": "editor can still see their own org's roster", - "passed": true - }, - { - "name": "cannot see another org's membership roster", - "passed": true - }, - { - "name": "a viewer role in one org doesn't grant admin power in another org", - "passed": true - }, - { - "name": "multi-org user can act as admin in the org where they hold that role", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security policies organizations membership roles auth.uid security definer RLS performance\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - } - ], - "resultChars": 88391 - } - ] - }, - "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", - "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", - "run": 3, - "sourcePath": "codex-gpt-5.6/build-rls-003-org-roles-permissions/run-3/result.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-storage-001-private-bucket-access", - "stage": "build", - "product": [ - "storage", - "database" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", + "name": "admin can delete any document in their org", "passed": true }, { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 01a03e35-268f-732a-a084-e1a1e1d706f5/receipt-alpha.pdf, 01a03e35-268f-732a-a084-e1a1e1d706f5/receipt-beta.pdf" + "name": "admin cannot affect another org", + "passed": true }, { - "name": "user B cannot read user A files", + "name": "WITH CHECK blocks editor from moving document to another org", "passed": true }, { - "name": "anon reads no files", + "name": "editor can still see their own org's roster", "passed": true }, { - "name": "user A can upload into own folder", + "name": "cannot see another org's membership roster", "passed": true }, { - "name": "user B cannot upload into user A folder", + "name": "a viewer role in one org doesn't grant admin power in another org", "passed": true }, { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Private bucket created; RLS remains enabled; authenticated SELECT and INSERT policies are owner-path scoped; supabase-js uses createSignedUrl with a 10-minute expiry." + "name": "multi-org user can act as admin in the org where they hold that role", + "passed": true } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Storage access control RLS policies foldername auth.uid createSignedUrl private bucket upload download\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" - } - ], - "resultChars": 39546 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- relevant best-practice references ---'; rg --files .agents/skills/supabase-postgres-best-practices/references | rg 'security|rls|function' | sort; printf '%s\\\\n' '--- changelog storage/security hits ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'storage|rls|signed url|breaking-change' | head -n 120\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 8860 - } - ] + "calls": [] }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "run": 1, - "sourcePath": "codex-gpt-5.6/build-storage-001-private-bucket-access/run-1/result.json" + "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", + "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-rls-003-org-roles-permissions/run-2/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, - "eval": "build-storage-001-private-bucket-access", + "eval": "build-rls-003-org-roles-permissions", "stage": "build", "product": [ - "storage", - "database" + "database", + "auth" ], "topic": [ "rls", - "sdk" + "security" ], "suite": "benchmark", "interface": "mcp", "passed": true, "checks": [ { - "name": "bucket user-files exists", + "name": "RLS enabled on documents", "passed": true }, { - "name": "bucket user-files is private", + "name": "viewer sees only documents in their org", "passed": true }, { - "name": "RLS still enabled on storage.objects", + "name": "viewer cannot insert", "passed": true }, { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 01a03e35-1534-73fc-a4be-acb86cab17be/receipt-alpha.pdf, 01a03e35-1534-73fc-a4be-acb86cab17be/receipt-beta.pdf" + "name": "editor can insert own org document", + "passed": true }, { - "name": "user B cannot read user A files", + "name": "editor can update own document", "passed": true }, { - "name": "anon reads no files", + "name": "editor cannot update another user's document", "passed": true }, { - "name": "user A can upload into own folder", + "name": "editor cannot delete another user's document", "passed": true }, { - "name": "user B cannot upload into user A folder", + "name": "admin can update any document in their org", "passed": true }, { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT policies, enabled RLS, and a 10-minute createSignedUrl implementation are all present." + "name": "admin can delete any document in their org", + "passed": true + }, + { + "name": "admin cannot affect another org", + "passed": true + }, + { + "name": "WITH CHECK blocks editor from moving document to another org", + "passed": true + }, + { + "name": "editor can still see their own org's roster", + "passed": true + }, + { + "name": "cannot see another org's membership roster", + "passed": true + }, + { + "name": "a viewer role in one org doesn't grant admin power in another org", + "passed": true + }, + { + "name": "multi-org user can act as admin in the org where they hold that role", + "passed": true } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Storage access control RLS policies foldername auth.uid signed URL createSignedUrl private bucket\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", - "title": "Why can't I upload/list/etc my public bucket?" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", - "title": "Custom Roles" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - } - ], - "resultChars": 33594 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog storage breaking change RLS signed URL 2026 Supabase", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/reference/javascript/storage-from-createsignedurl createSignedUrl expiresIn", - "pages": [] - } - ] + "calls": [] }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "run": 2, - "sourcePath": "codex-gpt-5.6/build-storage-001-private-bucket-access/run-2/result.json" + "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", + "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-rls-003-org-roles-permissions/run-3/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-storage-001-private-bucket-access", @@ -42673,347 +38771,115 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 01a03e35-18e4-7164-92bb-128e4456921a/receipt-alpha.pdf, 01a03e35-18e4-7164-92bb-128e4456921a/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT policies, intact RLS, and short-lived createSignedUrl sharing code are all present." + "passed": false, + "notes": "no row in storage.buckets with id or name 'user-files'" } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Storage access control RLS policy foldername auth.uid signed URL createSignedUrl private bucket upload download\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", - "title": "Why can't I upload/list/etc my public bucket?" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", - "title": "Custom Roles" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/design", - "title": "The Storage Schema" - } - ], - "resultChars": 44284 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript createSignedUrl expiresIn private bucket supabase-js\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", - "title": "Storage Image Transformations" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", - "title": "Manage Storage Image Transformations usage" - } - ], - "resultChars": 32107 - } - ] + "calls": [] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "run": 3, - "sourcePath": "codex-gpt-5.6/build-storage-001-private-bucket-access/run-3/result.json" + "run": 1, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-storage-001-private-bucket-access/run-1/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, - "eval": "build-tests-001-rls-tenant-isolation", + "eval": "build-storage-001-private-bucket-access", "stage": "build", "product": [ + "storage", "database" ], "topic": [ - "tests", - "rls" + "rls", + "sdk" ], "suite": "benchmark", - "interface": "cli", - "passed": true, + "interface": "mcp", + "passed": false, "checks": [ { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation.test.sql" - }, - { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "8 passed, 0 failed" - }, - { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "Correctly identifies the posts policy as allowing cross-organization reads and grounds this in the pgTAP baseline failures; does not blame notes." + "name": "bucket user-files exists", + "passed": false, + "notes": "no row in storage.buckets with id or name 'user-files'" } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Row Level Security testing pgTAP auth.uid set_config request.jwt.claims tenant isolation\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgtap", - "title": "pgTAP: Unit Testing" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/database/testing", - "title": "Testing Your Database" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - } - ], - "resultChars": 56902 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|row level|rls|pgtap|test db' | head -80; printf '\\\\n--- MIGRATION ---\\\\n'; cat supabase/migrations/20240101000000_init.sql; printf '\\\\n--- GIT STATUS ---\\\\n'; git status --short; printf '\\\\n--- CLI VERSION ---\\\\n'; supabase --version\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 226 - } - ] + "calls": [] }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "run": 1, - "sourcePath": "codex-gpt-5.6/build-tests-001-rls-tenant-isolation/run-1/result.json" + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-storage-001-private-bucket-access/run-2/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, - "eval": "build-tests-001-rls-tenant-isolation", + "eval": "build-storage-001-private-bucket-access", "stage": "build", "product": [ + "storage", "database" ], "topic": [ - "tests", - "rls" + "rls", + "sdk" ], "suite": "benchmark", - "interface": "cli", - "passed": true, + "interface": "mcp", + "passed": false, "checks": [ { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" - }, - { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "4 passed, 4 failed" - }, - { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "Correctly identifies the posts RLS flaw: membership is not correlated to posts.org_id, allowing cross-tenant reads. This is grounded in the pgTAP failure, while notes isolation is correctly reported as passing." + "name": "bucket user-files exists", + "passed": false, + "notes": "no row in storage.buckets with id or name 'user-files'" } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"pgTAP row level security testing auth.uid set_config request.jwt.claims Supabase database tests\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" - }, - { - "url": "https://supabase.com/docs/guides/realtime/authorization", - "title": "Realtime Authorization" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - } - ], - "resultChars": 64472 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Testing your database pgTAP Supabase plan tests database SQL files\", limit: 5) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, - { - "url": "https://supabase.com/docs/guides/database/testing", - "title": "Testing Your Database" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgtap", - "title": "pgTAP: Unit Testing" - } - ], - "resultChars": 47133 - } - ] + "calls": [] }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "run": 2, - "sourcePath": "codex-gpt-5.6/build-tests-001-rls-tenant-isolation/run-2/result.json" + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-storage-001-private-bucket-access/run-3/result.json" }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + { + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-tests-001-rls-tenant-isolation", @@ -43032,240 +38898,138 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation.sql" + "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "15 passed, 0 failed" + "notes": "8 passed, 0 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies the under-scoped `posts` policy as allowing cross-organization reads and grounds this in failing pgTAP isolation tests. It does not blame `notes` or dismiss the tests." + "judgeNotes": "Correctly identifies `posts` as the broken tenant-isolation policy, explains the cross-organization read flaw, and reports passing pgTAP results after fixing it." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security tenant isolation testing pgTAP auth.uid policies\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgtap", - "title": "pgTAP: Unit Testing" - }, - { - "url": "https://supabase.com/docs/guides/database/testing", - "title": "Testing Your Database" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - } - ], - "resultChars": 56902 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- CHANGELOG RLS/PGTAP HITS ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|row level|rls|pgtap' | head -160 || true; printf '%s\\\\n' '--- RELEVANT BEST PRACTICES ---'; find .agents/skills/supabase-postgres-best-practices/references -maxdepth 1 -type f | sort | rg 'security|rls|index' ; printf '%s\\\\n' '--- PROJECT FILES ---'; find supabase -maxdepth 4 -type f -print | sort; printf '%s\\\\n' '--- CONFIG ---'; sed -n '1,240p' supabase/config.toml 2>/dev/null || true; printf '%s\\\\n' '--- SQL CONTENT ---'; for f in \"'$(find supabase -type f '\"\\\\( -name '*.sql' -o -name '*.toml' \\\\) | sort); do echo \\\"===== \"'$f\"; sed -n '\"'1,260p' \\\"\"'$f\"; done'", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 18797 - } - ] + "calls": [] }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "run": 3, - "sourcePath": "codex-gpt-5.6/build-tests-001-rls-tenant-isolation/run-3/result.json" + "run": 1, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-tests-001-rls-tenant-isolation/run-1/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, - "eval": "build-vectors-001-rag-with-permissions", + "eval": "build-tests-001-rls-tenant-isolation", "stage": "build", "product": [ - "database", - "vectors" + "database" ], "topic": [ - "sql", + "tests", "rls" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "document_sections.embedding is vector(384)", + "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "vector(384)" + "notes": "1 file(s): supabase/tests/database/tenant_isolation.sql" }, { - "name": "HNSW index on the embedding column", + "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" + "notes": "10 passed, 0 failed" }, { - "name": "index operator class matches the search operator", + "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" - }, - { - "name": "user A search returns only own sections, best match first", - "passed": true - }, + "judgeNotes": "Correctly identifies the `posts` read policy as allowing cross-organization access, fixes it, and validates tenant isolation with passing pgTAP tests." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-tests-001-rls-tenant-isolation/run-2/result.json" + }, + { + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-luna", + "reasoningEffort": "medium" + }, + "eval": "build-tests-001-rls-tenant-isolation", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "tests", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ { - "name": "user B search returns only own sections, best match first", - "passed": true + "name": "pgTAP test file(s) written under supabase/tests/", + "passed": true, + "notes": "1 file(s): supabase/tests/database/tenant_isolation_test.sql" }, { - "name": "user A reads only own sections through the API", - "passed": true + "name": "pgTAP isolation tests ran and pass", + "passed": true, + "notes": "2 passed, 4 failed" }, { - "name": "user A reads only own documents through the API", - "passed": true + "name": "agent correctly identifies the posts isolation bug from test results", + "passed": true, + "judgeNotes": "Correctly identifies `posts` as having broken tenant isolation, grounds this in the pgTAP results, and correctly states that `notes` passes." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase semantic search gte-small vector dimensions pgvector match_documents RPC RLS\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-columns", - "title": "Vector columns" - }, - { - "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors", - "title": "Querying Vectors" - }, - { - "url": "https://supabase.com/docs/guides/ai/examples/headless-vector-search", - "title": "Adding generative Q&A for your documentation" - }, - { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - } - ], - "resultChars": 115085 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i \\\"breaking-change|vector|pgvector|edge function|row level|rls\\\" | sed -n '1,160p'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 21808 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"pgvector HNSW vector_ip_ops normalized embeddings RLS database functions security invoker auth.uid Supabase\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/ai", - "title": "AI & Vectors" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-columns", - "title": "Vector columns" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-indexes/hnsw-indexes", - "title": "HNSW indexes" - } - ], - "resultChars": 81432 - } - ] + "calls": [] }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "run": 1, - "sourcePath": "codex-gpt-5.6/build-vectors-001-rag-with-permissions/run-1/result.json" + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-tests-001-rls-tenant-isolation/run-3/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-vectors-001-rag-with-permissions", @@ -43280,188 +39044,61 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" + "passed": false, + "notes": "no embedding column" }, { "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops) WHERE (embedding IS NOT NULL)" + "passed": false, + "notes": "no index on embedding column" }, { "name": "index operator class matches the search operator", - "passed": true, - "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops) WHERE (embedding IS NOT NULL)" + "passed": false, + "notes": "match_document_sections not found" }, { "name": "user A search returns only own sections, best match first", - "passed": true + "passed": false, + "notes": "Could not find the function public.match_document_sections(match_count, query_embedding) in the schema cache" }, { "name": "user B search returns only own sections, best match first", - "passed": true + "passed": false, + "notes": "Could not find the function public.match_document_sections(match_count, query_embedding) in the schema cache" }, { "name": "user A reads only own sections through the API", - "passed": true + "passed": false }, { "name": "user A reads only own documents through the API", - "passed": true + "passed": false } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"gte-small embedding dimensions pgvector semantic search match_documents row level security rpc\", limit: 8) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-columns", - "title": "Vector columns" - }, - { - "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", - "title": "Choosing your Compute Add-on" - } - ], - "resultChars": 107348 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog pgvector vector breaking change Supabase", - "pages": [] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase.ai Session gte-small 384 dimensions mean_pool normalize\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", - "title": "Choosing your Compute Add-on" - }, - { - "url": "https://supabase.com/docs/guides/functions/ai-models", - "title": "Running AI Models" - }, - { - "url": "https://supabase.com/docs/guides/ai/concepts", - "title": "Concepts" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - } - ], - "resultChars": 56072 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"HNSW normalized embeddings vector_ip_ops create index extensions.vector_ip_ops iterative_scan strict_order RLS filtering\", limit: 6) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/vector-indexes/hnsw-indexes", - "title": "HNSW indexes" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/ai/going-to-prod", - "title": "Going to Production" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-indexes", - "title": "Vector indexes" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-indexes/ivf-indexes", - "title": "IVFFlat indexes" - } - ], - "resultChars": 38512 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"pgvector 0.8 hnsw.iterative_scan strict_order filtered vector search set local\", limit: 3) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/vector-indexes/hnsw-indexes", - "title": "HNSW indexes" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-indexes", - "title": "Vector indexes" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-indexes/ivf-indexes", - "title": "IVFFlat indexes" - } - ], - "resultChars": 15269 - } - ] + "calls": [] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "run": 2, - "sourcePath": "codex-gpt-5.6/build-vectors-001-rag-with-permissions/run-2/result.json" + "run": 1, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-vectors-001-rag-with-permissions/run-1/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "build-vectors-001-rag-with-permissions", @@ -43486,12 +39123,12 @@ { "name": "HNSW index on the embedding column", "passed": true, - "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" + "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { "name": "index operator class matches the search operator", "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { "name": "user A search returns only own sections, best match first", @@ -43511,430 +39148,120 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"pgvector semantic search vector columns HNSW index RPC row level security auth.uid ownership\", limit: 8) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-indexes", - "title": "Vector indexes" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-indexes/ivf-indexes", - "title": "IVFFlat indexes" - }, - { - "url": "https://supabase.com/docs/guides/ai/going-to-prod", - "title": "Going to Production" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-indexes/hnsw-indexes", - "title": "HNSW indexes" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/increase-vector-lookup-speeds-by-applying-an-hsnw-index-ohLHUM", - "title": "Increase vector lookup speeds by applying an HSNW index" - }, - { - "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors", - "title": "Querying Vectors" - } - ], - "resultChars": 80190 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- relevant best practices ---'; cat .agents/skills/supabase-postgres-best-practices/references/security-rls-basics.md; printf '\\\\n--- rls performance ---\\\\n'; cat .agents/skills/supabase-postgres-best-practices/references/security-rls-performance.md; printf '\\\\n--- privileges ---\\\\n'; cat .agents/skills/supabase-postgres-best-practices/references/security-privileges.md; printf '\\\\n--- vector/index guidance ---\\\\n'; cat .agents/skills/supabase-postgres-best-practices/references/query-index-types.md; printf '\\\\n--- changelog scan ---\\\\n'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|vector|rls|edge function' | head -160\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 17179 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase.ai Session gte-small embedding dimensions 384 pgvector HNSW cosine normalized\", limit: 10) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", - "title": "Choosing your Compute Add-on" - }, - { - "url": "https://supabase.com/docs/guides/ai/concepts", - "title": "Concepts" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/ai", - "title": "AI & Vectors" - }, - { - "url": "https://supabase.com/docs/guides/functions/ai-models", - "title": "Running AI Models" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-columns", - "title": "Vector columns" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-indexes/hnsw-indexes", - "title": "HNSW indexes" - } - ], - "resultChars": 122010 - } - ] + "calls": [] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "run": 3, - "sourcePath": "codex-gpt-5.6/build-vectors-001-rag-with-permissions/run-3/result.json" + "run": 2, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-vectors-001-rag-with-permissions/run-2/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", + "eval": "build-vectors-001-rag-with-permissions", + "stage": "build", "product": [ - "database" + "database", + "vectors" ], "topic": [ - "observability" + "sql", + "rls" ], "suite": "benchmark", - "interface": "cli", - "passed": false, + "interface": "mcp", + "passed": true, "checks": [ { - "name": "preserved existing app scrape job", + "name": "document_sections.embedding is vector(384)", + "passed": true, + "notes": "vector(384)" + }, + { + "name": "HNSW index on the embedding column", + "passed": true, + "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + }, + { + "name": "index operator class matches the search operator", + "passed": true, + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + }, + { + "name": "user A search returns only own sections, best match first", "passed": true }, { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "HTTPS Supabase metrics scrape uses the required path and Basic Auth password_file; the matching Compose secret is mounted, project target is rendered, and the existing app job remains." + "name": "user B search returns only own sections, best match first", + "passed": true }, { - "name": "documented live deployment and verification steps", - "passed": false, - "judgeNotes": "The secret-file command writes the literal placeholder `sb_secret_replace_me` rather than the created Secret API key, so the documented setup is mismatched and would fail authentication if followed verbatim." + "name": "user A reads only own sections through the API", + "passed": true + }, + { + "name": "user A reads only own documents through the API", + "passed": true } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"metrics Prometheus project metrics endpoint Grafana observability\", limit: 8) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } ... on CLICommandReference { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics#what-you-can-do-with-the-metrics-api", - "title": "What you can do with the Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics#choose-your-monitoring-stack", - "title": "Choose your monitoring stack" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics#additional-resources", - "title": "Additional resources" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#components", - "title": "Components" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#1-define-the-scrape-job", - "title": "1. Define the scrape job" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#collector-specific-notes", - "title": "Collector-specific notes" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#2-secure-the-credentials", - "title": "2. Secure the credentials" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#3-downstream-dashboards", - "title": "3. Downstream dashboards" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#4-alerts-and-automation", - "title": "4. Alerts and automation" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#5-multi-project-setups", - "title": "5. Multi-project setups" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#installation", - "title": "Installation" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#manual-setup", - "title": "Manual setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#prerequisites", - "title": "Prerequisites" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#1-create-a-grafana-cloud-stack", - "title": "1. Create a Grafana Cloud stack" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#2-install-the-supabase-integration-for-grafana-cloud", - "title": "2. Install the Supabase integration for Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#3-configure-the-supabase-integration", - "title": "3. Configure the Supabase integration" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#4-import-the-supabase-dashboard", - "title": "4. Import the Supabase dashboard" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#5-configure-alerts-optional", - "title": "5. Configure alerts (optional)" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#6-troubleshooting", - "title": "6. Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#architecture", - "title": "Architecture" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#1-deploy-prometheus", - "title": "1. Deploy Prometheus" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#2-deploy-grafana", - "title": "2. Deploy Grafana" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#3-import-supabase-dashboards", - "title": "3. Import Supabase dashboards" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#4-configure-alerting", - "title": "4. Configure alerting" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#5-operating-tips", - "title": "5. Operating tips" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", - "title": "How to View Database Metrics" - } - ], - "resultChars": 105401 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"set -o pipefail\nprintf '%s\\\\n' '--- changelog matches ---'\ncurl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'metrics|prometheus|breaking-change' | head -n 160 || true\nprintf '%s\\\\n' '--- repo files ---'\nrg --files -g '\"'!node_modules'\"' -g '\"'!vendor'\"' | sed -n '1,240p'\nprintf '%s\\\\n' '--- directories ---'\nfind . -maxdepth 3 -type d | sort | sed -n '1,200p'\nprintf '%s\\\\n' '--- git status ---'\ngit status --short\nprintf '%s\\\\n' '--- package/config hints ---'\nrg -n -i 'prometheus|grafana|observability|metrics|otel|datadog|victoria|mimir|scrape_config' . -g '\"'!node_modules'\"' -g '\"'!vendor'\"' -g '\"'!*.lock'\"' | head -n 300 || true\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 21315 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Grafana self-hosted Prometheus Supabase metrics basic_auth password secret API key scrape config\", limit: 5) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", + "query": "query { searchDocs(query: \"Supabase pgvector semantic search match function row level security auth.uid vector extension\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#architecture", - "title": "Architecture" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#1-deploy-prometheus", - "title": "1. Deploy Prometheus" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#2-deploy-grafana", - "title": "2. Deploy Grafana" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#3-import-supabase-dashboards", - "title": "3. Import Supabase dashboards" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#4-configure-alerting", - "title": "4. Configure alerting" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#5-operating-tips", - "title": "5. Operating tips" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#installation", - "title": "Installation" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#manual-setup", - "title": "Manual setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#prerequisites", - "title": "Prerequisites" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#1-create-a-grafana-cloud-stack", - "title": "1. Create a Grafana Cloud stack" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#2-install-the-supabase-integration-for-grafana-cloud", - "title": "2. Install the Supabase integration for Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#3-configure-the-supabase-integration", - "title": "3. Configure the Supabase integration" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#4-import-the-supabase-dashboard", - "title": "4. Import the Supabase dashboard" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#5-configure-alerts-optional", - "title": "5. Configure alerts (optional)" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#6-troubleshooting", - "title": "6. Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#components", - "title": "Components" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#1-define-the-scrape-job", - "title": "1. Define the scrape job" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#collector-specific-notes", - "title": "Collector-specific notes" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#2-secure-the-credentials", - "title": "2. Secure the credentials" + "url": "https://supabase.com/docs/guides/database/extensions/pgvector", + "title": "pgvector: Embeddings and vector similarity" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#3-downstream-dashboards", - "title": "3. Downstream dashboards" + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#4-alerts-and-automation", - "title": "4. Alerts and automation" + "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors", + "title": "Querying Vectors" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#5-multi-project-setups", - "title": "5. Multi-project setups" + "url": "https://supabase.com/docs/guides/ai/vector-columns", + "title": "Vector columns" } ], - "resultChars": 79022 + "resultChars": 62104 } ] }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "run": 1, - "sourcePath": "codex-gpt-5.6/deploy-database-001-prometheus-metrics/run-1/result.json" + "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", + "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-vectors-001-rag-with-permissions/run-3/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "deploy-database-001-prometheus-metrics", @@ -43947,7 +39274,7 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { "name": "preserved existing app scrape job", @@ -43955,91 +39282,40 @@ }, { "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "Adds a deployable HTTPS Supabase metrics scrape with the correct path, project target substitution, Basic Auth password_file, matching Compose secret mount, and preserves the app scrape." + "passed": false, + "judgeNotes": "Fails: basic_auth uses an environment-expanded hardcoded secret value (`password`) rather than `password_file`, and docker-compose.yml does not mount the required password file via a volume or Compose secret." }, { "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README documents Secret API key creation, matching Docker secret file placement, Compose startup/recreation, and concrete Prometheus target/PromQL verification." + "passed": false, + "judgeNotes": "README uses an environment variable instead of instructing users to place the matching secret file required by the rubric. Verification is concrete, but the required secret-file setup is missing." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase project Prometheus metrics endpoint customer v1 privileged metrics service role basic auth\", limit: 8) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/security/security-testing", - "title": "Security testing of your Supabase projects" - }, - { - "url": "https://supabase.com/docs/guides/platform/temporary-access", - "title": "Temporary access" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", - "title": "Scrape a project's metrics" - } - ], - "resultChars": 45771 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- CHANGELOG MATCHES ---' && curl -fsSL https://supabase.com/changelog.md | rg -i -n -C 2 'metrics|prometheus|api key|breaking-change' | head -160 || true\nprintf '%s\\\\n' '--- ROOT FILES ---' && find . -maxdepth 2 -type f -o -type d | sort\nprintf '%s\\\\n' '--- IGNORE FILES ---' && find . -name .gitignore -o -name .dockerignore | sort | xargs -r -n1 sh -c 'echo --- \"'$0; cat \"$0\"'\"'\nprintf '%s\\\\n' '--- PROMETHEUS VERSION HELP ---' && docker run --rm prom/prometheus:v3.7.3 --version\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 1674 + "source": "web_search", + "query": "site:supabase.com/docs metrics Prometheus project metrics endpoint ...", + "pages": [] } ] }, "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "run": 2, - "sourcePath": "codex-gpt-5.6/deploy-database-001-prometheus-metrics/run-2/result.json" + "run": 1, + "sourcePath": "codex-gpt-5.6-luna-no-skills/deploy-database-001-prometheus-metrics/run-1/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "deploy-database-001-prometheus-metrics", @@ -44052,7 +39328,7 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { "name": "preserved existing app scrape job", @@ -44060,292 +39336,103 @@ }, { "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "HTTPS Supabase scrape is correctly configured with the required metrics path, Basic Auth password_file, project target placeholder, preserved app job, and matching Compose secret mount." + "passed": false, + "judgeNotes": "Supabase uses basic_auth.password from an environment variable, not the required password_file. docker-compose.yml also does not mount a password file via a volume or Compose secret." }, { "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README provides correct Secret API key creation and file placement, Compose startup/recreation and Prometheus reload steps, plus concrete verification via the targets page and `up{job=\"supabase\"}`." + "passed": false, + "judgeNotes": "The setup will not go live: Prometheus does not expand `${...}` variables in prometheus.yml from Compose `env_file`. The target and Basic Auth credentials remain literal placeholders. Use rendered config or Docker secrets/password_file, document the matching secret file, and recreate Prometheus." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] + "available": [], + "loaded": [] }, "docs": { "calls": [ { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- SKILL REMAINDER ---'; sed -n '1,260p' .agents/skills/supabase/SKILL.md; printf '%s\\\\n' '--- GIT ---'; git status --short; printf '%s\\\\n' '--- OBS COMPOSE ---'; sed -n '1,240p' observability/docker-compose.yml; printf '%s\\\\n' '--- PROM ---'; sed -n '1,240p' observability/prometheus.yml; printf '%s\\\\n' '--- DATASOURCE ---'; sed -n '1,200p' observability/grafana/provisioning/datasources/prometheus.yml; printf '%s\\\\n' '--- README ---'; sed -n '1,260p' observability/README.md; printf '%s\\\\n' '--- ROOT FILES ---'; find . -maxdepth 3 -type f -not -path './.git/*' | sort | sed -n '1,240p'; printf '%s\\\\n' '--- CHANGELOG INDEX ---'; curl -fsSL https://supabase.com/changelog.md | sed -n '1,220p'\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 26324 + "source": "web_search", + "query": "site:supabase.com/docs metrics Prometheus Supabase project metrics endpoint ...", + "pages": [] }, { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase project metrics Prometheus endpoint bearer authentication project reference service role metrics\", limit: 8) { nodes { __typename title href content } } }", - "hasContent": true, + "source": "web_search", + "query": "https://supabase.com/docs/guides/observability/metrics/grafana-self-hosted.md", "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", - "title": "Scrape a project's metrics" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/database/connection-management", - "title": "Connection management" - }, - { - "url": "https://supabase.com/docs/guides/storage/analytics/connecting-to-analytics-bucket", - "title": "Iceberg Catalog" + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-self-hosted.md" } - ], - "resultChars": 38653 + ] } ] }, "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "run": 3, - "sourcePath": "codex-gpt-5.6/deploy-database-001-prometheus-metrics/run-3/result.json" + "run": 2, + "sourcePath": "codex-gpt-5.6-luna-no-skills/deploy-database-001-prometheus-metrics/run-2/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, - "eval": "deploy-functions-001-edge-function-secrets", + "eval": "deploy-database-001-prometheus-metrics", "stage": "deploy", "product": [ - "edge-functions" + "database" ], "topic": [ - "security" + "observability" ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { - "name": "WEATHER_API_KEY is set as a Function secret on the project", + "name": "preserved existing app scrape job", "passed": true }, { - "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" - }, - { - "name": "the weather function reads WEATHER_API_KEY from the environment", + "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "Meets all requirements: preserves the app scrape, uses the correct Supabase HTTPS endpoint and project target, configures HTTP Basic Auth with password_file, and mounts the matching file via a Compose secret." }, { - "name": "WEATHER_API_KEY value is not committed to the repo", - "passed": true + "name": "documented live deployment and verification steps", + "passed": false, + "judgeNotes": "Wrong Supabase Metrics API authentication: Basic Auth username should be `service_role`, not `user`. This mismatch prevents the documented Secret API key setup from working." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions deploy secrets environment variables Deno.env WEATHER_API_KEY invoke CORS verify_jwt config.toml\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/slack-bot-mention", - "title": "Slack Bot Mention Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - } - ], - "resultChars": 74113 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"set -e\nprintf '%s\\\\n' '--- CHANGELOG TAG CHECK ---'\ncurl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|edge function|functions|secret' | head -80 || true\nprintf '%s\\\\n' '--- ROOT ---'\nls -la\nprintf '%s\\\\n' '--- TREE ---'\nfind . -maxdepth 6 -type f -not -path './.git/*' | sort\nprintf '%s\\\\n' '--- ENV KEYS ONLY ---'\nsed -n 's/\"'^'\"\\\\([A-Za-z_][A-Za-z0-9_]*\\\\)=.*/\\\\1/p' .env\nprintf '%s\\\\n' '--- GIT ---'\ngit status --short && git branch --show-current\nprintf '%s\\\\n' '--- PROJECT REF ---'\ncat supabase/.temp/project-ref\nprintf '%s\\\\n' '--- CLI ---'\nsupabase --version\nprintf '%s\\\\n' '--- FUNCTIONS HELP ---'\nsupabase functions deploy --help | sed -n '1,180p'\nprintf '%s\\\\n' '--- SECRETS HELP ---'\nsupabase secrets set --help | sed -n '1,160p'\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3473 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions CORS browser example public function no verify JWT config.toml function secrets set env file deploy invoke\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/cors", - "title": "CORS (Cross-Origin Resource Sharing) support for Invoking from the browser" - }, - { - "url": "https://supabase.com/docs/guides/ai/examples/openai", - "title": "Generating OpenAI GPT3 completions" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/github-actions", - "title": "GitHub Actions" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/discord-bot", - "title": "Building a Discord Bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - } - ], - "resultChars": 75316 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"functions.supabase.co Project not specified Edge Functions URL hostname project ref\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/analytics/connecting-to-analytics-bucket", - "title": "Iceberg Catalog" - }, - { - "url": "https://supabase.com/docs/guides/platform/custom-domains", - "title": "Custom Domains" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/transfer-edge-function-from-one-project-to-another", - "title": "Transfer edge functions from one project to another" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - } - ], - "resultChars": 89592 - }, { "source": "web_search", - "query": "site:github.com/supabase \"Project not specified\" functions.supabase.co", + "query": "site:supabase.com/docs metrics Prometheus customer/v1/privileged/metrics Supabase", "pages": [] } ] }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "run": 1, - "sourcePath": "codex-gpt-5.6/deploy-functions-001-edge-function-secrets/run-1/result.json" + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", + "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-5.6-luna-no-skills/deploy-database-001-prometheus-metrics/run-3/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "deploy-functions-001-edge-function-secrets", @@ -44363,94 +39450,41 @@ { "name": "WEATHER_API_KEY is set as a Function secret on the project", "passed": true - }, - { - "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" - }, - { - "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment via Deno.env.get(\"WEATHER_API_KEY\")." - }, - { - "name": "WEATHER_API_KEY value is not committed to the repo", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions secrets environment variables supabase secrets set env file deploy CORS invoke\", limit: 8) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/storage-caching", - "title": "Integrating with Supabase Storage" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", - "title": "Transcription Telegram Bot" - } - ], - "resultChars": 68207 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- root ---'\nls -la\nprintf '%s\\\\n' '--- directories ---'\nfind . -maxdepth 3 -type d -not -path './.git/*' | sort | sed -n '1,240p'\nprintf '%s\\\\n' '--- key project files including hidden ---'\nfind . -maxdepth 4 -type f \\\\( -name '.env' -o -name 'config.toml' -o -name 'package.json' -o -name 'deno.json' -o -name '*.tsx' -o -name '*.ts' -o -name '*.js' -o -name '*.html' -o -name 'project-ref' \\\\) -print | sort | sed -n '1,300p'\nprintf '%s\\\\n' '--- supabase version ---'\nsupabase --version\nprintf '%s\\\\n' '--- env key names ---'\nfind . -maxdepth 3 -name '.env' -type f -print -exec sh -c 'sed -n \\\"s/\"'^'\"\\\\([A-Za-z_][A-Za-z0-9_]*\\\\)=.*/ \\\\1/p\\\" \\\"\"'$1\"'\"' _ {} \\\\;\nprintf '%s\\\\n' '--- supabase auth/project environment names ---'\nenv | sed -n 's/\"'^'\"\\\\(SUPABASE_[A-Za-z0-9_]*\\\\)=.*/\\\\1/p' | sort\nprintf '%s\\\\n' '--- changelog relevant entries ---'\ncurl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|edge function|functions|secret' | head -100 || true\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 18819 - } - ] + }, + { + "name": "the weather function is deployed to the project", + "passed": true, + "notes": "status ACTIVE" + }, + { + "name": "the weather function reads WEATHER_API_KEY from the environment", + "passed": true, + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment via Deno.env.get(\"WEATHER_API_KEY\")." + }, + { + "name": "WEATHER_API_KEY value is not committed to the repo", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "run": 2, - "sourcePath": "codex-gpt-5.6/deploy-functions-001-edge-function-secrets/run-2/result.json" + "run": 1, + "sourcePath": "codex-gpt-5.6-luna-no-skills/deploy-functions-001-edge-function-secrets/run-1/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "deploy-functions-001-edge-function-secrets", @@ -44477,7 +39511,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -44485,194 +39519,76 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions environment variables secrets deploy functions invoke verify_jwt config.toml\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" - } - ], - "resultChars": 30712 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"set -e\ncurl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|edge function|secret' | head -120 || true\nprintf '\\\\n--- CLI VERSION ---\\\\n'\nsupabase --version\nprintf '\\\\n--- FUNCTIONS HELP ---\\\\n'\nsupabase functions --help\nprintf '\\\\n--- DEPLOY HELP ---\\\\n'\nsupabase functions deploy --help\nprintf '\\\\n--- SECRETS HELP ---\\\\n'\nsupabase secrets --help\nprintf '\\\\n--- SET HELP ---\\\\n'\nsupabase secrets set --help\nprintf '\\\\n--- PROJECT REF ---\\\\n'\ncat supabase/.temp/project-ref\nprintf '\\\\n--- ROOT ENV NAMES ---\\\\n'\nsed -E 's/\"'^([A-Za-z_][A-Za-z0-9_]*)=.*/'\"\\\\1=/' .env\nprintf '\\\\n--- SUPABASE TREE ---\\\\n'\nfind supabase -maxdepth 3 -type f -print | sort\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 7409 - } - ] + "calls": [] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "run": 3, - "sourcePath": "codex-gpt-5.6/deploy-functions-001-edge-function-secrets/run-3/result.json" + "run": 2, + "sourcePath": "codex-gpt-5.6-luna-no-skills/deploy-functions-001-edge-function-secrets/run-2/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, - "eval": "deploy-self-hosting-001-docker-compose", + "eval": "deploy-functions-001-edge-function-secrets", "stage": "deploy", "product": [ - "database", - "auth", - "storage" + "edge-functions" ], "topic": [ - "self-hosting" + "security" ], "suite": "benchmark", "interface": "cli", "passed": true, "checks": [ { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", + "name": "WEATHER_API_KEY is set as a Function secret on the project", "passed": true }, { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true + "name": "the weather function is deployed to the project", + "passed": true, + "notes": "status ACTIVE" }, { - "name": "secrets rotated off the shipped defaults", - "passed": true + "name": "the weather function reads WEATHER_API_KEY from the environment", + "passed": true, + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", + "name": "WEATHER_API_KEY value is not committed to the repo", "passed": true } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"self-hosting Docker compose secrets JWT_SECRET ANON_KEY SERVICE_ROLE_KEY dashboard\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - } - ], - "resultChars": 181034 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Self-Hosting with Docker configuring securing Supabase production reverse proxy .env generate keys 2026 Envoy\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", - "title": "Configure Reverse Proxy and HTTPS" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting", - "title": "Self-Hosting" - } - ], - "resultChars": 88923 - } - ] + "calls": [] }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "run": 1, - "sourcePath": "codex-gpt-5.6/deploy-self-hosting-001-docker-compose/run-1/result.json" + "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", + "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-5.6-luna-no-skills/deploy-functions-001-edge-function-secrets/run-3/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "deploy-self-hosting-001-docker-compose", @@ -44707,80 +39623,30 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"self-hosting Docker compose secrets JWT_SECRET ANON_KEY SERVICE_ROLE_KEY POSTGRES_PASSWORD DASHBOARD_PASSWORD POOLER_TENANT_ID\", limit: 8) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/remove-superuser-access", - "title": "Remove Superuser Access from Studio" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", - "title": "Upgrade to Postgres 17" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - } - ], - "resultChars": 156613 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"set -euo pipefail\nrm -rf /tmp/supabase-upstream\ngit clone --depth 1 --filter=blob:none --sparse https://github.com/supabase/supabase.git /tmp/supabase-upstream >/tmp/supabase-clone.log 2>&1\ngit -C /tmp/supabase-upstream sparse-checkout set docker\nprintf '%s\\\\n' '--- upstream revision ---'\ngit -C /tmp/supabase-upstream rev-parse HEAD\ngit -C /tmp/supabase-upstream log -1 --format='%cI %s'\nprintf '%s\\\\n' '--- docker files ---'\nfind /tmp/supabase-upstream/docker -maxdepth 3 -type f | sort | sed -n '1,180p'\nprintf '%s\\\\n' '--- changelog recent headings ---'\ncurl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|self-host|docker' | head -60 || true\nprintf '%s\\\\n' '--- docker changelog top ---'\nsed -n '1,180p' /tmp/supabase-upstream/docker/CHANGELOG.md\nprintf '%s\\\\n' '--- env template ---'\nsed -n '1,260p' /tmp/supabase-upstream/docker/.env.example\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 34524 + "source": "web_search", + "query": "site:supabase.com/docs self-hosting Docker compose .env.example Supabase self-hosting", + "pages": [] } ] }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "run": 2, - "sourcePath": "codex-gpt-5.6/deploy-self-hosting-001-docker-compose/run-2/result.json" + "run": 1, + "sourcePath": "codex-gpt-5.6-luna-no-skills/deploy-self-hosting-001-docker-compose/run-1/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "deploy-self-hosting-001-docker-compose", @@ -44815,310 +39681,83 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"self-hosting Docker compose install secrets JWT_SECRET ANON_KEY SERVICE_ROLE_KEY POSTGRES_PASSWORD DASHBOARD_PASSWORD POOLER_TENANT_ID\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", - "title": "Upgrade to Postgres 17" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/remove-superuser-access", - "title": "Remove Superuser Access from Studio" - } - ], - "resultChars": 155475 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"set -e\nprintf '%s\\\\n' '--- git status ---'\ngit status --short --branch 2>/dev/null || true\nprintf '%s\\\\n' '--- current Supabase changelog tags ---'\ncurl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|self-host|docker' | head -80 || true\nprintf '%s\\\\n' '--- fetch official docker bundle ---'\nrm -rf /tmp/supabase-upstream\ngit clone --depth 1 --filter=blob:none --sparse https://github.com/supabase/supabase.git /tmp/supabase-upstream >/tmp/supabase-clone.log 2>&1\ngit -C /tmp/supabase-upstream sparse-checkout set docker\nprintf 'commit: '; git -C /tmp/supabase-upstream rev-parse HEAD\nprintf 'date: '; git -C /tmp/supabase-upstream show -s --format=%cI HEAD\nprintf '%s\\\\n' '--- docker files ---'\nfind /tmp/supabase-upstream/docker -maxdepth 3 -type f | sed 's#/tmp/supabase-upstream/docker/##' | sort | head -120\nprintf '%s\\\\n' '--- compose services ---'\nrg '\"'^ [a-zA-Z0-9_-]+:'\"' /tmp/supabase-upstream/docker/docker-compose.yml | head -60\nprintf '%s\\\\n' '--- env placeholders ---'\nrg '\"'^(POSTGRES_PASSWORD|JWT_SECRET|ANON_KEY|SERVICE_ROLE_KEY|DASHBOARD_USERNAME|DASHBOARD_PASSWORD|SECRET_KEY_BASE|VAULT_ENC_KEY|PG_META_CRYPTO_KEY|LOGFLARE_API_KEY|LOGFLARE_PUBLIC_ACCESS_TOKEN|LOGFLARE_PRIVATE_ACCESS_TOKEN|POOLER_TENANT_ID|SUPABASE_PUBLISHABLE_KEY|SUPABASE_SECRET_KEY|JWT_KEYS|JWT_JWKS)='\"' /tmp/supabase-upstream/docker/.env.example\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 2583 - } - ] + "calls": [] }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "run": 3, - "sourcePath": "codex-gpt-5.6/deploy-self-hosting-001-docker-compose/run-3/result.json" + "run": 2, + "sourcePath": "codex-gpt-5.6-luna-no-skills/deploy-self-hosting-001-docker-compose/run-2/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", + "eval": "deploy-self-hosting-001-docker-compose", + "stage": "deploy", "product": [ - "auth" + "database", + "auth", + "storage" ], "topic": [ - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true - }, - { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" - }, - { - "name": "deleted user's refresh token is rejected", - "passed": true - }, - { - "name": "deleted user cannot sign back in", - "passed": true - }, - { - "name": "other users keep their sessions and access", - "passed": true - }, - { - "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "Correctly diagnoses the soft delete, deletes the Auth user and sessions, blocks stale JWTs via restrictive RLS, accurately describes residual JWT validity, and correctly distinguishes publishable and secret keys." - } + "self-hosting" ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"delete user does not sign out access token remains valid sessions revoke refresh token RLS\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - } - ], - "resultChars": 78054 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Auth delete user access token not revoked JWT deleted user\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", - "title": "Custom Claims & Role-based Access Control (RBAC)" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - } - ], - "resultChars": 156450 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"publishable key secret key legacy anon service_role RLS frontend\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - } - ], - "resultChars": 104141 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | rg -n -i \"breaking|auth|api key|publishable|secret key|session\" | head -80'", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 14562 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Delete users auth.users delete user JWT remains valid until expiry\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", - "title": "Should I set a shorter Max-Age parameter on the cookies?" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/advanced-guide", - "title": "Advanced guide" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - } - ], - "resultChars": 119961 + "suite": "benchmark", + "interface": "cli", + "passed": false, + "checks": [ + { + "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", + "passed": true + }, + { + "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", + "passed": false, + "notes": "found CLI init: supabase-docker/supabase/config.toml" + }, + { + "name": "secrets rotated off the shipped defaults", + "passed": true + }, + { + "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "site:github.com/supabase/supabase docker self-hosting docker-compose.yml self-hosted", + "pages": [] } ] }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "run": 1, - "sourcePath": "codex-gpt-5.6/investigate-auth-001-deleted-user-access/run-1/result.json" + "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", + "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-5.6-luna-no-skills/deploy-self-hosting-001-docker-compose/run-3/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "investigate-auth-001-deleted-user-access", @@ -45144,13 +39783,12 @@ }, { "name": "delete-account revokes the user's sessions", - "passed": false, - "notes": "sessions left: 1" + "passed": true, + "notes": "sessions left: 0" }, { "name": "deleted user's refresh token is rejected", - "passed": false, - "notes": "refresh token still produces a session" + "passed": true }, { "name": "deleted user cannot sign back in", @@ -45164,167 +39802,153 @@ { "name": "diagnosed and explained session revocation", "passed": false, - "judgeNotes": "The shipped flow still only soft-deletes the profile. It does not delete the Auth user or revoke sessions/refresh tokens; global sign-out/admin deletion are merely recommendations, not implemented. Therefore it fails the required real revocation despite correctly handling stale JWTs via RLS and explaining API keys." + "judgeNotes": "The flow does not actually delete auth.users or remove the user’s identities; it only sets deleted_at/banned_until and deletes sessions. This falls short of the rubric’s required auth-user deletion (or identity-and-session removal). It also does not explicitly clarify that purely local JWT validation such as getClaims/custom middleware will continue accepting the token until expiry." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"delete user existing access token session revoke sign out JWT expiry auth sessions delete_account security definer publishable secret keys RLS\", limit: 8) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Supabase auth delete user invalidate sessions JWT already issued access token deleted_at\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/getting-started", - "title": "Getting Started with OAuth 2.1 Server" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/storage/s3/authentication", - "title": "S3 Authentication" + "url": "https://supabase.com/docs/guides/auth/jwts", + "title": "JSON Web Token (JWT)" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", + "title": "OAuth 2.1 Flows" } ], - "resultChars": 89615 + "resultChars": 102612 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"publishable key secret key legacy anon service_role RLS bypass\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Supabase revoke sessions delete auth.sessions user access token RLS\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/auth/users", + "title": "Users" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", + "title": "OAuth 2.1 Flows" } ], - "resultChars": 84253 + "resultChars": 90626 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"sign out scope global revoke refresh tokens access token remains valid until expiry deleted user\", limit: 8) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Supabase RLS check deleted users auth.uid profile deleted\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" - }, - { - "url": "https://supabase.com/docs/guides/auth/signout", - "title": "Signing out" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", - "title": "Should I set a shorter Max-Age parameter on the cookies?" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/auth/server-side/advanced-guide", - "title": "Advanced guide" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" }, { - "url": "https://supabase.com/docs/reference/swift/auth-admin-signout", - "title": "signOut()" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", + "title": "Build a User Management App with SvelteKit" }, { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" + "url": "https://supabase.com/docs/guides/auth/users", + "title": "Users" } ], - "resultChars": 73644 + "resultChars": 109708 }, { - "source": "web_search", - "query": "https://supabase.com/changelog.md", + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable keys secret keys Supabase frontend RLS\", limit: 3) { nodes { title href content } } }", + "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/getting-started", + "title": "Getting Started with OAuth 2.1 Server" } - ] + ], + "resultChars": 63381 }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i -C 2 'breaking-change|api key|publishable|secret key|session|sign.?out|access token' | sed -n '1,220p'\"", + "source": "search_docs", + "query": "{ searchDocs(query: \"Supabase API keys publishable secret key RLS service_role\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" } ], - "resultChars": 9107 - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/guides/auth/signout access tokens remain valid until expiry", - "pages": [] + "resultChars": 44203 } ] }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "run": 2, - "sourcePath": "codex-gpt-5.6/investigate-auth-001-deleted-user-access/run-2/result.json" + "run": 1, + "sourcePath": "codex-gpt-5.6-luna-no-skills/investigate-auth-001-deleted-user-access/run-1/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "investigate-auth-001-deleted-user-access", @@ -45368,144 +39992,106 @@ { "name": "diagnosed and explained session revocation", "passed": false, - "judgeNotes": "The deletion and RLS fixes are sound, and key guidance is correct. However, it omits the required caveat that the stateless access JWT itself remains valid until expiry and will still pass purely local validation such as getClaims() or signature/expiry middleware." + "judgeNotes": "The fix and key guidance are correct, but the JWT caveat is incomplete: it does not explain that access tokens are stateless and remain accepted until expiry by purely local validators such as getClaims() or signature/expiry-only middleware." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"delete user does not invalidate JWT access token revoke sessions auth.sessions delete account RPC publishable secret keys RLS\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" - }, - { - "url": "https://supabase.com/docs/guides/storage/management/delete-objects", - "title": "Delete Objects" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - } - ], - "resultChars": 73779 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JWT session_id claim auth.sessions RLS revoke session access token remains valid deleted user\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/getting-started", - "title": "Getting Started with OAuth 2.1 Server" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", - "title": "Should I set a shorter Max-Age parameter on the cookies?" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" - } - ], - "resultChars": 149055 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"publishable key secret key frontend RLS bypass service_role new API keys\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"publishable key secret key frontend RLS\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, { "url": "https://supabase.com/docs/guides/database/secure-data", "title": "Securing your data" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" } ], - "resultChars": 64041 - }, + "resultChars": 27698 + } + ] + }, + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-5.6-luna-no-skills/investigate-auth-001-deleted-user-access/run-2/result.json" + }, + { + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", + "product": [ + "auth" + ], + "topic": [ + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "victim session active before delete-account", + "passed": true + }, + { + "name": "delete_account flow ran for the victim", + "passed": true + }, + { + "name": "delete-account revokes the user's sessions", + "passed": false, + "notes": "sessions left: 1" + }, + { + "name": "deleted user's refresh token is rejected", + "passed": false, + "notes": "refresh token still produces a session" + }, + { + "name": "deleted user cannot sign back in", + "passed": false, + "notes": "deleted account can still sign in" + }, + { + "name": "other users keep their sessions and access", + "passed": true + }, + { + "name": "diagnosed and explained session revocation", + "passed": false, + "judgeNotes": "The fix remains a soft delete and does not delete the Auth user, identities, sessions, or refresh tokens, so sign-in/auth access is not truly revoked. It also omits that stateless JWTs remain valid for purely local validation (for example getClaims/custom middleware) until expiry." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"API keys publishable secret keys sb_publishable sb_secret RLS role frontend\", limit: 10) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase publishable keys secret keys frontend RLS service_role anon\", limit: 5) { nodes { ... on Guide { title href content } ... on ManagementApiReference { title href content } } } }", "hasContent": true, "pages": [ { @@ -45517,54 +40103,34 @@ "title": "Build a User Management App with RedwoodJS" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { "url": "https://supabase.com/docs/guides/auth/signing-keys", "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-react", - "title": "Build a User Management App with Ionic React" - }, - { - "url": "https://supabase.com/docs/guides/api/creating-routes", - "title": "Creating API Routes" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" } ], - "resultChars": 230278 + "resultChars": 91518 } ] }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.6/investigate-auth-001-deleted-user-access/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/investigate-auth-001-deleted-user-access/run-3/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "investigate-realtime-001-subscribed-no-events", @@ -45604,82 +40170,28 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified the missing orders publication entry, added only public.orders to supabase_realtime, and preserved courier_locations, RLS, and existing policies." + "judgeNotes": "Correctly identified the missing orders table in supabase_realtime, added only public.orders, and verified courier_locations remained included without changing RLS or policies." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Realtime postgres_changes publication add table supabase_realtime replication publication INSERT events subscribed no events\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/realtime-postgres-changes-troubleshooting", - "title": "Realtime: Postgres Changes Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", - "title": "Subscribing to Database Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol", - "title": "Realtime Protocol" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" - } - ], - "resultChars": 117138 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- repo files ---'; rg --files -g '\"'!node_modules'\"' -g '\"'!vendor'\"' | sed -n '1,160p'; printf '%s\\\\n' '--- realtime references ---'; rg -n \\\"postgres_changes|courier|orders|supabase_realtime\\\" -g '\"'!node_modules'\"' -g '\"'!vendor'\"' . || true; printf '%s\\\\n' '--- changelog realtime breaking tags ---'; curl -fsSL https://supabase.com/changelog.md | rg -n -i \\\"breaking-change|realtime|postgres changes\\\" | head -80\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 32526 - } - ] + "calls": [] }, "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.6/investigate-realtime-001-subscribed-no-events/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/investigate-realtime-001-subscribed-no-events/run-1/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "investigate-realtime-001-subscribed-no-events", @@ -45719,72 +40231,28 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified orders missing from supabase_realtime, added only public.orders, and preserved courier_locations, RLS, and existing policies." + "judgeNotes": "Correctly diagnosed orders as missing from supabase_realtime, added only public.orders, and preserved courier_locations, RLS, and policies." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Realtime postgres_changes enable table supabase_realtime publication ALTER PUBLICATION\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", - "title": "Subscribing to Database Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/realtime-postgres-changes-troubleshooting", - "title": "Realtime: Postgres Changes Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/realtime/architecture", - "title": "Realtime Architecture" - } - ], - "resultChars": 82022 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - } - ] + "calls": [] }, "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.6/investigate-realtime-001-subscribed-no-events/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/investigate-realtime-001-subscribed-no-events/run-2/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "investigate-realtime-001-subscribed-no-events", @@ -45824,74 +40292,28 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly diagnosed missing publication membership, added only public.orders to the existing supabase_realtime publication, and preserved courier_locations, RLS, and policies." + "judgeNotes": "Correctly diagnosed the missing orders table in supabase_realtime, added only public.orders, and verified courier_locations remained published without changing RLS or policies." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Realtime Postgres Changes add table supabase_realtime publication ALTER PUBLICATION postgres_changes INSERT subscribed no events\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", - "title": "Subscribing to Database Changes" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/realtime-postgres-changes-troubleshooting", - "title": "Realtime: Postgres Changes Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol", - "title": "Realtime Protocol" - } - ], - "resultChars": 117138 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i \\\"realtime|postgres changes|publication|breaking-change\\\" | sed -n '1,120p'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 10703 - } - ] + "calls": [] }, "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.6/investigate-realtime-001-subscribed-no-events/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/investigate-realtime-001-subscribed-no-events/run-3/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "investigate-reliability-003-edge-function-5xx-correlation", @@ -45909,28 +40331,22 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform and explicitly documented all eight recurring HTTP 503 gateway failures from 07:00Z through 12:00Z on 2026-04-28." + "judgeNotes": "Identified image-transform and the recurring eight HTTP 503 gateway failures from 07:00Z through 12:00Z on 2026-04-28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Correctly attributes the recurring image-transform 503s to the gateway/platform layer, supported by gateway-only logs with no runtime invocations, nearby successful calls on unchanged deployment/version, and a clear distinction from avatar-upload’s function-level 500." + "judgeNotes": "Attributes image-transform 503s to the Edge gateway/dispatch layer, supported by successful nearby requests on the unchanged version/deployment and a clear distinction from avatar-upload’s function-level 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The response gives concrete actions, including escalating to Supabase Support with the project, region, exact time window, and gateway request IDs." + "judgeNotes": "Provides multiple concrete actions, including checking platform incidents for the exact time window, adding retries, instrumenting the upload function, and investigating its storage/package dependency." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { "calls": [] @@ -45938,15 +40354,15 @@ "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.6/investigate-reliability-003-edge-function-5xx-correlation/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/investigate-reliability-003-edge-function-5xx-correlation/run-1/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "investigate-reliability-003-edge-function-5xx-correlation", @@ -45964,73 +40380,38 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified `image-transform` and the recurring pattern of 8 HTTP 503 gateway failures from 07:00–12:00 UTC on 2026-04-28." + "judgeNotes": "Identified image-transform and a recurring series of seven HTTP 503 gateway failures across 07:00Z–11:30Z on 2026-04-28, covering most of the eight expected failures." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Attributes the recurring 503s to the Edge gateway/platform layer, supported by their presence only in edge logs with no function invocations, while the unchanged deployment handled nearby requests successfully. It also distinguishes avatar-upload’s function-level 500." + "judgeNotes": "Attributes the 503s to the gateway/platform before function execution, supported by their presence only in edge logs with successful function executions around them and unchanged deployment details." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The response gives concrete actions, including opening a Supabase support incident with the project ID, exact time window, gateway request IDs, region, and asking for gateway/runtime infrastructure investigation." + "judgeNotes": "Provides concrete next steps, including escalating to Supabase support with specific gateway request IDs, adding retries, and investigating the avatar-upload dependency." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions 503 gateway error logs retry deployment troubleshooting\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/debugging", - "title": "Debugging guide" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", - "title": "Edge Function 503 error response" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", - "title": "Edge Function 504 error response" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - } - ], - "resultChars": 52469 - } - ] + "calls": [] }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.6/investigate-reliability-003-edge-function-5xx-correlation/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/investigate-reliability-003-edge-function-5xx-correlation/run-2/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "investigate-reliability-003-edge-function-5xx-correlation", @@ -46043,78 +40424,43 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified `image-transform` as the affected function and quantified the recurring pattern of 8 gateway 503s across the April 28 morning window (07:00–12:00 UTC)." + "judgeNotes": "Identified image-transform and all eight recurring HTTP 503 gateway failures from 07:00Z through 12:00Z on 2026-04-28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "Correctly attributes the 503s to gateway/dispatch and cites missing runtime invocations, but explicitly recommends redeploying `image-transform`, which is a rubric-defined failure." + "passed": true, + "judgeNotes": "Attributes recurring image-transform 503s to the gateway/platform layer, supported by edge-only errors, successful nearby runtime responses, missing deployment IDs, and distinction from avatar-upload’s application-level 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Provides multiple concrete actions, including opening a Supabase incident with a specific time window and gateway request IDs, redeploying the function, and running controlled tests." + "judgeNotes": "The assistant gave multiple concrete actions, including escalating a Supabase incident with exact timestamps and gateway request IDs, verifying deployment metadata, and separately investigating the correlated function error." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Storage logs upload 500 error log fields request path status code\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/debugging/logs", - "title": "Logs" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/debugging", - "title": "Debugging guide" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/advanced-log-filtering", - "title": "Advanced Log Querying and Filtering" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" - }, - { - "url": "https://supabase.com/docs/guides/storage/debugging/error-codes", - "title": "Error Codes" - } - ], - "resultChars": 107639 - } - ] + "calls": [] }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.6/investigate-reliability-003-edge-function-5xx-correlation/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/investigate-reliability-003-edge-function-5xx-correlation/run-3/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-dataapi-001-empty-results", @@ -46159,73 +40505,28 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and added authenticated owner-scoped SELECT and INSERT policies using auth.uid() with USING/WITH CHECK." + "judgeNotes": "Correctly diagnosed deny-all RLS, kept RLS enabled, and added authenticated owner-scoped SELECT and INSERT policies using auth.uid()." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- RLS references ---'; rg --files .agents/skills/supabase-postgres-best-practices/references | rg 'security|rls' | head -20; printf '%s\\\\n' '--- relevant guidance ---'; cat .agents/skills/supabase-postgres-best-practices/references/security-rls-performance.md 2>/dev/null || true; printf '%s\\\\n' '--- changelog tags ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -n 'breaking-change|data api|row level|rls' | head -30\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 8813 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Exposing a table to the Data API grants authenticated RLS policy auth.uid() insert select\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0002_auth_users_exposed", - "title": "Database Advisor: Lint 0002_auth_users_exposed" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter", - "title": "Build a User Management App with Flutter" - } - ], - "resultChars": 116067 - } - ] + "calls": [] }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.6/resolve-dataapi-001-empty-results/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/resolve-dataapi-001-empty-results/run-1/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-dataapi-001-empty-results", @@ -46274,70 +40575,24 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking|data api|row level|rls|grant|privilege' | head -80\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 15009 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Data API expose table grants authenticated select insert row level security auth.uid policies\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - } - ], - "resultChars": 90706 - } - ] + "calls": [] }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.6/resolve-dataapi-001-empty-results/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/resolve-dataapi-001-empty-results/run-2/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-dataapi-001-empty-results", @@ -46382,68 +40637,28 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and created authenticated owner-scoped SELECT and INSERT policies using auth.uid() with USING and WITH CHECK." + "judgeNotes": "Correctly diagnosed deny-all RLS, kept RLS enabled, and added authenticated owner-scoped SELECT and INSERT policies using auth.uid() with USING and WITH CHECK." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "web_search", - "query": "site:supabase.com/changelog breaking change Data API RLS policies grants authenticated 2026", - "pages": [] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Data API explicit grants authenticated table RLS select insert auth.uid\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - } - ], - "resultChars": 82263 - } - ] + "calls": [] }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.6/resolve-dataapi-001-empty-results/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/resolve-dataapi-001-empty-results/run-3/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-database-001-migration-history-mismatch", @@ -46460,19 +40675,23 @@ "checks": [ { "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true + "passed": false, + "notes": "avatar_url not found on public.profiles" }, { "name": "migration 20240220000000 is recorded in the remote history", - "passed": true + "passed": false, + "notes": "remote history versions: [\"20240101000000\",\"20240115000000\"]" }, { "name": "remote migration history matches local migration files", - "passed": true + "passed": false, + "notes": "remote-only versions still unreconciled: [\"20240115000000\"] (remote: [\"20240101000000\",\"20240115000000\"], local: [\"20240101000000\",\"20240220000000\"])" }, { "name": "local migrations are a valid reconciled sequence", - "passed": true + "passed": false, + "notes": "expected 20240101000000_create_profiles → → 20240220000000_add_avatar_url, strictly ascending; got [20240101000000_create_profiles.sql, 20240220000000_add_avatar_url.sql]" }, { "name": "production profile data is intact (not reset)", @@ -46481,63 +40700,28 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": false, - "judgeNotes": "The orphan bio migration was addressed by adding `20240115000000_add_profile_bio.sql`, and the dry run then accepted the history. However, no real remote `supabase db push` has recorded output showing `Applying migration` or `Finished supabase db push`; only a dry run and local `supabase db reset` show application. No prohibited workaround was shown." + "judgeNotes": "#12 attempted a real `supabase db push --db-url ... --yes`, but its recorded output does not show “Applying migration” or “Finished supabase db push”. No `supabase migration repair`, `db pull`, or local bio migration reconciliation is recorded. The final `psql` query is only inspection; no prohibited mutation workaround is visible." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI db push migration history repair linked project deploy migrations\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", - "title": "Repair the migration history table" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-push", - "title": "Push new migrations to the remote database" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - } - ], - "resultChars": 61657 - } - ] + "calls": [] }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.6/resolve-database-001-migration-history-mismatch/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/resolve-database-001-migration-history-mismatch/run-1/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-database-001-migration-history-mismatch", @@ -46554,19 +40738,23 @@ "checks": [ { "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true + "passed": false, + "notes": "avatar_url not found on public.profiles" }, { "name": "migration 20240220000000 is recorded in the remote history", - "passed": true + "passed": false, + "notes": "remote history versions: [\"20240101000000\",\"20240115000000\"]" }, { "name": "remote migration history matches local migration files", - "passed": true + "passed": false, + "notes": "remote-only versions still unreconciled: [\"20240115000000\"] (remote: [\"20240101000000\",\"20240115000000\"], local: [\"20240101000000\",\"20240220000000\"])" }, { "name": "local migrations are a valid reconciled sequence", - "passed": true + "passed": false, + "notes": "expected 20240101000000_create_profiles → → 20240220000000_add_avatar_url, strictly ascending; got [20240101000000_create_profiles.sql, 20240220000000_add_avatar_url.sql]" }, { "name": "production profile data is intact (not reset)", @@ -46575,73 +40763,28 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": false, - "judgeNotes": "`supabase db push --db-url \"$DB_URL\" --yes` was run, but its recorded output does not show “Applying migration” or “Finished supabase db push,” so successful CLI application is not established by the rubric. The orphan bio migration was reconciled locally by adding `20240115000000_add_profile_bio.sql`; no prohibited workaround was shown." + "judgeNotes": "No migration was applied or history reconciled. `supabase db push` in #3 and #6 failed; #10 only reported the database was up to date, with no “Applying migration” or successful application evidence. No `supabase migration repair` or `supabase db pull` was run. No prohibited mutation workaround was observed." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"set -o pipefail\nprintf '%s\\\\n' '--- workspace files ---'\nrg --files -g 'AGENTS.md' -g '\"'!node_modules'\"' -g '\"'!vendor'\"' | sort\nprintf '%s\\\\n' '--- git status ---'\ngit status --short --branch\nprintf '%s\\\\n' '--- supabase files ---'\nfind supabase -maxdepth 3 -type f -print 2>/dev/null | sort\nprintf '%s\\\\n' '--- CLI version ---'\nsupabase --version\nprintf '%s\\\\n' '--- project metadata ---'\nfind supabase/.temp -maxdepth 1 -type f -print -exec sh -c 'printf \\\" \\\"; cat \\\"\"'$1\"; printf \"'\"\\\\n\\\"' _ {} \\\\; 2>/dev/null || true\nprintf '%s\\\\n' '--- config schema paths ---'\nrg -n 'schema_paths|project_id|major_version' supabase/config.toml 2>/dev/null || true\nprintf '%s\\\\n' '--- changelog breaking changes ---'\ncurl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking.change|migration|database' | head -80\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 14804 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI deploy migrations hosted project db push migration list repair remote history\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", - "title": "Repair the migration history table" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-push", - "title": "Push new migrations to the remote database" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-migration-list", - "title": "List local and remote migrations" - } - ], - "resultChars": 46165 - } - ] + "calls": [] }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.6/resolve-database-001-migration-history-mismatch/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/resolve-database-001-migration-history-mismatch/run-2/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-database-001-migration-history-mismatch", @@ -46654,7 +40797,7 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "the avatar_url column is applied on the hosted profiles table", @@ -46678,73 +40821,29 @@ }, { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": false, - "judgeNotes": "The only evidenced successful push was `supabase db push --dry-run`, which does not count. The real push in #11 has no recorded “Applying migration” or “Finished supabase db push” output. Adding `20240115000000_add_profile_bio.sql` locally reconciled the orphan for the dry run, but actual application/reconciliation by a successful push is unproven. No prohibited mutation workaround was shown; the `psql` use was read-only." + "passed": true, + "judgeNotes": "`supabase db push --db-url ...` applied `20240220000000_add_avatar_url.sql` and reported both “Applying migration” and “Finished supabase db push.” The orphan history was reconciled by adding local `20240115000000_add_profile_bio.sql`, then running that push. The `psql` commands were read-only inspections; no prohibited workaround was used." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- files ---'; rg --files -g 'AGENTS.md' -g 'supabase/**' -g '\"'!supabase/.temp/**'\"' -g '.gitignore' -g 'README*' | sort; printf '%s\\\\n' '--- cli ---'; supabase --version; printf '%s\\\\n' '--- status ---'; git status --short --branch; printf '%s\\\\n' '--- config ---'; sed -n '1,240p' supabase/config.toml 2>/dev/null || true; printf '%s\\\\n' '--- skill deploy guidance ---'; rg -n -C 5 'deploy|push|remote|migration repair|migration list|linked' .claude/skills/supabase/SKILL.md; printf '%s\\\\n' '--- changelog breaking changes ---'; curl -fsSL https://supabase.com/changelog.md | rg -n -i -m 20 'breaking.change|migration|database|cli' || true\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3433 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI link IPv4 pooler db push migration list linked IPv6 not supported\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/replication/manual-replication-faq", - "title": "Manual replication FAQ" - }, - { - "url": "https://supabase.com/docs/guides/platform/ipv4-address", - "title": "Dedicated IPv4 Address for Ingress" - }, - { - "url": "https://supabase.com/docs/guides/functions/limits", - "title": "Limits" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres", - "title": "Connect to your database" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/supabase-cli-failed-sasl-auth-or-invalid-scram-server-final-message" - } - ], - "resultChars": 31863 - } - ] + "calls": [] }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.6/resolve-database-001-migration-history-mismatch/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/resolve-database-001-migration-history-mismatch/run-3/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-performance-001-slow-query-cpu-spike", @@ -46758,94 +40857,49 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", - "passed": true - }, - { - "name": "created index covering user_id and created_at", - "passed": true - }, - { - "name": "query plan uses an index and avoids sequential scan", - "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" - }, - { - "name": "inserts still work", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- relevant rules ---'; for f in .agents/skills/supabase-postgres-best-practices/references/query-missing-indexes.md .agents/skills/supabase-postgres-best-practices/references/query-composite-indexes.md .agents/skills/supabase-postgres-best-practices/references/query-explain-analyze.md .agents/skills/supabase-postgres-best-practices/references/query-index-types.md; do if [ -f \\\"\"'$f\" ]; then echo \"### $f\"; cat \"$f\"; fi; done; printf '\"'%s\\\\n' '--- changelog performance tags ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|postgres|index|performance' | head -n 120\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 12839 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"database query optimization index advisor pg_stat_statements EXPLAIN ANALYZE indexes\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" - }, - { - "url": "https://supabase.com/docs/guides/database/inspect", - "title": "Database debugging and monitoring" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/indexes", - "title": "Managing Indexes in Postgres" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" - }, - { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod", - "title": "Production Checklist" - } - ], - "resultChars": 43223 - } - ] + "passed": false, + "checks": [ + { + "name": "inspected pg_stat_statements for query performance", + "passed": true + }, + { + "name": "ran EXPLAIN on the expensive query", + "passed": false + }, + { + "name": "created index covering user_id and created_at", + "passed": true + }, + { + "name": "query plan uses an index and avoids sequential scan", + "passed": true, + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + }, + { + "name": "inserts still work", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] }, "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.6/resolve-performance-001-slow-query-cpu-spike/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/resolve-performance-001-slow-query-cpu-spike/run-1/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-performance-001-slow-query-cpu-spike", @@ -46876,7 +40930,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -46884,69 +40938,24 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- files ---'; rg --files -g 'AGENTS.md' -g 'supabase/**' -g '\"'!node_modules'\"' | head -200; printf '%s\\\\n' '--- query index guidance ---'; cat .agents/skills/supabase-postgres-best-practices/references/query-missing-indexes.md; if [ -f .agents/skills/supabase-postgres-best-practices/references/query-composite-indexes.md ]; then cat .agents/skills/supabase-postgres-best-practices/references/query-composite-indexes.md; fi; printf '%s\\\\n' '--- changelog tags ---'; curl -fsSL https://supabase.com/changelog.md | rg -i 'breaking-change|postgres|index' | head -40\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 10257 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"database query optimization indexes EXPLAIN pg_stat_statements\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/inspect", - "title": "Database debugging and monitoring" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/hypopg", - "title": "HypoPG: Hypothetical indexes" - } - ], - "resultChars": 32466 - } - ] + "calls": [] }, "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.6/resolve-performance-001-slow-query-cpu-spike/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/resolve-performance-001-slow-query-cpu-spike/run-2/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-performance-001-slow-query-cpu-spike", @@ -46977,7 +40986,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -46985,79 +40994,24 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Postgres EXPLAIN ANALYZE index multicolumn ORDER BY LIMIT query performance\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/orioledb", - "title": "OrioleDB Overview" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/understanding-postgresql-explain-output-Un9dqX", - "title": "Understanding Postgres EXPLAIN Output" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/running-explain-analyze-on-functions", - "title": "Running EXPLAIN ANALYZE on functions" - }, - { - "url": "https://supabase.com/docs/guides/database/inspect", - "title": "Database debugging and monitoring" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" - } - ], - "resultChars": 43484 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | rg -n -i \"breaking-change|postgres|index|query\" | head -40'", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 7663 - } - ] + "calls": [] }, "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.6/resolve-performance-001-slow-query-cpu-spike/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/resolve-performance-001-slow-query-cpu-spike/run-3/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-security-002-rls-cross-tenant-leak", @@ -47112,79 +41066,24 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security multi tenant workspace membership policies auth.uid notes\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - }, - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - } - ], - "resultChars": 79153 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -i -n -m 20 'breaking-change|row level|rls|policy' || true\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 4630 - } - ] + "calls": [] }, "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-5.6/resolve-security-002-rls-cross-tenant-leak/run-1/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/resolve-security-002-rls-cross-tenant-leak/run-1/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-security-002-rls-cross-tenant-leak", @@ -47239,69 +41138,24 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- files ---'; rg --files -g 'AGENTS.md' -g 'supabase/**' -g '*.sql' -g 'package.json' -g 'config.toml' | sed -n '1,200p'; printf '%s\\\\n' '--- changelog tags ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -m 20 'breaking-change|rls|policy|postgres' || true\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3778 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security multi tenant workspace membership policies auth.uid exists select\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - } - ], - "resultChars": 68601 - } - ] + "calls": [] }, "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-5.6/resolve-security-002-rls-cross-tenant-leak/run-2/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/resolve-security-002-rls-cross-tenant-leak/run-2/result.json" }, { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "modelId": "gpt-5.6-luna", "reasoningEffort": "medium" }, "eval": "resolve-security-002-rls-cross-tenant-leak", @@ -47356,70 +41210,16 @@ } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security multi tenant workspace membership policies auth.uid security definer views\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } totalCount } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/tables", - "title": "Tables and Data" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - } - ], - "resultChars": 98931 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,160p'; printf '\\\\n--- workspace files ---\\\\n'; find . -maxdepth 3 -type f | sort | sed -n '1,200p'\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 9213 - } - ] + "calls": [] }, "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-5.6/resolve-security-002-rls-cross-tenant-leak/run-3/result.json" + "sourcePath": "codex-gpt-5.6-luna-no-skills/resolve-security-002-rls-cross-tenant-leak/run-3/result.json" }, { "experiment": "codex-gpt-5.6-no-skills", @@ -47453,7 +41253,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 3133ee58-0b09-4a05-ae72-7bcd7a3c0475, signUp returned {\"userId\":\"3133ee58-0b09-4a05-ae72-7bcd7a3c0475\"}" + "notes": "db user 8f782757-4ed0-4974-90db-112f128e5e31, signUp returned {\"userId\":\"8f782757-4ed0-4974-90db-112f128e5e31\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -47468,7 +41268,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"3133ee58-0b09-4a05-ae72-7bcd7a3c0475\"}" + "notes": "{\"userId\":\"8f782757-4ed0-4974-90db-112f128e5e31\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -47491,99 +41291,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js JavaScript createClient signUp user metadata signInWithPassword getSession select single profiles\", limit: 6) { nodes { __typename title href content ... on ClientLibraryFunctionReference { language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-linkedin", - "title": "Login with LinkedIn" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", - "title": "Login with Figma" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-slack", - "title": "Login with Slack" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-bitbucket", - "title": "Login with Bitbucket" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-gitlab", - "title": "Login with GitLab" - } - ], - "resultChars": 114732 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"auth.signUp JavaScript options data metadata\", limit: 5) { nodes { __typename title href content ... on ClientLibraryFunctionReference { language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signup" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinanonymously" - } - ], - "resultChars": 6182 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript signInWithPassword getUser select single row supabase-js\", limit: 8) { nodes { __typename title href content ... on ClientLibraryFunctionReference { language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", - "title": "Login with Google" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - } - ], - "resultChars": 164392 - } - ] + "calls": [] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", @@ -47622,7 +41330,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user a1bc487f-5318-4517-bfdd-da240ae726fb, signUp returned {\"userId\":\"a1bc487f-5318-4517-bfdd-da240ae726fb\"}" + "notes": "db user 63d60eb1-d253-4e2e-8c02-3a434c35f651, signUp returned {\"userId\":\"63d60eb1-d253-4e2e-8c02-3a434c35f651\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -47637,7 +41345,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"a1bc487f-5318-4517-bfdd-da240ae726fb\"}" + "notes": "{\"userId\":\"63d60eb1-d253-4e2e-8c02-3a434c35f651\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -47660,48 +41368,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createClient signUp user metadata signInWithPassword auth session profiles row level security select single\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/enterprise-sso", - "title": "Enterprise Single Sign-On" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" - } - ], - "resultChars": 115593 - } - ] + "calls": [] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", @@ -47740,7 +41407,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 4b971d58-a8aa-42af-a5c3-672eda5af181, signUp returned {\"userId\":\"4b971d58-a8aa-42af-a5c3-672eda5af181\"}" + "notes": "db user e98aa4f4-94d4-43ad-93bd-5a37c89ed4a1, signUp returned {\"userId\":\"e98aa4f4-94d4-43ad-93bd-5a37c89ed4a1\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -47754,101 +41421,31 @@ }, { "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"4b971d58-a8aa-42af-a5c3-672eda5af181\"}" - }, - { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" - }, - { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/src/auth.mjs" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createClient signUp options data signInWithPassword getSession select single auth user metadata browser\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", - "title": "Login with Google" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-azure", - "title": "Login with Azure (Microsoft)" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", - "title": "Login with Figma" - } - ], - "resultChars": 128258 - }, - { - "source": "search_docs", - "query": "query { signUp: searchDocs(query: \"JavaScript auth signUp email password options data user metadata\", limit: 3) { nodes { title href content } } signIn: searchDocs(query: \"JavaScript signInWithPassword email password\", limit: 3) { nodes { title href content } } select: searchDocs(query: \"JavaScript select single row Supabase\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", - "title": "Build a User Management App with React" - }, - { - "url": "https://supabase.com/docs/guides/api/sql-to-api", - "title": "Converting SQL to JavaScript API" - }, - { - "url": "https://supabase.com/docs/guides/api/automatic-retries-in-supabase-js", - "title": "How to do automatic retries with `supabase-js`" - } - ], - "resultChars": 104046 - } - ] + "passed": true, + "notes": "{\"userId\":\"e98aa4f4-94d4-43ad-93bd-5a37c89ed4a1\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", @@ -47915,36 +41512,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Row Level Security authenticated users SELECT policy anon no rows migrations seed local development\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - } - ], - "resultChars": 95821 - } - ] + "calls": [] }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", @@ -48011,36 +41579,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase row level security authenticated role select policy migrations seed data local development\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - } - ], - "resultChars": 43733 - } - ] + "calls": [] }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", @@ -48107,36 +41646,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Row Level Security authenticated users select policy anon denied migrations seed data local development\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" - } - ], - "resultChars": 94489 - } - ] + "calls": [] }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", @@ -48163,11 +41673,11 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "supabase db diff used to generate the migration", - "passed": false + "passed": true }, { "name": "schema file updated to include description column", @@ -48187,28 +41697,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI local development database migrations alter table add column\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - } - ], - "resultChars": 54403 - } - ] + "calls": [] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", @@ -48259,28 +41748,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI local database migration add column table\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - } - ], - "resultChars": 54403 - } - ] + "calls": [] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", @@ -48311,497 +41779,32 @@ "checks": [ { "name": "supabase db diff used to generate the migration", - "passed": false - }, - { - "name": "schema file updated to include description column", - "passed": true - }, - { - "name": "a new migration was generated for the change", - "passed": true - }, - { - "name": "description column exists in the live database", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI create migration alter table add column local database migration\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - } - ], - "resultChars": 54403 - } - ] - }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "run": 3, - "sourcePath": "codex-gpt-5.6-no-skills/build-cli-002-declarative-schema/run-3/result.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" - }, - { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" - }, - { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 5) from the queue" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Queues pgmq create queue send message read delete Edge Functions cron schedule every minute\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - } - ], - "resultChars": 46311 - } - ] - }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "run": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-cli-003-pg-cron-queue-workflow/run-1/result.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" - }, - { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 1 -> 2" - }, - { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 6) from the queue" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Queues pgmq create queue send read delete Edge Function JavaScript cron schedule every minute\", limit: 8) { nodes { __typename title href content ... on Guide { subsections { nodes { title href content } } } ... on ClientLibraryFunctionReference { language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions#examples", - "title": "Examples" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions#invoke-an-edge-function-every-minute", - "title": "Invoke an Edge Function every minute" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/cron#how-does-cron-work", - "title": "How does Cron work?" - }, - { - "url": "https://supabase.com/docs/guides/cron#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#delete-usage", - "title": "Usage " - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#analyzing-responses", - "title": "Analyzing responses" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#debugging-requests", - "title": "Debugging requests" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#enable-the-extension", - "title": "Enable the extension" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_get", - "title": "http_get" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-signature", - "title": "Signature " - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-usage", - "title": "Usage " - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_post", - "title": "http_post" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-signature", - "title": "Signature " - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-usage", - "title": "Usage " - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_delete", - "title": "http_delete" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-signature-1", - "title": "Signature " - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#inspecting-request-data", - "title": "Inspecting request data" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#inspecting-failed-requests", - "title": "Inspecting failed requests" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#configuration", - "title": "Configuration" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-current-settings", - "title": "Get current settings" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#alter-settings", - "title": "Alter settings" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#examples", - "title": "Examples" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#invoke-a-supabase-edge-function", - "title": "Invoke a Supabase Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#call-an-endpoint-every-minute-with-pg_cron", - "title": "Call an endpoint every minute with pg_cron" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#execute-pg_net-in-a-trigger", - "title": "Execute pg_net in a trigger" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#send-multiple-table-rows-in-one-request", - "title": "Send multiple table rows in one request" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#limitations", - "title": "Limitations" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues#features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/queues#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#schedule-a-job", - "title": "Schedule a job" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#edit-a-job", - "title": "Edit a job" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#activatedeactivate-a-job", - "title": "Activate/Deactivate a job" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#unschedule-a-job", - "title": "Unschedule a job" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#inspecting-job-runs", - "title": "Inspecting job runs" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#examples", - "title": "Examples" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#delete-data-every-week", - "title": "Delete data every week" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#run-a-vacuum-every-day", - "title": "Run a vacuum every day" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#call-a-database-function-every-5-minutes", - "title": "Call a database function every 5 minutes" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#call-a-database-stored-procedure", - "title": "Call a database stored procedure" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#invoke-supabase-edge-function-every-30-seconds", - "title": "Invoke Supabase Edge Function every 30 seconds" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#caution-scheduling-system-maintenance", - "title": "Caution: Scheduling system maintenance" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#concepts", - "title": "Concepts" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#consuming-messages-in-an-edge-function", - "title": "Consuming messages in an Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#concepts", - "title": "Concepts" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#pull-based-queue", - "title": "Pull-Based Queue" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#message", - "title": "Message" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#queue-types", - "title": "Queue types" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#create-queues", - "title": "Create Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#what-happens-when-you-create-a-queue", - "title": "What happens when you create a queue?" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#expose-queues-to-client-side-consumers", - "title": "Expose Queues to client-side consumers" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#enable-rls-on-your-tables-in-pgmq-schema", - "title": "Enable RLS on your tables in pgmq schema" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#grant-permissions-to-pgmq_public-database-functions", - "title": "Grant permissions to pgmq_public database functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#enqueueing-and-dequeueing-messages", - "title": "Enqueueing and dequeueing messages" - }, - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" - }, - { - "url": "https://supabase.com/docs/guides/cron/install#uninstall", - "title": "Uninstall" - } - ], - "resultChars": 130874 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Queues create queue SQL pgmq.create send message pop pgmq_public Edge Functions service role\", limit: 6) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - } - ], - "resultChars": 73074 - } - ] + "passed": false + }, + { + "name": "schema file updated to include description column", + "passed": true + }, + { + "name": "a new migration was generated for the change", + "passed": true + }, + { + "name": "description column exists in the live database", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "run": 2, - "sourcePath": "codex-gpt-5.6-no-skills/build-cli-003-pg-cron-queue-workflow/run-2/result.json" + "docs": { + "calls": [] + }, + "prompt": "Add a description text column to the `products` table in my local Supabase stack", + "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-5.6-no-skills/build-cli-002-declarative-schema/run-3/result.json" }, { "experiment": "codex-gpt-5.6-no-skills", @@ -48841,7 +41844,7 @@ { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 161) from the queue" + "notes": "function removed the seeded message (id 3) from the queue" } ], "skills": { @@ -48851,515 +41854,16 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Queues pgmq create queue send read delete JavaScript Edge Function cron schedule every minute\", limit: 8) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } ... on ClientLibraryFunctionReference { language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/cron#how-does-cron-work", - "title": "How does Cron work?" - }, - { - "url": "https://supabase.com/docs/guides/cron#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions#examples", - "title": "Examples" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions#invoke-an-edge-function-every-minute", - "title": "Invoke an Edge Function every minute" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues#features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/queues#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#delete-usage", - "title": "Usage " - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#analyzing-responses", - "title": "Analyzing responses" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#debugging-requests", - "title": "Debugging requests" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#enable-the-extension", - "title": "Enable the extension" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_get", - "title": "http_get" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-signature", - "title": "Signature " - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-usage", - "title": "Usage " - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_post", - "title": "http_post" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-signature", - "title": "Signature " - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-usage", - "title": "Usage " - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_delete", - "title": "http_delete" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-signature-1", - "title": "Signature " - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#inspecting-request-data", - "title": "Inspecting request data" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#inspecting-failed-requests", - "title": "Inspecting failed requests" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#configuration", - "title": "Configuration" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-current-settings", - "title": "Get current settings" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#alter-settings", - "title": "Alter settings" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#examples", - "title": "Examples" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#invoke-a-supabase-edge-function", - "title": "Invoke a Supabase Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#call-an-endpoint-every-minute-with-pg_cron", - "title": "Call an endpoint every minute with pg_cron" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#execute-pg_net-in-a-trigger", - "title": "Execute pg_net in a trigger" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#send-multiple-table-rows-in-one-request", - "title": "Send multiple table rows in one request" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#limitations", - "title": "Limitations" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#schedule-a-job", - "title": "Schedule a job" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#edit-a-job", - "title": "Edit a job" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#activatedeactivate-a-job", - "title": "Activate/Deactivate a job" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#unschedule-a-job", - "title": "Unschedule a job" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#inspecting-job-runs", - "title": "Inspecting job runs" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#examples", - "title": "Examples" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#delete-data-every-week", - "title": "Delete data every week" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#run-a-vacuum-every-day", - "title": "Run a vacuum every day" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#call-a-database-function-every-5-minutes", - "title": "Call a database function every 5 minutes" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#call-a-database-stored-procedure", - "title": "Call a database stored procedure" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#invoke-supabase-edge-function-every-30-seconds", - "title": "Invoke Supabase Edge Function every 30 seconds" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#caution-scheduling-system-maintenance", - "title": "Caution: Scheduling system maintenance" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#concepts", - "title": "Concepts" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#consuming-messages-in-an-edge-function", - "title": "Consuming messages in an Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#concepts", - "title": "Concepts" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#pull-based-queue", - "title": "Pull-Based Queue" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#message", - "title": "Message" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#queue-types", - "title": "Queue types" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#create-queues", - "title": "Create Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#what-happens-when-you-create-a-queue", - "title": "What happens when you create a queue?" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#expose-queues-to-client-side-consumers", - "title": "Expose Queues to client-side consumers" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#enable-rls-on-your-tables-in-pgmq-schema", - "title": "Enable RLS on your tables in pgmq schema" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#grant-permissions-to-pgmq_public-database-functions", - "title": "Grant permissions to pgmq_public database functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#enqueueing-and-dequeueing-messages", - "title": "Enqueueing and dequeueing messages" - }, - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" - }, - { - "url": "https://supabase.com/docs/guides/cron/install#uninstall", - "title": "Uninstall" - } - ], - "resultChars": 128765 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Queues create queue pgmq.create SQL read messages delete message pgmq_public\", limit: 6) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues#features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/queues#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicpopqueue_name", - "title": "pgmq_public.pop(queue_name)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicsendqueue_name-message-sleep_seconds", - "title": "pgmq_public.send(queue_name, message, sleep_seconds)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicsend_batchqueue_name-messages-sleep_seconds", - "title": "pgmq_public.send_batch(queue_name, messages, sleep_seconds)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicarchivequeue_name-message_id", - "title": "pgmq_public.archive(queue_name, message_id)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicdeletequeue_name-message_id", - "title": "pgmq_public.delete(queue_name, message_id)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicreadqueue_name-sleep_seconds-n", - "title": "pgmq_public.read(queue_name, sleep_seconds, n)" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#concepts", - "title": "Concepts" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#pull-based-queue", - "title": "Pull-Based Queue" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#message", - "title": "Message" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#queue-types", - "title": "Queue types" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#create-queues", - "title": "Create Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#what-happens-when-you-create-a-queue", - "title": "What happens when you create a queue?" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#expose-queues-to-client-side-consumers", - "title": "Expose Queues to client-side consumers" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#enable-rls-on-your-tables-in-pgmq-schema", - "title": "Enable RLS on your tables in pgmq schema" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#grant-permissions-to-pgmq_public-database-functions", - "title": "Grant permissions to pgmq_public database functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#enqueueing-and-dequeueing-messages", - "title": "Enqueueing and dequeueing messages" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#concepts", - "title": "Concepts" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#consuming-messages-in-an-edge-function", - "title": "Consuming messages in an Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", - "title": "Expose Queues for local and self-hosted Supabase" - }, - { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues#expose-queues-with-supabase-cli", - "title": "Expose Queues with Supabase CLI" - }, - { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues#expose-queues-with-docker-compose", - "title": "Expose queues with Docker compose" - }, - { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues#stop-exposing-queues", - "title": "Stop exposing queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#enable-the-extension", - "title": "Enable the extension" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#get-usage", - "title": "Usage " - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#queue-management", - "title": "Queue management" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#create", - "title": "create" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#create_unlogged", - "title": "create_unlogged" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#detach_archive", - "title": "detach_archive" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#drop_queue", - "title": "drop_queue" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#sending-messages", - "title": "Sending messages" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#send", - "title": "send" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#send_batch", - "title": "send_batch" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#reading-messages", - "title": "Reading messages" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#read", - "title": "read" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#read_with_poll", - "title": "read_with_poll" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#pop", - "title": "pop" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#deletingarchiving-messages", - "title": "Deleting/Archiving messages" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#delete-single", - "title": "delete (single)" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#delete-batch", - "title": "delete (batch)" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#purge_queue", - "title": "purge_queue" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#archive-single", - "title": "archive (single)" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#archive-batch", - "title": "archive (batch)" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#utilities", - "title": "Utilities" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#set_vt", - "title": "set_vt" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#list_queues", - "title": "list_queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#metrics", - "title": "metrics" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#metrics_all", - "title": "metrics_all" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#types", - "title": "Types" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#message_record", - "title": "message_record" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#resources", - "title": "Resources" - } - ], - "resultChars": 135004 + "source": "web_search", + "query": "site:supabase.com/docs/guides/queues consume messages Edge Function pgmq_public read delete ...", + "pages": [] } ] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "run": 3, - "sourcePath": "codex-gpt-5.6-no-skills/build-cli-003-pg-cron-queue-workflow/run-3/result.json" + "run": 1, + "sourcePath": "codex-gpt-5.6-no-skills/build-cli-003-pg-cron-queue-workflow/run-1/result.json" }, { "experiment": "codex-gpt-5.6-no-skills", @@ -49370,44 +41874,89 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-dataapi-001-relational-report", + "eval": "build-cli-003-pg-cron-queue-workflow", "stage": "build", "product": [ - "data-api", - "database" + "database", + "edge-functions", + "cron", + "queues" ], "topic": [ + "sql", "sdk" ], "suite": "benchmark", "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { - "name": "report runs and prints JSON", + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", "passed": true, - "notes": "exit 0" + "notes": "schedule='* * * * *', active=true" }, { - "name": "report numbers match the database (per customer, sorted)", + "name": "cron command enqueues to the 'tasks' queue", "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + "notes": "queue depth 1 -> 2" }, { - "name": "tables stay locked down (publishable key reads nothing)", + "name": "process-tasks function drains the queue", "passed": true, - "notes": "publishable read errored: permission denied for table customers" + "notes": "function removed the seeded message (id 4) from the queue" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-cli-003-pg-cron-queue-workflow/run-2/result.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-cli-003-pg-cron-queue-workflow", + "stage": "build", + "product": [ + "database", + "edge-functions", + "cron", + "queues" + ], + "topic": [ + "sql", + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "passed": true, + "notes": "schedule='* * * * *', active=true" }, { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "name": "cron command enqueues to the 'tasks' queue", + "passed": true, + "notes": "queue depth 1 -> 2" }, { - "name": "report queries via the Data API, not raw SQL", + "name": "process-tasks function drains the queue", "passed": true, - "notes": "no psql / raw Postgres driver usage found" + "notes": "function removed the seeded message (id 6) from the queue" } ], "skills": { @@ -49415,41 +41964,12 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript select foreign table nested relationships service role secret key supabase-js\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", - "title": "Performing administration tasks on the server side with a secret key" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", - "title": "Engineering for Scale" - } - ], - "resultChars": 34436 - } - ] + "calls": [] }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "run": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-001-relational-report/run-1/result.json" + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-5.6-no-skills/build-cli-003-pg-cron-queue-workflow/run-3/result.json" }, { "experiment": "codex-gpt-5.6-no-skills", @@ -49505,91 +42025,68 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase javascript select nested relationships count aggregation\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [], - "resultChars": 109 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript nested tables select foreign key relationships inner join supabase-js\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/joins-and-nesting", - "title": "Querying Joins and Nested tables" - }, - { - "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", - "title": "Engineering for Scale" - }, - { - "url": "https://supabase.com/docs/guides/api/automatic-retries-in-supabase-js", - "title": "How to do automatic retries with `supabase-js`" - }, - { - "url": "https://supabase.com/docs/guides/api/creating-routes", - "title": "Creating API Routes" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - }, - { - "url": "https://supabase.com/docs/guides/api/rest/generating-types", - "title": "Generating TypeScript Types" - }, - { - "url": "https://supabase.com/docs/guides/api/sql-to-api", - "title": "Converting SQL to JavaScript API" - }, - { - "url": "https://supabase.com/docs/guides/database/tables", - "title": "Tables and Data" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" - } - ], - "resultChars": 103853 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"secret key apikey Authorization header sb_secret Supabase REST\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/api/creating-routes", - "title": "Creating API Routes" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", - "title": "Supabase CLI" - } - ], - "resultChars": 74442 - } - ] + "calls": [] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "run": 1, + "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-001-relational-report/run-1/result.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", @@ -49650,29 +42147,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript select referenced tables nested relationships pagination range max rows service_role secret key\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/reference/javascript/using-modifiers-limit" - }, - { - "url": "https://supabase.com/docs/reference/javascript/using-modifiers-range" - }, - { - "url": "https://supabase.com/docs/reference/javascript/using-modifiers-maxaffected" - } - ], - "resultChars": 5150 - } - ] + "calls": [] }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", @@ -49733,36 +42208,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"REST API secret key apikey Authorization header service role select referenced tables embedded relationships JavaScript fetch\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/api/creating-routes", - "title": "Creating API Routes" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - } - ], - "resultChars": 85871 - } - ] + "calls": [] }, "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", @@ -49823,35 +42269,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"REST API JavaScript fetch nested relationships foreign key select service role secret key apikey Authorization\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - }, - { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-regenerateclientsecret" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - } - ], - "resultChars": 106810 - } - ] + "calls": [] }, "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", @@ -49912,36 +42330,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase javascript select foreign tables joins embedded resources filter column comparison\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/sql-to-api", - "title": "Converting SQL to JavaScript API" - }, - { - "url": "https://supabase.com/docs/guides/database/joins-and-nesting", - "title": "Querying Joins and Nested tables" - }, - { - "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", - "title": "Engineering for Scale" - }, - { - "url": "https://supabase.com/docs/guides/ai/langchain", - "title": "LangChain" - }, - { - "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors", - "title": "Querying Vectors" - } - ], - "resultChars": 63450 - } - ] + "calls": [] }, "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", @@ -49995,36 +42384,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI restore pg_dump local database binary custom format dump\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" - }, - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-dump", - "title": "Dumps data or schemas from the remote database" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/download-logical-backups", - "title": "How to download logical backups in Supabase with physical backups enabled?" - } - ], - "resultChars": 37299 - } - ] + "calls": [] }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", @@ -50078,36 +42438,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI local restore pg_restore binary dump existing database migrate\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" - }, - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", - "title": "Migrate from Vercel Postgres to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/neon", - "title": "Migrate from Neon to Supabase" - } - ], - "resultChars": 57773 - } - ] + "calls": [] }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", @@ -50161,36 +42492,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI local development restore pg_restore existing database dump\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", - "title": "Migrate from Vercel Postgres to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/neon", - "title": "Migrate from Neon to Supabase" - } - ], - "resultChars": 42818 - } - ] + "calls": [] }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", @@ -50253,36 +42555,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions authenticate user Authorization header getUser JWT createClient service role RLS\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - } - ], - "resultChars": 63849 - } - ] + "calls": [] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", @@ -50348,7 +42621,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions authenticate user Authorization header createClient auth.getUser RLS\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "query": "query { searchDocs(query: \"Edge Functions Supabase client Authorization header auth getUser row level security service role bypass RLS\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -50356,19 +42629,23 @@ "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" } ], - "resultChars": 36221 + "resultChars": 60625 } ] }, @@ -50420,12 +42697,12 @@ { "name": "user A cannot force-read user B note", "passed": true, - "notes": "status=403" + "notes": "status=200" }, { "name": "user B cannot force-read user A note", "passed": true, - "notes": "status=403" + "notes": "status=200" } ], "skills": { @@ -50433,36 +42710,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions Supabase client Authorization header getUser JWT row level security service role bypass\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - } - ], - "resultChars": 63022 - } - ] + "calls": [] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", @@ -50493,7 +42741,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "seed rows present", @@ -50503,42 +42751,42 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"f904d4cd-2c9c-4ca3-8739-31c475f87171\",\"metric\":\"steps_a_mta4f7z6\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"26685ed8-4fae-4129-a3bd-d8ce81179b99\",\"metric\":\"steps_a_mtsfug5s\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"f904d4cd-2c9c-4ca3-8739-31c475f87171\",\"metric\":\"steps_a_mta4f7z6\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"26685ed8-4fae-4129-a3bd-d8ce81179b99\",\"metric\":\"steps_a_mtsfug5s\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"ca072caa-4db3-428a-a972-3d19b0b5df5c\",\"metric\":\"steps_b_mta4f7z6\",\"value\":222}]" + "passed": false, + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" + "passed": false, + "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" } ], "skills": { @@ -50546,76 +42794,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions verify_jwt false service role apikey header Authorization getUser createClient Deno\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - } - ], - "resultChars": 62717 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Securing Edge Functions apikey secret key in-code authorization verify_jwt false SUPABASE_SECRET_KEYS\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - } - ], - "resultChars": 43306 - } - ] + "calls": [] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", @@ -50656,161 +42835,54 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { "name": "user with JWT reads only their own rows", - "passed": false, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": false, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": false, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" - }, - { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" - }, - { - "name": "implementation uses @supabase/server", "passed": true, - "notes": "imports @supabase/server / withSupabase" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions JWT verification service role apikey Authorization header createClient getUser Deno\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - }, - { - "url": "https://supabase.com/docs/guides/functions/http-methods", - "title": "Routing" - } - ], - "resultChars": 62966 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"config.toml functions verify_jwt false Edge Function manual JWT authorization getUser token service_role key\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - } - ], - "resultChars": 80750 - }, + "notes": "status 200: [{\"user_id\":\"3e1a19b8-6e3e-40e5-bce9-9cc7f54c3d32\",\"metric\":\"steps_a_mtsfylbo\",\"value\":111}]" + }, + { + "name": "user cannot read another user's rows by passing user_id", + "passed": true, + "notes": "status 200: [{\"user_id\":\"3e1a19b8-6e3e-40e5-bce9-9cc7f54c3d32\",\"metric\":\"steps_a_mtsfylbo\",\"value\":111}]" + }, + { + "name": "service key bypasses RLS to read the target user's rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"0cc937e8-1ef7-43c3-8b87-59bb5b9940ba\",\"metric\":\"steps_b_mtsfylbo\",\"value\":222}]" + }, + { + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 401: {\"error\":\"Unauthorized\"}" + }, + { + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 401: {\"error\":\"Unauthorized\"}" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 401: {\"error\":\"Unauthorized\"}" + }, + { + "name": "implementation uses @supabase/server", + "passed": false, + "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"withSupabase auth ['user', 'secret'] ctx authMode supabaseAdmin userClaims edge functions\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - } - ], - "resultChars": 31975 + "source": "web_search", + "query": "site:supabase.com/docs \"@supabase/server\" secret key Edge Functions API key ...", + "pages": [] } ] }, @@ -50843,7 +42915,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "seed rows present", @@ -50858,17 +42930,17 @@ { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"7f5755ed-91ed-4996-ae88-955768d2611a\",\"metric\":\"steps_a_mta4eehw\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"22174d0c-926f-4a79-b0e1-f0c2a8a4f288\",\"metric\":\"steps_a_mtsfu2ds\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"7f5755ed-91ed-4996-ae88-955768d2611a\",\"metric\":\"steps_a_mta4eehw\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"22174d0c-926f-4a79-b0e1-f0c2a8a4f288\",\"metric\":\"steps_a_mtsfu2ds\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"fc332674-7e4f-4408-a622-81d802058117\",\"metric\":\"steps_b_mta4eehw\",\"value\":222}]" + "passed": false, + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { "name": "non-service key is not granted service access", @@ -50878,7 +42950,7 @@ { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { "name": "a user token in the apikey slot is not treated as the service key", @@ -50887,8 +42959,8 @@ }, { "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" + "passed": false, + "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" } ], "skills": { @@ -50896,132 +42968,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions authenticate JWT Authorization header service_role apikey createClient Deno\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - } - ], - "resultChars": 45212 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions config.toml verify_jwt false built-in secrets SUPABASE_SERVICE_ROLE_KEY\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - } - ], - "resultChars": 68924 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions default environment variables SUPABASE_URL SUPABASE_ANON_KEY SUPABASE_SERVICE_ROLE_KEY SUPABASE_SECRET_KEY\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - } - ], - "resultChars": 41809 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"@supabase/server withSupabase ctx userClaims sub supabase authMode user secret Edge Function\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - } - ], - "resultChars": 39346 - } - ] + "calls": [] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", @@ -51117,28 +43064,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Row Level Security security definer helper function recursion policies auth uid\", limit: 3) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0003_auth_rls_initplan", - "title": "Database Advisor: Lint 0003_auth_rls_initplan" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - } - ], - "resultChars": 22916 - } - ] + "calls": [] }, "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", @@ -51330,36 +43256,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Row Level Security security definer function avoid recursion policy auth.uid organizations memberships\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0003_auth_rls_initplan", - "title": "Database Advisor: Lint 0003_auth_rls_initplan" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - } - ], - "resultChars": 74649 - } - ] + "calls": [] }, "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", @@ -51387,45 +43284,12 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 01a03e35-3ea5-71fe-974e-0762bb7d1ec6/receipt-alpha.pdf, 01a03e35-3ea5-71fe-974e-0762bb7d1ec6/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT policies, enabled RLS, and expiring createSignedUrl code are all present." + "passed": false, + "notes": "no row in storage.buckets with id or name 'user-files'" } ], "skills": { @@ -51433,55 +43297,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Storage access control RLS policies foldername auth.uid createSignedUrl private bucket supabase-js\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", - "title": "Custom Roles" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" - } - ], - "resultChars": 15220 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript createSignedUrl storage expiresIn supabase-js\", limit: 3) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", - "title": "Storage Image Transformations" - } - ], - "resultChars": 25020 - } - ] + "calls": [] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", @@ -51509,45 +43325,12 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 01a03e35-014a-73ee-988a-cd0926846483/receipt-alpha.pdf, 01a03e35-014a-73ee-988a-cd0926846483/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT policies, unchanged RLS, and expiring createSignedUrl sharing code are all provided." + "passed": false, + "notes": "no row in storage.buckets with id or name 'user-files'" } ], "skills": { @@ -51555,55 +43338,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Storage access control RLS policies storage.objects bucket_id foldername auth.uid signed URL createSignedUrl\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", - "title": "Custom Roles" - } - ], - "resultChars": 22016 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase javascript createSignedUrl path expiresIn storage\", limit: 3) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", - "title": "Storage Image Transformations" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" - } - ], - "resultChars": 25020 - } - ] + "calls": [] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", @@ -51631,45 +43366,12 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 01a03e35-1d07-7347-987c-880daa07a2f0/receipt-alpha.pdf, 01a03e35-1d07-7347-987c-880daa07a2f0/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT policies, and expiring createSignedUrl code are correctly provided; RLS was not disabled." + "passed": false, + "notes": "no row in storage.buckets with id or name 'user-files'" } ], "skills": { @@ -51677,35 +43379,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Storage access control RLS policies storage.foldername authenticated user id createSignedUrl JavaScript\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/storage/debugging/error-codes", - "title": "Error Codes" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" - } - ], - "resultChars": 24225 - } - ] + "calls": [] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", @@ -51733,58 +43407,29 @@ "suite": "benchmark", "interface": "cli", "passed": true, - "checks": [ - { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation.test.sql" - }, - { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "6 passed, 3 failed" - }, - { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "Correctly identifies the posts policy as allowing cross-tenant reads, explains the missing org_id correlation, and grounds the conclusion in reproducible pgTAP failures. It correctly states notes isolation passes." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase database testing pgTAP RLS auth.uid set request.jwt.claims tenant isolation\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - } - ], - "resultChars": 89166 - } - ] + "checks": [ + { + "name": "pgTAP test file(s) written under supabase/tests/", + "passed": true, + "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" + }, + { + "name": "pgTAP isolation tests ran and pass", + "passed": true, + "notes": "6 passed, 2 failed" + }, + { + "name": "agent correctly identifies the posts isolation bug from test results", + "passed": true, + "judgeNotes": "Correctly identifies the posts policy as allowing cross-tenant reads, grounds this in the failing pgTAP result, and distinguishes notes as correctly isolated." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", @@ -51821,12 +43466,12 @@ { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "4 passed, 3 failed" + "notes": "4 passed, 2 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies the posts RLS policy as leaking cross-tenant rows, grounds this in the failing pgTAP assertion, and distinguishes notes as correctly isolated." + "judgeNotes": "Correctly identifies `posts` as leaking cross-tenant reads, grounds this in failing pgTAP results, and confirms `notes` isolation tests pass." } ], "skills": { @@ -51834,36 +43479,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase database testing pgTAP row level security auth.uid set local role authenticated\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" - }, - { - "url": "https://supabase.com/docs/guides/database/testing", - "title": "Testing Your Database" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, - { - "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", - "title": "Custom Claims & Role-based Access Control (RBAC)" - }, - { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" - } - ], - "resultChars": 58515 - } - ] + "calls": [] }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", @@ -51895,17 +43511,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation.sql" + "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "6 passed, 3 failed" + "notes": "4 passed, 5 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies the posts policy flaw: members can read another tenant’s posts, grounded in the failing pgTAP assertion. It correctly states notes remain isolated." + "judgeNotes": "Correctly identifies the `posts` cross-tenant read policy flaw, grounds it in failing pgTAP results, and correctly states that `notes` read isolation passes." } ], "skills": { @@ -51913,36 +43529,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase database testing pgTAP RLS auth.uid tenant isolation tests\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgtap", - "title": "pgTAP: Unit Testing" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - } - ], - "resultChars": 71814 - } - ] + "calls": [] }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", @@ -51980,12 +43567,12 @@ { "name": "HNSW index on the embedding column", "passed": true, - "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" }, { "name": "index operator class matches the search operator", "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" }, { "name": "user A search returns only own sections, best match first", @@ -52012,31 +43599,31 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"pgvector semantic search match documents row level security auth.uid rpc security invoker\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"pgvector semantic search match_documents row level security auth.uid SQL function vector cosine\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, { "url": "https://supabase.com/docs/guides/ai/semantic-search", "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/ai/hybrid-search", + "title": "Hybrid search" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" + }, + { + "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors", + "title": "Querying Vectors" }, { - "url": "https://supabase.com/docs/guides/database/full-text-search", - "title": "Full Text Search" + "url": "https://supabase.com/docs/guides/ai/vector-columns", + "title": "Vector columns" } ], - "resultChars": 107889 + "resultChars": 71759 } ] }, @@ -52066,38 +43653,12 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" - }, - { - "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" - }, - { - "name": "index operator class matches the search operator", - "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" - }, - { - "name": "user A search returns only own sections, best match first", - "passed": true - }, - { - "name": "user B search returns only own sections, best match first", - "passed": true - }, - { - "name": "user A reads only own sections through the API", - "passed": true - }, - { - "name": "user A reads only own documents through the API", - "passed": true + "name": "created auth sessions", + "passed": false, + "notes": "Internal server error" } ], "skills": { @@ -52108,31 +43669,31 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase gte-small embedding dimensions pgvector semantic search match_documents RLS security invoker\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "query": "query { searchDocs(query: \"pgvector semantic search row level security RPC auth uid security invoker\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" }, { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" + "url": "https://supabase.com/docs/guides/database/tables", + "title": "Tables and Data" }, { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" + "url": "https://supabase.com/docs/guides/database/functions", + "title": "Database Functions" } ], - "resultChars": 68780 + "resultChars": 99218 } ] }, @@ -52204,7 +43765,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase semantic search gte-small vector 384 match_documents RPC pgvector HNSW RLS security invoker\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "query": "query { searchDocs(query: \"Supabase gte-small embedding dimensions pgvector semantic search match_documents RLS\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -52220,15 +43781,15 @@ "title": "RAG with Permissions" }, { - "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors", - "title": "Querying Vectors" + "url": "https://supabase.com/docs/guides/ai/concepts", + "title": "Concepts" }, { - "url": "https://supabase.com/docs/guides/ai/vector-columns", - "title": "Vector columns" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" } ], - "resultChars": 63644 + "resultChars": 68096 } ] }, @@ -52265,12 +43826,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "HTTPS Supabase metrics scrape is correctly configured with the required path, Basic Auth password_file, matching Compose secret mount, project target, and preserved app job." + "judgeNotes": "HTTPS Supabase scrape uses the required path and Basic Auth password_file; the Compose secret wiring matches, and the existing app scrape remains." }, { "name": "documented live deployment and verification steps", "passed": false, - "judgeNotes": "The secret-file command writes the literal placeholder `sb_secret_REPLACE_ME` instead of clearly inserting the created Secret API key, so the mounted secret would not match and authentication would fail." + "judgeNotes": "Prometheus Basic Auth uses `username: username`; Supabase’s privileged metrics endpoint requires the expected `service_role` username with the secret API key as the password. README does not correct this, so verification would fail." } ], "skills": { @@ -52280,40 +43841,9 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase project metrics Prometheus endpoint service_role metrics authentication observability\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" - }, - { - "url": "https://supabase.com/docs/guides/database/connection-management", - "title": "Connection management" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", - "title": "How to View Database Metrics" - } - ], - "resultChars": 35772 + "source": "web_search", + "query": "site:supabase.com/docs/guides/platform metrics Prometheus customer/v1/privileged/metrics ...", + "pages": [] } ] }, @@ -52350,12 +43880,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "HTTPS Supabase Metrics API scrape is correctly configured with the required path, project target, Basic Auth password_file, preserved app job, and matching Compose secret mount." + "judgeNotes": "HTTPS Supabase metrics scrape uses the correct path and Basic Auth password_file; Compose mounts the matching secret, and the existing app scrape remains." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README documents creating a Supabase Secret API key, placing it in the matching Compose secret file, recreating Prometheus, and verifying the Supabase target is UP in Prometheus." + "judgeNotes": "README documents secret API key creation, matching Compose secret file placement, stack startup/recreation and reload behavior, plus concrete verification through Prometheus targets and troubleshooting." } ], "skills": { @@ -52365,40 +43895,9 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"Prometheus metrics endpoint hosted project /customer/v1/privileged/metrics basic auth service role\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", - "title": "Scrape a project's metrics" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/security/security-testing", - "title": "Security testing of your Supabase projects" - } - ], - "resultChars": 48116 + "source": "web_search", + "query": "site:supabase.com/docs/guides/platform/metrics Prometheus metrics endpoint customer v1 privileged metrics authentication", + "pages": [] } ] }, @@ -52435,12 +43934,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "Supabase scrape is deployable: HTTPS, correct metrics path, Basic Auth with matching password_file secret, valid project target, and existing app scrape preserved." + "judgeNotes": "HTTPS Supabase Metrics API scrape is correctly configured with the required path, Basic Auth password_file, project target, preserved app job, and matching Docker Compose secret mount." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README documents creating a Secret API key, writing the matching Docker secret file, recreating the Compose services, and verifying the Supabase target is UP in Prometheus." + "judgeNotes": "README documents secret API key creation, matching secret file placement, Compose deployment/recreation, and concrete verification through Prometheus Targets and PromQL." } ], "skills": { @@ -52450,40 +43949,9 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"Prometheus metrics customer v1 privileged metrics service role basic auth hosted project\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", - "title": "Scrape a project's metrics" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/security/security-testing", - "title": "Security testing of your Supabase projects" - } - ], - "resultChars": 48116 + "source": "web_search", + "query": "site:supabase.com/docs/guides/platform metrics Prometheus customer/v1/privileged/metrics basic_auth", + "pages": [] } ] }, @@ -52525,7 +43993,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment via Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -52537,36 +44005,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions environment variables secrets set deploy no-verify-jwt Deno.env.get\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", - "title": "Transcription Telegram Bot" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-email-hook", - "title": "Send Email Hook" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - } - ], - "resultChars": 77900 - } - ] + "calls": [] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", @@ -52606,7 +44045,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment via Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -52618,36 +44057,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions environment variables secrets deploy supabase secrets set env file Deno.env.get\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", - "title": "Transcription Telegram Bot" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" - } - ], - "resultChars": 37900 - } - ] + "calls": [] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", @@ -52699,36 +44109,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions environment variables secrets set deploy no verify jwt CORS Deno.env.get\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - } - ], - "resultChars": 36624 - } - ] + "calls": [] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", @@ -52782,64 +44163,9 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"self-hosting Docker docker-compose .env secrets JWT_SECRET ANON_KEY SERVICE_ROLE_KEY POOLER_TENANT_ID VAULT_ENC_KEY latest\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates", - "title": "Custom Email Templates" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting", - "title": "Self-Hosting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", - "title": "Upgrade to Postgres 17" - } - ], - "resultChars": 131989 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Docker self-hosting Quick Start Linux git clone supabase docker copy .env.example generate-keys.sh\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - } - ], - "resultChars": 72942 + "source": "web_search", + "query": "site:supabase.com/docs/guides/self-hosting/docker Supabase self-host Docker compose official ...", + "pages": [] } ] }, @@ -52893,80 +44219,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"self-hosting Docker docker-compose .env JWT_SECRET ANON_KEY SERVICE_ROLE_KEY DASHBOARD_PASSWORD POOLER_TENANT_ID SMTP docker compose\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates", - "title": "Custom Email Templates" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-phone-mfa", - "title": "Configure Phone Login & MFA" - } - ], - "resultChars": 164395 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Self-Hosting with Docker quick start git clone supabase repository docker .env.example generate-keys.sh minimum requirements production security\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting", - "title": "Self-Hosting" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" - } - ], - "resultChars": 87804 - } - ] + "calls": [] }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", @@ -52993,71 +44246,36 @@ "self-hosting" ], "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true - }, - { - "name": "secrets rotated off the shipped defaults", - "passed": true - }, - { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"self-hosting Docker compose secrets JWT_SECRET ANON_KEY SERVICE_ROLE_KEY POSTGRES_PASSWORD DASHBOARD_PASSWORD POOLER_TENANT_ID Supabase\", limit: 8) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", - "title": "Upgrade to Postgres 17" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting", - "title": "Self-Hosting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", - "title": "Configure Reverse Proxy and HTTPS" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - } - ], - "resultChars": 114388 + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", + "passed": true + }, + { + "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", + "passed": true + }, + { + "name": "secrets rotated off the shipped defaults", + "passed": true + }, + { + "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "site:supabase.com/docs/guides/self-hosting/docker Supabase Docker self-hosting official ...", + "pages": [] } ] }, @@ -53116,7 +44334,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "Correctly diagnoses the soft-delete issue, hard-deletes the auth user and sessions, closes stale-JWT Data API access via RLS, explains remaining local JWT validation until expiry, and accurately distinguishes publishable from secret keys." + "judgeNotes": "Correctly diagnoses the soft-delete issue, deletes the Auth user to revoke sessions and refresh tokens, closes stale-JWT Data API access through RLS, preserves the local JWT-expiry caveat, and accurately distinguishes publishable from secret keys." } ], "skills": { @@ -53127,143 +44345,98 @@ "calls": [ { "source": "search_docs", - "query": "query { keys: searchDocs(query: \"publishable key secret key anon service_role RLS frontend\", limit: 5) { nodes { ... on Guide { title href content } } } sessions: searchDocs(query: \"delete user access token remains valid JWT session revoked until expiry\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "query": "query {\n keys: searchDocs(query: \"publishable secret API keys anon service_role RLS frontend migration\", limit: 5) {\n nodes { title href content }\n }\n deletion: searchDocs(query: \"delete user JWT remains valid until expiry refresh token session RLS\", limit: 5) {\n nodes { title href content }\n }\n}", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - }, { "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", "title": "Build a User Management App with RedwoodJS" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" }, { - "url": "https://supabase.com/docs/guides/platform/temporary-access", - "title": "Temporary access" + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - } - ], - "resultChars": 178931 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"sign out revoke refresh token access token valid until expiry JWT Supabase Auth\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ { "url": "https://supabase.com/docs/guides/auth/sessions", "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", - "title": "Build a Supabase Integration" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" + "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", + "title": "Should I set a shorter Max-Age parameter on the cookies?" }, { "url": "https://supabase.com/docs/guides/auth/server-side/advanced-guide", "title": "Advanced guide" }, { - "url": "https://supabase.com/docs/guides/auth/signout", - "title": "Signing out" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" }, { - "url": "https://supabase.com/docs/guides/auth/third-party/overview", - "title": "Third-party auth" + "url": "https://supabase.com/docs/guides/realtime/authorization", + "title": "Realtime Authorization" } ], - "resultChars": 122313 + "resultChars": 139700 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"delete user auth.users SQL database function cascade Supabase\", limit: 6) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "query": "query { searchDocs(query: \"Understanding API keys publishable secret keys bypass RLS\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", - "title": "Custom Claims & Role-based Access Control (RBAC)" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" }, { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0002_auth_users_exposed", - "title": "Database Advisor: Lint 0002_auth_users_exposed" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" } ], - "resultChars": 45437 + "resultChars": 45590 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"API keys publishable secret key sb_publishable sb_secret role RLS bypass\", limit: 8) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"sign out revoke refresh tokens access token valid until expires JWT\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", + "title": "OAuth 2.1 Flows" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", + "title": "Should I set a shorter Max-Age parameter on the cookies?" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" + "url": "https://supabase.com/docs/guides/platform/sso/enterprise-mcp-authentication", + "title": "Enterprise-Managed Authentication for MCP" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/reference/javascript/auth-signout" } ], - "resultChars": 102758 + "resultChars": 58413 } ] }, @@ -53322,7 +44495,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "Correctly diagnoses the soft-delete flaw, deletes the Auth user and sessions, blocks stale JWTs through RLS active-profile checks, acknowledges JWT validity until expiry, and accurately distinguishes publishable and secret keys." + "judgeNotes": "Correctly diagnoses soft deletion, hard-deletes the Auth user and cascading sessions/refresh tokens, closes stale-JWT access for current Data API tables via RLS, notes JWT validity until expiry for other/local paths, and accurately distinguishes publishable versus secret keys." } ], "skills": { @@ -53333,7 +44506,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"delete user auth admin deleteUser JWT remains valid RLS revoked session access publishable secret keys legacy anon service_role\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"delete user JWT remains valid until expiry sign out revoke sessions\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -53341,438 +44514,102 @@ "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { "url": "https://supabase.com/docs/reference/swift/auth-admin-signout", "title": "signOut()" - } - ], - "resultChars": 100375 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"delete user auth.users SQL security definer delete account JWT remains valid until expires RLS\", limit: 10) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ + }, { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" + "url": "https://supabase.com/docs/reference/javascript/auth-signout" }, { "url": "https://supabase.com/docs/guides/auth/managing-user-data", "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" - }, - { - "url": "https://supabase.com/docs/guides/storage/management/delete-objects", - "title": "Delete Objects" - }, - { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" } ], - "resultChars": 121879 + "resultChars": 54351 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"publishable key secret key frontend RLS bypass legacy anon service_role API keys\", limit: 6) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", + "query": "query { searchDocs(query: \"publishable key secret key RLS anon service_role new API keys\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control#access-policies", - "title": "Access policies" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control#policy-examples", - "title": "Policy examples" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control#bypassing-access-controls", - "title": "Bypassing access controls" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" }, { "url": "https://supabase.com/docs/guides/auth/signing-keys", "title": "JWT Signing Keys" }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#how-to-create-mint-jwts-if-access-to-the-private-key-or-shared-secret-is-not-possible", - "title": "How to create (mint) JWTs if access to the private key or shared secret is not possible?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#why-is-a-5-minute-wait-imposed-when-changing-signing-key-states", - "title": "Why is a 5 minute wait imposed when changing signing key states?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#why-is-deleting-the-legacy-jwt-secret-disallowed", - "title": "Why is deleting the legacy JWT secret disallowed?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#why-does-revoking-the-legacy-jwt-secret-require-disabling-of-anon-and-service_role-api-keys", - "title": "Why does revoking the legacy JWT secret require disabling of anon and service_role API keys?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#using-jwt-based-anon-key-in-a-mobile-desktop-or-cli-application-and-need-to-rotate-a-service_role-jwt-secret", - "title": "Using JWT-based anon key in a mobile, desktop, or CLI application and need to rotate a service_role JWT secret?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#overview", - "title": "Overview" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#benefits-of-the-signing-keys-system", - "title": "Benefits of the signing keys system" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#getting-started", - "title": "Getting started" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#rotating-and-revoking-keys", - "title": "Rotating and revoking keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#lifetime-of-a-signing-key", - "title": "Lifetime of a signing key" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#public-key-discovery-and-caching", - "title": "Public key discovery and caching" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#choosing-the-right-signing-algorithm", - "title": "Choosing the right signing algorithm" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#frequently-asked-questions", - "title": "Frequently asked questions" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#why-is-it-not-possible-to-extract-the-private-key-or-shared-secret-from-supabase", - "title": "Why is it not possible to extract the private key or shared secret from Supabase?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#why-are-anon-and-service_role-jwt-based-keys-no-longer-recommended", - "title": "Why are anon and service_role JWT-based keys no longer recommended?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#can-you-still-use-an-old-anon-and-service-role-api-keys-after-enabling-the-publishable-and-secret-keys", - "title": "Can you still use an old anon and service-role API keys after enabling the publishable and secret keys?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#how-to-deactivate-the-anon-and-service_role-jwt-based-api-keys-after-moving-to-publishable-and-secret-keys", - "title": "How to deactivate the anon and service_role JWT-based API keys after moving to publishable and secret keys?" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys#how-are-publishable-and-secret-keys-implemented-on-the-hosted-platform", - "title": "How are publishable and secret keys implemented on the hosted platform?" - }, { "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#before-you-start", - "title": "Before you start" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-1-create-the-new-api-keys", - "title": "Step 1: Create the new API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-2-swap-the-publishable-key-in-client-code", - "title": "Step 2: Swap the publishable key in client code" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-3-swap-the-secret-key-in-backend-code", - "title": "Step 3: Swap the secret key in backend code" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#database-webhooks-and-pg_net", - "title": "Database Webhooks and pg_net" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-4-update-edge-functions", - "title": "Step 4: Update Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-1-read-the-new-keys-from-the-environment", - "title": "Option 1: Read the new keys from the environment" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-2-adopt-the-supabaseserver-sdk", - "title": "Option 2: Adopt the @supabase/server SDK" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-5-verify-nothing-uses-the-legacy-keys", - "title": "Step 5: Verify nothing uses the legacy keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-6-deactivate-the-legacy-keys", - "title": "Step 6: Deactivate the legacy keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#known-limitations", - "title": "Known limitations" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#next-steps", - "title": "Next steps" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#row-level-security-in-supabase", - "title": "Row Level Security in Supabase" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#policies", - "title": "Policies" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#enabling-row-level-security", - "title": "Enabling Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#auto-enable-rls-for-new-tables", - "title": "Auto-enable RLS for new tables" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#authenticated-and-unauthenticated-roles", - "title": "Authenticated and unauthenticated roles" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#creating-policies", - "title": "Creating policies" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#select-policies", - "title": "SELECT policies" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#insert-policies", - "title": "INSERT policies" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#update-policies", - "title": "UPDATE policies" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#delete-policies", - "title": "DELETE policies" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#views", - "title": "Views" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#helper-functions", - "title": "Helper functions" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#authuid", - "title": "auth.uid()" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#authjwt", - "title": "auth.jwt()" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#mfa", - "title": "MFA" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#bypassing-row-level-security", - "title": "Bypassing Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#rls-performance-recommendations", - "title": "RLS performance recommendations" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#add-indexes", - "title": "Add indexes" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#call-functions-with-select", - "title": "Call functions with select" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-1", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#add-filters-to-every-query", - "title": "Add filters to every query" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-2", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#use-security-definer-functions", - "title": "Use security definer functions" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#minimize-joins", - "title": "Minimize joins" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-3", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#specify-roles-in-your-policies", - "title": "Specify roles in your policies" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-4", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#more-resources", - "title": "More resources" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data#connecting-your-app-securely", - "title": "Connecting your app securely" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data#data-api", - "title": "Data API" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data#edge-functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data#direct-database-connections", - "title": "Direct database connections" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data#frontend-access", - "title": "Frontend access" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data#more-information", - "title": "More information" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#overview", - "title": "Overview" + "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", + "title": "Creating a Supabase client for SSR" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#publishable-keys", - "title": "Publishable keys" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#interaction-with-supabase-auth", - "title": "Interaction with Supabase Auth" + "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", + "title": "Build a Social Auth App with Expo React Native" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#security-considerations", - "title": "Security considerations" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", + "title": "Build a User Management App with SvelteKit" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#what-secret-keys-allow-access-to", - "title": "What secret keys allow access to" - }, + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + } + ], + "resultChars": 252692 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"delete user auth.users SQL JWT valid until expires\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#best-practices-for-handling-secret-keys", - "title": "Best practices for handling secret keys" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#what-to-do-if-a-secret-key-or-service_role-has-been-leaked-or-compromised", - "title": "What to do if a secret key or service_role has been leaked or compromised?" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#known-limitations-and-compatibility-differences", - "title": "Known limitations and compatibility differences" + "url": "https://supabase.com/docs/guides/realtime/authorization", + "title": "Realtime Authorization" } ], - "resultChars": 356737 + "resultChars": 31737 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"site:supabase.com/docs/guides/getting-started/api-keys publishable key safe frontend secret key bypass RLS\", limit: 3) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"Understanding API keys publishable secret bypass RLS\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-react", - "title": "Build a User Management App with Ionic React" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" } ], - "resultChars": 60950 + "resultChars": 45590 } ] }, @@ -53831,7 +44668,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "Correctly diagnoses soft deletion, implements auth-user/session removal with RLS stale-JWT protection, explains JWT expiry/local-validation caveats consistently, and accurately distinguishes publishable versus secret keys." + "judgeNotes": "Correctly diagnosed the soft-delete-only flow, deleted the Auth user and cascading sessions/identity data, added live-user RLS checks to close stale-JWT Data API access, accurately retained the JWT-expiry caveat for local validation, and correctly distinguished frontend publishable keys from server-only RLS-bypassing secret keys." } ], "skills": { @@ -53842,63 +44679,62 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"delete user JWT remains valid until expiry RLS auth.users sessions revoke\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"delete user JWT remains valid access token session sign out refresh token RLS auth.users\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, { "url": "https://supabase.com/docs/guides/auth/managing-user-data", "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", - "title": "Should I set a shorter Max-Age parameter on the cookies?" + "url": "https://supabase.com/docs/reference/swift/auth-admin-signout", + "title": "signOut()" }, { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" + "url": "https://supabase.com/docs/reference/javascript/auth-signout" + }, + { + "url": "https://supabase.com/docs/guides/auth/users", + "title": "Users" } ], - "resultChars": 67910 + "resultChars": 32873 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"publishable key secret key legacy anon service_role RLS frontend\", limit: 6) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"publishable secret keys RLS anon service_role frontend bypass RLS\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { "url": "https://supabase.com/docs/guides/auth/jwt-fields", "title": "JWT Claims Reference" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" } ], - "resultChars": 104141 + "resultChars": 63742 } ] }, @@ -53953,7 +44789,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified the missing publication membership, added only public.orders to the existing supabase_realtime publication, and verified courier_locations remained included with RLS and policies unchanged." + "judgeNotes": "Correctly identified the missing orders publication entry, added public.orders to the existing supabase_realtime publication, and preserved courier_locations, RLS, and policies." } ], "skills": { @@ -53964,7 +44800,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Postgres Changes add table supabase_realtime publication ALTER PUBLICATION\", limit: 3) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"Postgres Changes enable table supabase_realtime publication\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -53972,11 +44808,15 @@ "title": "Subscribing to Database Changes" }, { - "url": "https://supabase.com/docs/guides/database/replication/pipelines", - "title": "Set up Pipelines" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" + }, + { + "url": "https://supabase.com/docs/guides/realtime/benchmarks", + "title": "Benchmarks" } ], - "resultChars": 24655 + "resultChars": 66974 } ] }, @@ -54031,7 +44871,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified the missing orders publication membership, added only public.orders to supabase_realtime, preserved courier_locations and RLS/policies, and verified the result." + "judgeNotes": "Correctly identified and fixed the missing orders table in the existing supabase_realtime publication while preserving courier_locations, RLS, and policies." } ], "skills": { @@ -54092,7 +44932,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified the missing publication membership, added only public.orders to supabase_realtime, and preserved courier_locations, RLS, and existing policies." + "judgeNotes": "Correctly identified and fixed the missing orders publication entry while preserving courier_locations, RLS, and existing policies." } ], "skills": { @@ -54103,7 +44943,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Postgres Changes add table supabase_realtime publication ALTER PUBLICATION\", limit: 3) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Postgres Changes add table supabase_realtime publication\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -54115,11 +44955,11 @@ "title": "Realtime: Postgres Changes Troubleshooting" }, { - "url": "https://supabase.com/docs/guides/database/replication/pipelines", - "title": "Set up Pipelines" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" } ], - "resultChars": 34979 + "resultChars": 66918 } ] }, @@ -54152,17 +44992,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified `image-transform` as the affected function and the recurring pattern of eight HTTP 503 gateway failures from 07:00Z–12:00Z on 2026-04-28." + "judgeNotes": "Identified image-transform as the affected function and explicitly reported 8 recurring HTTP 503 failures from 07:00–12:00 UTC on 2026-04-28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Correctly attributes the recurring image-transform 503s to the gateway/platform layer, supported by gateway-only logs with no matching function executions, successful nearby requests on the unchanged deployment, and distinction from avatar-upload’s runtime 500." + "judgeNotes": "Attributes the recurring 503s to the Supabase gateway/platform layer, supported by absent runtime/invocation details for failures, successful nearby requests on the unchanged deployment, and a clear distinction from avatar-upload’s function-level 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Provides concrete actions, including opening a Supabase support ticket with the project ID, exact time window, and gateway request IDs, plus retry and alerting recommendations." + "judgeNotes": "Recommended concrete actions, including opening a Supabase support ticket with gateway request IDs, adding bounded retries, monitoring the incident window, and investigating the correlated application error." } ], "skills": { @@ -54170,7 +45010,36 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions 503 gateway error function not invoked troubleshooting retries\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/status-codes", + "title": "Status codes" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", + "title": "Edge Function 503 error response" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", + "title": "Edge Function 504 error response" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-404-error-response", + "title": "Edge Function 404 error response" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + } + ], + "resultChars": 42641 + } + ] }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", @@ -54201,17 +45070,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified `image-transform` as affected and documented 8 recurring HTTP 503 gateway failures from 07:00Z–12:00Z on 2026-04-28." + "judgeNotes": "Identified `image-transform` and the recurring eight HTTP 503 gateway failures from 07:00Z through 12:00Z on 2026-04-28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Attributes the recurring 503s to the Supabase Edge gateway/routing layer, supported by gateway-only failure logs with no matching function invocations and successful requests on the unchanged deployment. It also distinguishes the separate function-level avatar-upload 500." + "judgeNotes": "Attributes the 503s to the Supabase gateway/platform layer, supported by gateway-only failures with no function execution and successful nearby requests on the same deployment; it also distinguishes the unrelated function-level avatar-upload 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Provides concrete next steps, including escalation to Supabase Support with project, region, time window, and gateway request IDs, plus targeted retry, alerting, and logging actions." + "judgeNotes": "It recommends opening a Supabase support ticket with the project, region, gateway request IDs, and incident window, plus checking response details and adding targeted retries." } ], "skills": { @@ -54219,7 +45088,41 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions 503 gateway request does not reach function runtime troubleshooting\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/status-codes", + "title": "Status codes" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", + "title": "Edge Function 503 error response" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-404-error-response", + "title": "Edge Function 404 error response" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", + "title": "Edge Function 504 error response" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + } + ], + "resultChars": 38658 + }, + { + "source": "web_search", + "query": "site:status.supabase.com April 28 2026 Edge Functions incident 503 ...", + "pages": [] + } + ] }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", @@ -54250,17 +45153,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified `image-transform` and the recurring pattern of 8 HTTP 503 failures from 07:00Z–12:00Z on 2026-04-28." + "judgeNotes": "Identified `image-transform` and the recurring pattern of eight HTTP 503 gateway failures from 07:00Z–12:00Z on 2026-04-28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Attributes the recurring 503s to the Edge Functions gateway/routing layer and supports this with absent runtime invocations, nearby successful executions on the same deployment, and a clear distinction from avatar-upload’s function-level 500." + "judgeNotes": "Attributes the 503s to the Edge gateway/platform and grounds this in absent function execution logs plus nearby successful invocations; it also distinguishes the unrelated function-level avatar-upload 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Provides a concrete escalation path with project, region, deployment, request IDs, time window, and specific gateway/runtime evidence, plus actionable retry and monitoring changes." + "judgeNotes": "Recommended opening a Supabase support ticket with the project reference, exact time window, and gateway request IDs, plus concrete retry and alerting actions." } ], "skills": { @@ -54268,7 +45171,36 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions 503 gateway function logs troubleshooting\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/status-codes", + "title": "Status codes" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", + "title": "Edge Function 503 error response" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", + "title": "Edge Function 504 error response" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + } + ], + "resultChars": 46973 + } + ] }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", @@ -54326,7 +45258,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly diagnosed deny-all RLS, kept RLS enabled, and created authenticated owner-scoped SELECT and INSERT policies using auth.uid() with USING and WITH CHECK." + "judgeNotes": "Correctly diagnosed deny-all RLS, kept RLS enabled, and added authenticated owner-scoped SELECT and INSERT policies using auth.uid() with USING and WITH CHECK." } ], "skills": { @@ -54334,32 +45266,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Row Level Security auth.uid select insert policy authenticated users own rows\", limit: 5) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" - } - ], - "resultChars": 68712 - } - ] + "calls": [] }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", @@ -54417,7 +45324,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly diagnosed deny-all RLS, kept RLS enabled, and created authenticated owner-scoped SELECT and INSERT policies using auth.uid() with USING and WITH CHECK." + "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and added authenticated-only SELECT and INSERT policies scoped to user_id = auth.uid() using USING and WITH CHECK." } ], "skills": { @@ -54425,28 +45332,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Row Level Security auth.uid select insert policy authenticated using with check\", limit: 3) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0024_permissive_rls_policy", - "title": "Database Advisor: Lint 0024_permissive_rls_policy" - } - ], - "resultChars": 54133 - } - ] + "calls": [] }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", @@ -54504,7 +45390,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and created authenticated owner-scoped SELECT USING and INSERT WITH CHECK policies based on auth.uid() = user_id." + "judgeNotes": "Correctly diagnosed deny-all RLS, kept RLS enabled, and added authenticated owner-scoped SELECT and INSERT policies using auth.uid() with USING and WITH CHECK." } ], "skills": { @@ -54512,36 +45398,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Row Level Security policies auth.uid select insert Data API grants bookmarks\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - } - ], - "resultChars": 84758 - } - ] + "calls": [] }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", @@ -54592,7 +45449,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": false, - "judgeNotes": "History was reconciled by adding `20240115000000_add_profile_bio.sql`. However, the recorded output for `supabase db push --db-url \"$DB_URL\" --yes` does not show `Applying migration ...` or `Finished supabase db push`, so the required successful remote CLI push is not proven. No prohibited mutation workaround was shown; psql use was read-only." + "judgeNotes": "`supabase db push --db-url \"$POOLER_URI\" --yes` was issued, and the local bio migration file was restored, but the recorded push output shows neither “Applying migration ...” nor “Finished supabase db push.” Therefore actual CLI application of `avatar_url` is not proven. No prohibited workaround is visible." } ], "skills": { @@ -54600,36 +45457,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI migration deploy linked hosted project db push migration list repair\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", - "title": "Repair the migration history table" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-push", - "title": "Push new migrations to the remote database" - } - ], - "resultChars": 61657 - } - ] + "calls": [] }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", @@ -54655,7 +45483,7 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "the avatar_url column is applied on the hosted profiles table", @@ -54679,8 +45507,8 @@ }, { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": false, - "judgeNotes": "`supabase db push --db-url \"$DB_URL\" --yes` was run, and the local bio file reconciled history via that push. However, the recorded push output does not show “Applying migration” or “Finished supabase db push,” as required. No forbidden direct-SQL mutation or reset workaround was shown." + "passed": true, + "judgeNotes": "Added local 20240115000000_add_profile_bio.sql to reconcile the orphan history, then `supabase db push --db-url ... --yes` applied 20240220000000_add_avatar_url.sql and finished successfully. The psql usage was read-only; no prohibited workaround occurred." } ], "skills": { @@ -54688,36 +45516,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI custom API URL profile SUPABASE_API_URL link project ref db push\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/database/inspect", - "title": "Database debugging and monitoring" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/guides/storage/analytics/connecting-to-analytics-bucket", - "title": "Iceberg Catalog" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-link", - "title": "Link to a Supabase project" - } - ], - "resultChars": 55668 - } - ] + "calls": [] }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", @@ -54768,7 +45567,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "`supabase db push --db-url \"$DB_URL\" --yes` applied the avatar migration; final schema/history confirm `avatar_url` and version `20240220000000`. History was reconciled by adding `20240115000000_add_profile_bio.sql` locally, then using the CLI push workflow. No prohibited workaround; psql was read-only inspection." + "judgeNotes": "`supabase db push --db-url \"$eval_db_url\" --yes` applied `20240220000000_add_avatar_url.sql`, showing both “Applying migration” and “Finished supabase db push.” The orphan bio history was reconciled by adding local `20240115000000_add_profile_bio.sql`; migration list then matched remote before the successful push. The `psql` commands were read-only verification; no prohibited workaround was used." } ], "skills": { @@ -54776,36 +45575,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI database migrations db push repair migration history hosted project\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-push", - "title": "Push new migrations to the remote database" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", - "title": "Repair the migration history table" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - } - ], - "resultChars": 61657 - } - ] + "calls": [] }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", @@ -54849,7 +45619,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_id_created_at_desc (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -55227,7 +45997,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 3fc3e2ee-4307-4b23-8def-06f72c2153e4, signUp returned {\"userId\":\"3fc3e2ee-4307-4b23-8def-06f72c2153e4\"}" + "notes": "db user 35bda052-4b2b-498b-93a9-8135e48a05e2, signUp returned {\"userId\":\"35bda052-4b2b-498b-93a9-8135e48a05e2\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -55242,7 +46012,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"3fc3e2ee-4307-4b23-8def-06f72c2153e4\"}" + "notes": "{\"userId\":\"35bda052-4b2b-498b-93a9-8135e48a05e2\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -55272,14 +46042,138 @@ "docs": { "calls": [ { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | grep -i -B2 -A4 'breaking' | head -60", + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 3454 + "resultChars": 94610 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"supabase-js signUp with email password and user metadata display name, signInWithPassword, getUser\") { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-getuser" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", + "title": "Sign in with Google" + }, + { + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", + "title": "Build a User Management App with Refine" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-expo-react-native", + "title": "Build a User Management App with Expo React Native" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-azure", + "title": "Sign in with Azure (Microsoft)" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-apple", + "title": "Sign in with Apple" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", + "title": "signInWithPassword()" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signup" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-spotify", + "title": "Sign in with Spotify" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", + "title": "Passwordless email sign-in" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-web3", + "title": "Sign in with Web3" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", + "title": "Build a User Management App with React" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", + "title": "Build a User Management App with Angular" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-signup" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", + "title": "Sign in with Figma" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nuxt-3", + "title": "Build a User Management App with Nuxt 3" + }, + { + "url": "https://supabase.com/docs/reference/csharp/auth-signinwithpassword", + "title": "SignIn(email, password)" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-gitlab", + "title": "Sign in with GitLab" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-signup", + "title": "signUp()" + } + ], + "resultChars": 473816 } ] }, @@ -55319,7 +46213,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user dbb2165e-4efc-4ced-bcff-b6b2d6aae372, signUp returned {\"userId\":\"dbb2165e-4efc-4ced-bcff-b6b2d6aae372\"}" + "notes": "db user bcb65cd4-8cab-4e9b-aa2d-593733b92c72, signUp returned {\"userId\":\"bcb65cd4-8cab-4e9b-aa2d-593733b92c72\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -55334,7 +46228,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"dbb2165e-4efc-4ced-bcff-b6b2d6aae372\"}" + "notes": "{\"userId\":\"bcb65cd4-8cab-4e9b-aa2d-593733b92c72\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -55362,141 +46256,7 @@ ] }, "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | head -100", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 5442 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"supabase-js signUp email password user metadata display name signInWithPassword getUser\") { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", - "title": "Login with Google" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", - "title": "signInWithPassword()" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", - "title": "Build a User Management App with Refine" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nuxt-3", - "title": "Build a User Management App with Nuxt 3" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-expo-react-native", - "title": "Build a User Management App with Expo React Native" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-signup", - "title": "signUp()" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-apple", - "title": "Login with Apple" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-azure", - "title": "Login with Azure (Microsoft)" - }, - { - "url": "https://supabase.com/docs/reference/csharp/auth-signinwithpassword", - "title": "SignIn(email, password)" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signup" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-web3", - "title": "Sign in with Web3" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", - "title": "Build a User Management App with React" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", - "title": "Build a User Management App with Angular" - }, - { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-react", - "title": "Build a User Management App with Ionic React" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-vue", - "title": "Build a User Management App with Ionic Vue" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", - "title": "Login with Figma" - } - ], - "resultChars": 501339 - } - ] + "calls": [] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", @@ -55534,7 +46294,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 1839e3ce-4e67-4ca2-b385-8b2a60152d66, signUp returned {\"userId\":\"1839e3ce-4e67-4ca2-b385-8b2a60152d66\"}" + "notes": "db user 99b82d82-78b4-467d-98fe-25f9fe7fa79b, signUp returned {\"userId\":\"99b82d82-78b4-467d-98fe-25f9fe7fa79b\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -55549,7 +46309,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"1839e3ce-4e67-4ca2-b385-8b2a60152d66\"}" + "notes": "{\"userId\":\"99b82d82-78b4-467d-98fe-25f9fe7fa79b\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -55579,128 +46339,41 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "{ searchDocs(query: \"supabase-js signUp email password options data user metadata display name signInWithPassword getUser\") { nodes { title href content } } }", + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" - }, + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 94610 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"signUp email password user metadata display name signInWithPassword getUser\", limit: 5) { nodes { title href content ... on ClientLibraryFunctionReference { language methodName } } } }", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", - "title": "Login with Google" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", - "title": "Build a User Management App with Refine" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" - }, - { - "url": "https://supabase.com/docs/guides/auth/general-configuration", - "title": "General configuration" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-expo-react-native", - "title": "Build a User Management App with Expo React Native" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-azure", - "title": "Login with Azure (Microsoft)" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-web3", - "title": "Sign in with Web3" - }, - { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-apple", - "title": "Login with Apple" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" - }, { "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", "title": "signInWithPassword()" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nuxt-3", - "title": "Build a User Management App with Nuxt 3" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", - "title": "Build a User Management App with React" + "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" }, { "url": "https://supabase.com/docs/reference/csharp/auth-signinwithpassword", "title": "SignIn(email, password)" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", - "title": "Build a User Management App with Angular" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", - "title": "Login with Figma" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/google-auth-fails-for-some-users-XcFXEu", - "title": "Google Auth fails for some users" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-gitlab", - "title": "Login with GitLab" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-identity-linking", - "title": "Identity Linking" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" } ], - "resultChars": 473656 + "resultChars": 17930 } ] }, @@ -55774,7 +46447,18 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | head -100", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5442 + } + ] }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", @@ -55848,15 +46532,14 @@ "docs": { "calls": [ { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | head -100", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 94610 + "resultChars": 5442 } ] }, @@ -55898,7 +46581,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 3 rows" + "notes": "found 2 rows" }, { "name": "row level security is enabled on todos", @@ -55916,7 +46599,7 @@ { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "3 rows" + "notes": "2 rows" } ], "skills": { @@ -56091,25 +46774,23 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "supabase db diff used to generate the migration", - "passed": false + "passed": true }, { "name": "schema file updated to include description column", - "passed": false, - "notes": "description not found in any schema file" + "passed": true }, { "name": "a new migration was generated for the change", - "passed": false, - "notes": "found 1 migration file(s)" + "passed": true }, { "name": "description column exists in the live database", - "passed": false + "passed": true } ], "skills": { @@ -56117,7 +46798,10 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [] @@ -56164,7 +46848,7 @@ { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 36) from the queue" + "notes": "function removed the seeded message (id 38) from the queue" } ], "skills": { @@ -56179,27 +46863,79 @@ }, "docs": { "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function read messages from pgmq queue delete pop\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" + } + ], + "resultChars": 57952 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"pgmq queues cron schedule send message to queue every minute\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-phone-mfa", + "title": "Configure Phone sign-in & MFA" + }, + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + } + ], + "resultChars": 74270 + }, { "source": "web_fetch", - "query": "https://supabase.com/changelog.md", + "query": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions.md", "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions.md" } ], - "resultChars": 94610 + "resultChars": 3874 }, { "source": "web_fetch", - "query": "https://supabase.com/docs/guides/queues/quickstart.md", + "query": "https://supabase.com/docs/guides/cron.md", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/queues/quickstart.md" + "url": "https://supabase.com/docs/guides/cron.md" } ], - "resultChars": 8777 + "resultChars": 1405 }, { "source": "web_fetch", @@ -56251,12 +46987,12 @@ { "name": "cron command enqueues to the 'tasks' queue", "passed": true, - "notes": "queue depth 1 -> 2" + "notes": "queue depth 0 -> 1" }, { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 42) from the queue" + "notes": "function removed the seeded message (id 3) from the queue" } ], "skills": { @@ -56271,61 +47007,44 @@ }, "docs": { "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 94610 + }, { "source": "search_docs", - "query": "{ searchDocs(query: \"pgmq queues cron schedule job every minute send message to queue\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pg_cron schedule job every minute pgmq send queue\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, { "url": "https://supabase.com/docs/guides/queues/pgmq", "title": "PGMQ Extension" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" }, { "url": "https://supabase.com/docs/guides/queues/quickstart", "title": "Quickstart" } ], - "resultChars": 75189 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | grep -i -B2 -A2 -E \"breaking|pgmq|cron|queue\" | head -60", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 4462 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/queues/quickstart.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/quickstart.md" - } - ], - "resultChars": 8777 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/cron/quickstart.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/cron/quickstart.md" - } - ], - "resultChars": 6155 + "resultChars": 48142 } ] }, @@ -56366,12 +47085,12 @@ { "name": "cron command enqueues to the 'tasks' queue", "passed": true, - "notes": "queue depth 1 -> 2" + "notes": "queue depth 0 -> 1" }, { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 39) from the queue" + "notes": "function removed the seeded message (id 37) from the queue" } ], "skills": { @@ -56388,55 +47107,35 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"queues pgmq create queue consume edge function pop delete\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pg_cron schedule job every minute pgmq send queue\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, { "url": "https://supabase.com/docs/guides/queues/pgmq", "title": "PGMQ Extension" }, { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" }, { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" }, { "url": "https://supabase.com/docs/guides/queues/quickstart", "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - } - ], - "resultChars": 50772 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron schedule job every minute cron.schedule\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-sms-hook", - "title": "Send SMS Hook" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-email-hook", - "title": "Send Email Hook" } ], - "resultChars": 79601 + "resultChars": 48142 }, { "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | grep -i -B2 -A2 -E \"breaking\" | grep -i -E \"cron|queue|pgmq|edge function|deno\" | head -20; echo \"exit: $?\"", + "query": "curl -s https://supabase.com/changelog.md | grep -i -B2 -A2 -E \"breaking\" | grep -i -E \"cron|pgmq|queue|edge\" | head; echo \"exit: $?\"", "pages": [ { "url": "https://supabase.com/changelog.md" @@ -56666,7 +47365,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "report runs and prints JSON", @@ -56685,8 +47384,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "passed": true, + "notes": "imports found in: app/restock.mjs" }, { "name": "report queries via the Data API, not raw SQL", @@ -56893,19 +47592,7 @@ ] }, "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 94610 - } - ] + "calls": [] }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", @@ -56964,19 +47651,7 @@ ] }, "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 94610 - } - ] + "calls": [] }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", @@ -57008,227 +47683,102 @@ "passed": true }, { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", - "passed": true - }, - { - "name": "tasks_team_status_idx index survived the restore", - "passed": true - }, - { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "run": 3, - "sourcePath": "opencode-kimi-k3/build-database-001-migrate-postgres-to-supabase/run-3/result.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "rls", - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" - }, - { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" - }, - { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" - }, - { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=200" - }, - { - "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=200" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge functions get user from JWT authorization header createClient\") { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getclaims" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/push-notifications", - "title": "Sending Push Notifications" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - }, - { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" - }, - { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-getclient" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", - "title": "Build a User Management App with React" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-getclaims", - "title": "getClaims()" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", - "title": "Build a User Management App with Angular" - }, - { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/egress", - "title": "Manage Egress usage" - } - ], - "resultChars": 311012 - } + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "run": 3, + "sourcePath": "opencode-kimi-k3/build-database-001-migrate-postgres-to-supabase/run-3/result.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-functions-004-service-role-bypass", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "rls", + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "rejects missing auth", + "passed": true, + "notes": "status=401" + }, + { + "name": "user A reads own note", + "passed": true, + "notes": "status=200" + }, + { + "name": "reads only with the caller's JWT", + "passed": true, + "notes": "bearer_tokens=2, all_match=true" + }, + { + "name": "user A cannot force-read user B note", + "passed": true, + "notes": "status=200" + }, + { + "name": "user B cannot force-read user A note", + "passed": true, + "notes": "status=200" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" ] }, + "docs": { + "calls": [] + }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", "run": 1, @@ -57294,47 +47844,7 @@ ] }, "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 94610 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function verify user JWT auth getUser authorization header\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - } - ], - "resultChars": 63501 - } - ] + "calls": [] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", @@ -57401,19 +47911,7 @@ ] }, "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 94610 - } - ] + "calls": [] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", @@ -57453,37 +47951,37 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"MISSING_CREDENTIALS\",\"message\":\"[@supabase/server] No credentials found on the request. This endpoint accepts auth mode(s):" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"5fcddcd7-e280-4817-b7cd-7425f2569863\",\"metric\":\"steps_a_mta4j9dk\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"bc2a4e9c-c3a4-4c20-a226-a2b3c3c3bba1\",\"metric\":\"steps_a_mtsfx2zu\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"5fcddcd7-e280-4817-b7cd-7425f2569863\",\"metric\":\"steps_a_mta4j9dk\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"bc2a4e9c-c3a4-4c20-a226-a2b3c3c3bba1\",\"metric\":\"steps_a_mtsfx2zu\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"266b08a2-7807-4483-9bef-7efe388852c1\",\"metric\":\"steps_b_mta4j9dk\",\"value\":222}]" + "notes": "status 200: [{\"user_id\":\"6be3ed11-fac9-4629-bb2f-41d412e8f0c1\",\"metric\":\"steps_b_mtsfx2zu\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_JWT\",\"message\":\"[@supabase/server] The JWT in the Authorization header failed verification: its header is missing \\" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" }, { "name": "implementation uses @supabase/server", @@ -57497,8 +47995,7 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { @@ -57516,273 +48013,55 @@ }, { "source": "search_docs", - "query": "{ searchDocs(query: \"edge function environment variables SUPABASE_SECRET_KEY publishable key service role\") { nodes { title href content } } }", + "query": "{ searchDocs(query: \"edge function verify user JWT getUser authorization header\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - }, - { - "url": "https://supabase.com/docs/guides/integrations/vercel-marketplace", - "title": "Vercel Marketplace" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/functions/storage-caching", - "title": "Integrating with Supabase Storage" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", - "title": "Transcription Telegram Bot" - }, - { - "url": "https://supabase.com/docs/guides/deployment", - "title": "Deployment & Branching" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/functions/architecture", - "title": "Edge Functions Architecture" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" - }, - { - "url": "https://supabase.com/docs/guides/functions/unit-test", - "title": "Testing your Edge Functions" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/functions/examples/cloudflare-turnstile", - "title": "CAPTCHA support with Cloudflare Turnstile" + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", - "title": "Configure Reverse Proxy and HTTPS" + "url": "https://supabase.com/docs/guides/functions/status-codes", + "title": "Status codes" }, { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" } ], - "resultChars": 390580 + "resultChars": 66931 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"edge function auth getUser verify JWT service role key apikey header\") { nodes { title href content } } }", + "query": "{ searchDocs(query: \"edge functions environment variables SUPABASE_URL SUPABASE_ANON_KEY SUPABASE_SERVICE_ROLE_KEY SUPABASE_SECRET_KEY publishable\", limit: 4) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports", - "title": "Reports" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getclaims" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-createuser" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-get-project-legacy-api-keys", - "title": "Check whether JWT based legacy (anon, service_role) API keys are enabled. This API endpoint will be removed in the future, check for HTTP 404 Not Found." + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - } - ], - "resultChars": 360218 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/functions/auth.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth.md" - } - ], - "resultChars": 7275 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/functions/secrets.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets.md" - } - ], - "resultChars": 5106 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/functions/auth-headers.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-headers.md" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" } ], - "resultChars": 3331 + "resultChars": 34922 } ] }, @@ -57814,7 +48093,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "seed rows present", @@ -57824,42 +48103,42 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 404: Function not found" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"MISSING_CREDENTIALS\",\"message\":\"[@supabase/server] No credentials found on the request. This endpoint accepts auth mode(s):" }, { "name": "user with JWT reads only their own rows", - "passed": false, - "notes": "status 404: Function not found" + "passed": true, + "notes": "status 200: [{\"user_id\":\"fe665e4e-d65d-4a80-969e-0693eb41c8b9\",\"metric\":\"steps_a_mtsfwsw1\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", - "passed": false, - "notes": "status 404: Function not found" + "passed": true, + "notes": "status 200: [{\"user_id\":\"fe665e4e-d65d-4a80-969e-0693eb41c8b9\",\"metric\":\"steps_a_mtsfwsw1\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", - "passed": false, - "notes": "status 404: Function not found" + "passed": true, + "notes": "status 200: [{\"user_id\":\"cc93f6db-ffe6-4764-83ec-bc042e1917cf\",\"metric\":\"steps_b_mtsfwsw1\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 404: Function not found" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 404: Function not found" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_JWT\",\"message\":\"[@supabase/server] The JWT in the Authorization header failed verification: its header is missing \\" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 404: Function not found" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" }, { "name": "implementation uses @supabase/server", - "passed": false, - "notes": "could not locate function source to inspect" + "passed": true, + "notes": "imports @supabase/server / withSupabase" } ], "skills": { @@ -57873,7 +48152,83 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 94610 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function environment variables SUPABASE_URL SUPABASE_SECRET_KEY publishable key\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + } + ], + "resultChars": 67435 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"@supabase/server SDK edge functions createClient\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/server-side/migrating-to-ssr-from-auth-helpers", + "title": "Migrating to the SSR package from Auth Helpers" + }, + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-getclient" + }, + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-updateclient" + } + ], + "resultChars": 15612 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/functions/auth.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth.md" + } + ], + "resultChars": 7820 + } + ] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", @@ -57913,37 +48268,37 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"MISSING_CREDENTIALS\",\"message\":\"[@supabase/server] No credentials found on the request. This endpoint accepts auth mode(s):" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"b99c63b6-8203-4a24-af97-ad9aa5a7a781\",\"metric\":\"steps_a_mta4m4hh\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"e0726bcd-d4e8-4a6d-a49b-5818c66de7d9\",\"metric\":\"steps_a_mtsfvfz5\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"b99c63b6-8203-4a24-af97-ad9aa5a7a781\",\"metric\":\"steps_a_mta4m4hh\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"e0726bcd-d4e8-4a6d-a49b-5818c66de7d9\",\"metric\":\"steps_a_mtsfvfz5\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"1ca2015a-a000-4b2a-a381-e62c01a5c092\",\"metric\":\"steps_b_mta4m4hh\",\"value\":222}]" + "notes": "status 200: [{\"user_id\":\"72ccc740-c18a-4def-8396-aa162f1dfc69\",\"metric\":\"steps_b_mtsfvfz5\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_JWT\",\"message\":\"[@supabase/server] The JWT in the Authorization header failed verification: its header is missing \\" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" }, { "name": "implementation uses @supabase/server", @@ -57962,72 +48317,44 @@ }, "docs": { "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 94610 - }, { "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server SDK edge functions\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"edge functions environment variables SUPABASE_URL SUPABASE_PUBLISHABLE_KEY SUPABASE_SECRET_KEY service role\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-functions", - "title": "Manage Supabase Edge functions" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" } ], - "resultChars": 44996 + "resultChars": 40567 }, { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge functions environment variables SUPABASE_URL SUPABASE_ANON_KEY secret key publishable\", limit: 5) { nodes { title href content } } }", + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/storage-caching", - "title": "Integrating with Supabase Storage" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 37737 + "resultChars": 94610 } ] }, @@ -58133,142 +48460,31 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"Supabase row level security policies auth.uid organization membership roles\") { nodes { title href content } } }", + "query": "{ searchDocs(query: \"RLS security definer function private schema revoke execute policy\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", - "title": "Custom Claims & Role-based Access Control (RBAC)" - }, - { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" - }, - { - "url": "https://supabase.com/docs/guides/platform/access-control", - "title": "Access Control" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/firebase-auth", - "title": "Firebase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", - "title": "Custom Roles" - }, - { - "url": "https://supabase.com/docs/guides/platform/mfa/org-mfa-enforcement", - "title": "Enforce MFA on Organization" - }, { "url": "https://supabase.com/docs/guides/api/securing-your-api", "title": "Securing your API" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod", - "title": "Production Checklist" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/auth/enterprise-sso", - "title": "Enterprise Single Sign-On" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - }, - { - "url": "https://supabase.com/docs/guides/integrations/vercel-marketplace", - "title": "Vercel Marketplace" - }, - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/security/platform-audit-logs", - "title": "Platform Audit Logs" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server", - "title": "OAuth 2.1 Server" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", - "title": "Build a User Management App with Refine" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso", - "title": "Enable SSO for Your Organization" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" }, { - "url": "https://supabase.com/docs/guides/database/postgres/roles", - "title": "Postgres Roles" + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" }, { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" + "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", + "title": "Declarative database schemas" }, { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - } - ], - "resultChars": 12 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" } ], - "resultChars": 94610 + "resultChars": 93338 } ] }, @@ -58516,7 +48732,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 01a03e3a-1811-7652-83d2-cd1520ec6cb3/receipt-alpha.pdf, 01a03e3a-1811-7652-83d2-cd1520ec6cb3/receipt-beta.pdf" + "notes": "saw: 01a0803e-49cb-7326-bef5-3251ae975642/receipt-alpha.pdf, 01a0803e-49cb-7326-bef5-3251ae975642/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -58537,7 +48753,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Creates a private user-files bucket, retains owner-scoped authenticated RLS policies for SELECT and INSERT (plus UPDATE/DELETE), and uses createSignedUrl with a 10-minute expiry." + "judgeNotes": "Creates a private user-files bucket, authenticated owner-scoped SELECT and INSERT RLS policies, preserves RLS, and uses createSignedUrl with a 600-second expiry." } ], "skills": { @@ -58565,142 +48781,220 @@ }, { "source": "search_docs", - "query": "{ searchDocs(query: \"storage access control RLS policy bucket private\") { nodes { title href content } } }", + "query": "{ searchDocs(query: \"createSignedUrl temporary expiring link share private file\") { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", - "title": "Build a User Management App with Angular" + "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-expo-react-native", - "title": "Build a User Management App with Expo React Native" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/reference/csharp/storage-from-createsignedurl", + "title": "From().CreateSignedUrl()" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-angular", - "title": "Build a User Management App with Ionic Angular" + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" }, { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" + "url": "https://supabase.com/docs/reference/python/storage-from-createsignedurl", + "title": "from_.create_signed_url()" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-swift", - "title": "Build a User Management App with Swift and SwiftUI" + "url": "https://supabase.com/docs/reference/dart/file-buckets-createsigneduploadurl" }, { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" + "url": "https://supabase.com/docs/reference/python/storage-from-createsignedurls", + "title": "from_.create_signed_urls()" }, { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" + "url": "https://supabase.com/docs/guides/storage/cdn/smart-cdn", + "title": "Smart CDN" }, { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" + "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", + "title": "Storage Image Transformations" }, { - "url": "https://supabase.com/docs/guides/storage/debugging/error-codes", - "title": "Error Codes" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-uploadtosignedurl" + }, + { + "url": "https://supabase.com/docs/reference/swift/storage-from-createsignedurls", + "title": "from.createSignedUrls()" + }, + { + "url": "https://supabase.com/docs/reference/swift/storage-from-createsignedurl", + "title": "from.createSignedUrl()" + }, + { + "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads", + "title": "Resumable Uploads" + }, + { + "url": "https://supabase.com/docs/reference/python/storage-from-createsigneduploadurl", + "title": "from_.create_signed_upload_url()" + }, + { + "url": "https://supabase.com/docs/reference/swift/storage-from-createsigneduploadurl", + "title": "from.createSignedUploadURL()" + }, + { + "url": "https://supabase.com/docs/reference/kotlin/storage-from-createsignedurl", + "title": "from.createSignedUrl()" + }, + { + "url": "https://supabase.com/docs/reference/kotlin/storage-from-createsigneduploadurl", + "title": "from.createSignedUploadUrl()" + }, + { + "url": "https://supabase.com/docs/reference/swift/storage-from-uploadtosignedurl", + "title": "from.uploadToSignedUrl()" + }, + { + "url": "https://supabase.com/docs/reference/kotlin/storage-from-createsignedurls", + "title": "from.createSignedUrls()" + }, + { + "url": "https://supabase.com/docs/reference/kotlin/storage-from-uploadtosignedurl", + "title": "from.uploadToSignedUrl()" + } + ], + "resultChars": 53606 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"storage row level security policy foldername owner upload download private bucket\") { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" + }, + { + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" + }, + { + "url": "https://supabase.com/docs/guides/storage/quickstart", + "title": "Storage Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", + "title": "Why can't I upload/list/etc my public bucket?" }, { "url": "https://supabase.com/docs/guides/storage/schema/design", "title": "The Storage Schema" }, { - "url": "https://supabase.com/docs/guides/storage/cdn/fundamentals", - "title": "Storage CDN" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" }, { "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", "title": "Custom Roles" }, + { + "url": "https://supabase.com/docs/guides/storage/cdn/fundamentals", + "title": "Storage CDN" + }, + { + "url": "https://supabase.com/docs/guides/storage/uploads/file-limits", + "title": "Limits" + }, + { + "url": "https://supabase.com/docs/guides/platform/delete-project", + "title": "Deleting Your Project" + }, + { + "url": "https://supabase.com/docs/guides/ai-tools/mcp", + "title": "Supabase MCP Server" + }, { "url": "https://supabase.com/docs/guides/local-development/database-migrations", "title": "Database migrations" }, { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" + "url": "https://supabase.com/docs/guides/troubleshooting/supabase-storage-inefficient-folder-operations-and-hierarchical-rls-challenges-b05a4d", + "title": "Supabase Storage: Inefficient folder operations and hierarchical RLS challenges" + }, + { + "url": "https://supabase.com/docs/guides/storage", + "title": "Storage" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/storage-error-403-forbidden-new-row-violates-row-level-security-policy-on-upload-a94384", + "title": "Storage error: 403 Forbidden: 'new row violates row-level security policy' on upload" + }, + { + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" }, { "url": "https://supabase.com/docs/guides/storage/production/scaling", "title": "Storage Optimizations" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", - "title": "Why can't I upload/list/etc my public bucket?" + "url": "https://supabase.com/docs/guides/storage/s3/authentication", + "title": "S3 Authentication" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" + "url": "https://supabase.com/docs/guides/storage/management/download-objects", + "title": "Download Objects" }, { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/storage-error-403-forbidden-new-row-violates-row-level-security-policy-on-upload-a94384", - "title": "Storage error: 403 Forbidden: 'new row violates row-level security policy' on upload" + "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", + "title": "Copy Objects" }, { - "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", - "title": "Declarative database schemas" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/firebase-storage", + "title": "Migrated from Firebase Storage to Supabase" }, { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" + "url": "https://supabase.com/docs/guides/functions/examples/amazon-bedrock-image-generator", + "title": "Generate Images with Amazon Bedrock" }, { - "url": "https://supabase.com/docs/guides/realtime/authorization", - "title": "Realtime Authorization" + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/docs/guides/storage/s3/compatibility", + "title": "S3 Compatibility" }, { "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/storage/s3/authentication", - "title": "S3 Authentication" + "url": "https://supabase.com/docs/guides/functions/ephemeral-storage", + "title": "File Storage" }, { - "url": "https://supabase.com/docs/guides/storage/management/delete-objects", - "title": "Delete Objects" - } - ], - "resultChars": 321494 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/storage/security/access-control.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/security/access-control.md" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-updatebucket" } ], - "resultChars": 4169 + "resultChars": 198603 } ] }, @@ -58746,7 +49040,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 01a03e39-7e5d-766f-a1e0-4432dfe4d7ff/receipt-alpha.pdf, 01a03e39-7e5d-766f-a1e0-4432dfe4d7ff/receipt-beta.pdf" + "notes": "saw: 01a0803e-29a8-750d-94bc-1ee7f6fb41d2/receipt-alpha.pdf, 01a0803e-29a8-750d-94bc-1ee7f6fb41d2/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -58767,7 +49061,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT policies, RLS preserved, and expiring createSignedUrl code are all provided." + "judgeNotes": "Private user-files bucket, authenticated owner-folder SELECT/INSERT RLS policies, RLS enforcement verified, and short-lived createSignedUrl client code are all provided." } ], "skills": { @@ -58795,43 +49089,31 @@ }, { "source": "search_docs", - "query": "{ searchDocs(query: \"storage bucket private row level security policies path user id signed URL supabase-js createSignedUrl\", limit: 8) { nodes { title href content } totalCount } }", + "query": "{ searchDocs(query: \"storage access control RLS policies bucket path user id folder\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", - "title": "Storage Image Transformations" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", - "title": "Manage Storage Image Transformations usage" - }, - { - "url": "https://supabase.com/docs/guides/storage/s3/authentication", - "title": "S3 Authentication" + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", "title": "Configure S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads", - "title": "Resumable Uploads" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", - "title": "Custom Roles" } ], - "resultChars": 63594 + "resultChars": 60185 }, { "source": "web_fetch", @@ -58854,36 +49136,24 @@ } ] }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/storage/schema/helper-functions.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions.md" - } - ], - "resultChars": 3333 - }, { "source": "search_docs", - "query": "{ searchDocs(query: \"createSignedUrl expiresIn supabase-js storage from\", limit: 3) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"createSignedUrl storage from expiresIn signed url download option\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" + "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", + "title": "Storage Image Transformations" }, { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", - "title": "Manage Storage Image Transformations usage" + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" }, { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", - "title": "Storage Image Transformations" + "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" } ], - "resultChars": 28966 + "resultChars": 23674 } ] }, @@ -58929,7 +49199,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 01a03e38-aa80-7447-a501-f9256d8af6f0/receipt-alpha.pdf, 01a03e38-aa80-7447-a501-f9256d8af6f0/receipt-beta.pdf" + "notes": "saw: 01a0803c-fbc2-738c-bb6c-5f800f59caa6/receipt-alpha.pdf, 01a0803c-fbc2-738c-bb6c-5f800f59caa6/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -58950,7 +49220,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Private bucket, RLS-enabled owner-scoped SELECT/INSERT policies for authenticated users, and short-lived createSignedUrl sharing are all correctly implemented." + "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT policies, RLS preserved, and short-lived createSignedUrl sharing code are all provided." } ], "skills": { @@ -58965,20 +49235,9 @@ }, "docs": { "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 94610 - }, { "source": "search_docs", - "query": "query { searchDocs(query: \"storage access control RLS policies storage.objects bucket private\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"storage access control RLS policies bucket private auth.uid folder name\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -58990,30 +49249,48 @@ "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", - "title": "Build a User Management App with Angular" + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-expo-react-native", - "title": "Build a User Management App with Expo React Native" + "url": "https://supabase.com/docs/guides/storage/debugging/error-codes", + "title": "Error Codes" } ], - "resultChars": 76131 + "resultChars": 33308 }, { "source": "web_fetch", - "query": "https://supabase.com/docs/guides/storage/security/access-control.md", + "query": "https://supabase.com/changelog.md", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/security/access-control.md" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 4169 + "resultChars": 94610 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"createSignedUrl storage javascript expiresIn download\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", + "title": "Storage Image Transformations" + }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" + } + ], + "resultChars": 19572 } ] }, @@ -59051,12 +49328,12 @@ { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "5 passed, 3 failed" + "notes": "11 passed, 4 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies `posts` as having the tenant-isolation flaw, explains the missing org correlation, and grounds the conclusion in pgTAP failures showing cross-org post reads. It also correctly states that `notes` isolation passes." + "judgeNotes": "Correctly identifies `posts` as the broken tenant-isolation policy and grounds the conclusion in pgTAP failure 9 and live reproduction; it correctly distinguishes `notes` as isolated." } ], "skills": { @@ -59070,19 +49347,7 @@ ] }, "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 94610 - } - ] + "calls": [] }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", @@ -59113,17 +49378,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "2 file(s): supabase/tests/tenant_isolation_happy_path_test.sql, supabase/tests/tenant_isolation_negative_test.sql" + "notes": "1 file(s): supabase/tests/tenant_isolation.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "2 passed, 2 failed" + "notes": "13 passed, 0 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies the `posts` SELECT policy as allowing cross-organization reads and grounds this in the pgTAP failures. It also correctly distinguishes `notes` as properly isolated." + "judgeNotes": "Correctly identifies the `posts` RLS policy as allowing cross-tenant reads, grounds this in the two failing pgTAP assertions, and notes that `notes` isolation passed." } ], "skills": { @@ -59137,19 +49402,7 @@ ] }, "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 94610 - } - ] + "calls": [] }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", @@ -59180,17 +49433,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" + "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "11 passed, 1 failed" + "notes": "9 passed, 5 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies `posts` as the broken tenant-isolation policy, grounded in the failing pgTAP cross-org read test and live verification. It correctly distinguishes `notes` as isolated." + "judgeNotes": "Correctly identifies `posts` as the tenant-isolation flaw and grounds the conclusion in pgTAP cross-tenant read failures, while recognizing `notes` isolation as correct." } ], "skills": { @@ -59234,9 +49487,37 @@ "passed": false, "checks": [ { - "name": "created auth sessions", + "name": "document_sections.embedding is vector(384)", "passed": false, - "notes": "Internal server error" + "notes": "no embedding column" + }, + { + "name": "HNSW index on the embedding column", + "passed": false, + "notes": "no index on embedding column" + }, + { + "name": "index operator class matches the search operator", + "passed": false, + "notes": "match_document_sections not found" + }, + { + "name": "user A search returns only own sections, best match first", + "passed": false, + "notes": "Could not find the function public.match_document_sections(match_count, query_embedding) in the schema cache" + }, + { + "name": "user B search returns only own sections, best match first", + "passed": false, + "notes": "Could not find the function public.match_document_sections(match_count, query_embedding) in the schema cache" + }, + { + "name": "user A reads only own sections through the API", + "passed": false + }, + { + "name": "user A reads only own documents through the API", + "passed": false } ], "skills": { @@ -59253,42 +49534,23 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"semantic search pgvector match documents function embedding column\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"semantic search pgvector match documents function hnsw\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, { "url": "https://supabase.com/docs/guides/ai/hybrid-search", "title": "Hybrid search" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/ai/vector-columns", - "title": "Vector columns" - } - ], - "resultChars": 70040 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/ai/vector-indexes", + "title": "Vector indexes" } ], - "resultChars": 94610 + "resultChars": 29477 } ] }, @@ -59317,12 +49579,38 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { - "name": "created auth sessions", - "passed": false, - "notes": "Internal server error" + "name": "document_sections.embedding is vector(384)", + "passed": true, + "notes": "vector(384)" + }, + { + "name": "HNSW index on the embedding column", + "passed": true, + "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + }, + { + "name": "index operator class matches the search operator", + "passed": true, + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + }, + { + "name": "user A search returns only own sections, best match first", + "passed": true + }, + { + "name": "user B search returns only own sections, best match first", + "passed": true + }, + { + "name": "user A reads only own sections through the API", + "passed": true + }, + { + "name": "user A reads only own documents through the API", + "passed": true } ], "skills": { @@ -59339,31 +49627,31 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"pgvector semantic search match_document_sections RPC function vector extension\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"gte-small embedding dimensions semantic search match_documents rpc pgvector\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" + "url": "https://supabase.com/docs/guides/ai/vector-columns", + "title": "Vector columns" }, { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" + "url": "https://supabase.com/docs/guides/ai/hybrid-search", + "title": "Hybrid search" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" + "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", + "title": "Choosing your Compute Add-on" }, { - "url": "https://supabase.com/docs/guides/ai/vector-columns", - "title": "Vector columns" + "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", + "title": "Semantic Search" } ], - "resultChars": 45321 + "resultChars": 61240 } ] }, @@ -59402,12 +49690,12 @@ { "name": "HNSW index on the embedding column", "passed": true, - "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { "name": "index operator class matches the search operator", "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { "name": "user A search returns only own sections, best match first", @@ -59437,7 +49725,36 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"semantic search pgvector match_documents function embedding column RLS\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/hybrid-search", + "title": "Hybrid search" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + }, + { + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" + }, + { + "url": "https://supabase.com/docs/guides/ai/vector-columns", + "title": "Vector columns" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + } + ], + "resultChars": 78090 + } + ] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", @@ -59462,7 +49779,7 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "preserved existing app scrape job", @@ -59470,13 +49787,13 @@ }, { "name": "configured the Supabase Metrics API scrape correctly", - "passed": false, - "judgeNotes": "No Supabase Metrics API scrape is configured. prometheus.yml only preserves the app job; it lacks the required HTTPS project target, metrics path, Basic Auth password_file, and docker-compose secret/volume mount." + "passed": true, + "judgeNotes": "Meets all requirements: HTTPS Supabase target, correct metrics path, Basic Auth with password_file, matching read-only secret mount, and preserved app scrape." }, { "name": "documented live deployment and verification steps", - "passed": false, - "judgeNotes": "README lacks Secret API key creation and matching secret-file placement, does not restart/reload the Compose stack after configuration, and provides no concrete verification through Prometheus targets, PromQL, Grafana, or equivalent." + "passed": true, + "judgeNotes": "README documents Secret API key creation, matching secret file placement, Compose restart or Prometheus reload, and concrete verification via /targets and PromQL." } ], "skills": { @@ -59484,10 +49801,57 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 94610 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/platform/metrics.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/metrics.md" + } + ], + "resultChars": 5326 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/observability/metrics/grafana-self-hosted.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-self-hosted.md" + } + ], + "resultChars": 3757 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/observability/metrics/vendor-agnostic.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/observability/metrics/vendor-agnostic.md" + } + ], + "resultChars": 5052 + } + ] }, "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", @@ -59521,12 +49885,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "HTTPS Supabase metrics scrape is correctly configured with the required path, Basic Auth password_file, matching mounted secrets directory, project target, and preserved app job." + "judgeNotes": "HTTPS Supabase metrics scrape uses the correct path and project target, Basic Auth with password_file, matching Docker volume mount, and preserves the app job." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README provides correct Secret API key creation, matching mounted secret-file path, Compose start/reload commands, and concrete verification via Prometheus targets and Grafana." + "judgeNotes": "README documents creating an sb_secret API key, storing it in the matching mounted secret file, starting/reloading Compose, and verifying the Supabase scrape via Prometheus targets or an up{job=\"supabase\"} query." } ], "skills": { @@ -59541,42 +49905,74 @@ "docs": { "calls": [ { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | grep -i -B2 -A2 -E \"metric|prometheus|observability|breaking\" | head -60", + "source": "search_docs", + "query": "{ searchDocs(query: \"metrics endpoint prometheus scrape project metrics API\", limit: 8) { nodes { title href content } } }", + "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", + "title": "Scrape a project's metrics" + }, + { + "url": "https://supabase.com/docs/guides/observability/metrics", + "title": "Metrics API" + }, + { + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" + }, + { + "url": "https://supabase.com/docs/guides/observability/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" + }, + { + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" + }, + { + "url": "https://supabase.com/docs/guides/observability/access-data", + "title": "Observe the data" } ], - "resultChars": 4160 + "resultChars": 22374 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"metrics endpoint prometheus scrape project metrics\", limit: 8) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"management API create project API key secret key sb_secret\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", - "title": "Scrape a project's metrics" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", + "title": "Build a User Management App with Angular" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" + "url": "https://supabase.com/docs/reference/api/v1-create-project-api-key", + "title": "Creates a new API key for the project" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + } + ], + "resultChars": 112617 + }, + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | grep -i -B2 -A2 \"metric\" | head -40", + "pages": [ + { + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 19934 + "resultChars": 1183 } ] }, @@ -59603,7 +49999,7 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { "name": "preserved existing app scrape job", @@ -59611,13 +50007,13 @@ }, { "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "HTTPS Supabase target, required metrics path, Basic Auth with password_file, preserved app scrape, and matching Docker volume mount are all present." + "passed": false, + "judgeNotes": "No Supabase Metrics API scrape is configured, and no password_file secret is mounted in docker-compose.yml." }, { "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README documents Secret API key creation, matching mounted secret file, stack start/reload, and concrete verification through Prometheus targets and authenticated metrics endpoint." + "passed": false, + "judgeNotes": "README lacks Secret API key creation and matching secret-file setup, does not restart/reload the Compose stack after configuration, and provides no concrete verification via Prometheus targets, PromQL, Grafana, or equivalent." } ], "skills": { @@ -59631,44 +50027,53 @@ }, "docs": { "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 94610 - }, { "source": "search_docs", - "query": "{ searchDocs(query: \"metrics endpoint prometheus scrape hosted project\", limit: 8) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"metrics endpoint prometheus scrape hosted project\") { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-self-hosted", "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", + "url": "https://supabase.com/docs/guides/observability/metrics", "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" + "url": "https://supabase.com/docs/guides/observability/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" }, { "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", "title": "Scrape a project's metrics" + }, + { + "url": "https://supabase.com/docs/guides/observability/access-data", + "title": "Observe the data" + }, + { + "url": "https://supabase.com/docs/guides/observability/automate-with-agents/usage", + "title": "Capacity monitor" + }, + { + "url": "https://supabase.com/docs/guides/platform/read-replicas", + "title": "Read Replicas" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", + "title": "How to View Database Metrics" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/grafana-not-displaying-data-sXJrMj", + "title": "Grafana not displaying data" } ], - "resultChars": 19934 + "resultChars": 36223 } ] }, @@ -59695,22 +50100,21 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "WEATHER_API_KEY is set as a Function secret on the project", - "passed": false, - "notes": "secrets present: []" + "passed": true }, { "name": "the weather function is deployed to the project", - "passed": false, - "notes": "function not found on the project (status 404)" + "passed": true, + "notes": "status ACTIVE" }, { "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": false, - "notes": "could not read supabase/functions/weather/*" + "passed": true, + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -59722,7 +50126,9 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] @@ -59820,7 +50226,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment via Deno.env.get." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -59840,41 +50246,31 @@ "calls": [ { "source": "search_docs", - "query": "{searchDocs(query:\"deploy edge function CLI secrets set env file\",limit:5){nodes{title href content}}}", + "query": "{ searchDocs(query: \"edge function environment variables secrets Deno.env.get deploy\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", - "title": "Transcription Telegram Bot" + "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", + "title": "Inspecting edge function environment variables" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", "title": "Automatic embeddings" - } - ], - "resultChars": 58240 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | grep -i -B2 -A2 -E \"edge function|secrets|deno\" | head -60", - "pages": [ + }, { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" } ], - "resultChars": 3522 + "resultChars": 60341 } ] }, @@ -59933,17 +50329,6 @@ }, "docs": { "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 94610 - }, { "source": "web_fetch", "query": "https://supabase.com/docs/guides/self-hosting/docker.md", @@ -59953,7 +50338,7 @@ "url": "https://supabase.com/docs/guides/self-hosting/docker.md" } ], - "resultChars": 28536 + "resultChars": 29564 } ] }, @@ -60014,36 +50399,34 @@ "calls": [ { "source": "web_fetch", - "query": "https://supabase.com/changelog/48048-self-hosted-supabase-envoy-becomes-the-default-api-gateway-b", + "query": "https://supabase.com/docs/guides/self-hosting/docker.md", "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog/48048-self-hosted-supabase-envoy-becomes-the-default-api-gateway-b" + "url": "https://supabase.com/docs/guides/self-hosting/docker.md" } ], - "resultChars": 5943 + "resultChars": 29564 }, { - "source": "web_fetch", - "query": "https://supabase.com/changelog/47093-self-hosted-supabase-api-external-url-to-include-auth-v1", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | head -100", "pages": [ { - "url": "https://supabase.com/changelog/47093-self-hosted-supabase-api-external-url-to-include-auth-v1" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 4755 + "resultChars": 5442 }, { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/self-hosting/docker.md", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | sed -n '100,200p'", "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker.md" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 28536 + "resultChars": 6080 } ] }, @@ -60103,413 +50486,53 @@ "docs": { "calls": [ { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | head -100", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 94610 + "resultChars": 5442 }, { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/self-hosting/docker.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/docker.md" - } - ], - "resultChars": 28536 - } - ] - }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "run": 3, - "sourcePath": "opencode-kimi-k3/deploy-self-hosting-001-docker-compose/run-3/result.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", - "product": [ - "auth" - ], - "topic": [ - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true - }, - { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" - }, - { - "name": "deleted user's refresh token is rejected", - "passed": true - }, - { - "name": "deleted user cannot sign back in", - "passed": true - }, - { - "name": "other users keep their sessions and access", - "passed": true - }, - { - "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "Correctly diagnoses the soft-delete issue, implements hard deletion with session/refresh-token revocation, accurately explains the remaining stateless JWT expiry window and mitigations, and correctly distinguishes frontend publishable keys from server-only RLS-bypassing secret keys." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key vs legacy anon service_role API keys RLS\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/quickstarts/refine", - "title": "Use Supabase with Refine" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - } - ], - "resultChars": 65078 - } - ] - }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "run": 1, - "sourcePath": "opencode-kimi-k3/investigate-auth-001-deleted-user-access/run-1/result.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", - "product": [ - "auth" - ], - "topic": [ - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true - }, - { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" - }, - { - "name": "deleted user's refresh token is rejected", - "passed": true - }, - { - "name": "deleted user cannot sign back in", - "passed": true - }, - { - "name": "other users keep their sessions and access", - "passed": true - }, - { - "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "Correctly identifies the soft-delete flaw, implements real auth-user/session/refresh-token removal, accurately explains the remaining stateless JWT expiry window with mitigations, and correctly distinguishes frontend publishable keys from server-only RLS-bypassing secret keys." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | sed -n '100,220p'", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 94610 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"delete user account self-service security definer function auth.users\") { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", - "title": "deleteUser()" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/reference/kotlin/auth-admin-deleteuser", - "title": "deleteUser()" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-admin-listusers" - }, - { - "url": "https://supabase.com/docs/reference/python/auth-admin-deleteuser", - "title": "delete_user()" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-listusers" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/dashboard-errors-when-managing-users-N1ls4A", - "title": "Errors when creating / updating / deleting users" - }, - { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0029_authenticated_security_definer_function_executable", - "title": "Database Advisor: Lint 0029_authenticated_security_definer_function_executable" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-passkey-delete" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-admin-listusers", - "title": "listUsers()" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deletefactor" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/testing-best-practices", - "title": "SSO Testing and Best Practices" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso", - "title": "Enable SSO for Your Organization" - } - ], - "resultChars": 97410 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"publishable secret API keys vs legacy anon service_role RLS\") { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/quickstarts/refine", - "title": "Use Supabase with Refine" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/rotating-anon-service-and-jwt-secrets-1Jq6yd", - "title": "Rotating Anon, Service, and JWT Secrets" - }, - { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0012_auth_allow_anonymous_sign_ins", - "title": "Database Advisor: Lint 0012_auth_allow_anonymous_sign_ins" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles", - "title": "Postgres Roles" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/deprecated-rls-features-Pm77Zs", - "title": "Deprecated RLS features" - }, - { - "url": "https://supabase.com/docs/guides/storage/s3/authentication", - "title": "S3 Authentication" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - }, - { - "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", - "title": "Custom Claims & Role-based Access Control (RBAC)" - } - ], - "resultChars": 12 + "resultChars": 7168 }, { "source": "web_fetch", - "query": "https://supabase.com/docs/guides/api/api-keys.md", + "query": "https://supabase.com/changelog/48048-self-hosted-supabase-envoy-becomes-the-default-api-gateway-b", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/api/api-keys.md" + "url": "https://supabase.com/changelog/48048-self-hosted-supabase-envoy-becomes-the-default-api-gateway-b" } ], - "resultChars": 13494 + "resultChars": 5943 }, { "source": "web_fetch", - "query": "https://supabase.com/docs/guides/auth/managing-user-data.md", + "query": "https://supabase.com/docs/guides/self-hosting/docker.md", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/managing-user-data.md" + "url": "https://supabase.com/docs/guides/self-hosting/docker.md" } ], - "resultChars": 7386 + "resultChars": 29564 } ] }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "run": 2, - "sourcePath": "opencode-kimi-k3/investigate-auth-001-deleted-user-access/run-2/result.json" + "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", + "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", + "run": 3, + "sourcePath": "opencode-kimi-k3/deploy-self-hosting-001-docker-compose/run-3/result.json" }, { "experiment": "opencode-kimi-k3", @@ -60560,7 +50583,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "Correctly diagnoses the ineffective soft delete, implements auth-user deletion with cascading session/refresh-token revocation, accurately explains the remaining stateless JWT window and mitigations, and correctly distinguishes frontend publishable keys from server-only RLS-bypassing secret keys." + "judgeNotes": "Correctly diagnoses soft deletion, implements auth-user deletion with session/refresh-token revocation, accurately explains the stale JWT window and mitigations, and distinguishes publishable frontend keys from server-only secret keys that bypass RLS." } ], "skills": { @@ -60577,43 +50600,163 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"delete user account self serve RPC security definer function delete auth user\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"publishable key secret key vs anon service_role API keys RLS\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + "url": "https://supabase.com/docs/guides/getting-started/quickstarts/refine", + "title": "Use Supabase with Refine" }, { - "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", - "title": "deleteUser()" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" + } + ], + "resultChars": 65078 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"delete user account self-service RPC delete auth.users cascade sessions\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/guides/auth/managing-user-data", "title": "User Management" }, { - "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" }, { - "url": "https://supabase.com/docs/reference/python/auth-admin-deleteuser", - "title": "delete_user()" + "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", + "title": "deleteUser()" + }, + { + "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", + "title": "Multiple SSO Providers" } ], - "resultChars": 7953 + "resultChars": 39771 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"publishable key secret key vs anon service_role legacy API keys RLS\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"new API keys publishable sb_publishable secret sb_secret migrate from anon service_role\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/quickstarts/refine", - "title": "Use Supabase with Refine" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { "url": "https://supabase.com/docs/guides/auth/signing-keys", "title": "JWT Signing Keys" }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + } + ], + "resultChars": 60811 + } + ] + }, + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "run": 1, + "sourcePath": "opencode-kimi-k3/investigate-auth-001-deleted-user-access/run-1/result.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", + "product": [ + "auth" + ], + "topic": [ + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "victim session active before delete-account", + "passed": true + }, + { + "name": "delete_account flow ran for the victim", + "passed": true + }, + { + "name": "delete-account revokes the user's sessions", + "passed": true, + "notes": "sessions left: 0" + }, + { + "name": "deleted user's refresh token is rejected", + "passed": true + }, + { + "name": "deleted user cannot sign back in", + "passed": true + }, + { + "name": "other users keep their sessions and access", + "passed": true + }, + { + "name": "diagnosed and explained session revocation", + "passed": true, + "judgeNotes": "Correctly diagnoses the ineffective soft delete, implements and verifies auth-user/session/refresh-token removal, accurately explains the stale JWT expiry window and mitigation, and correctly distinguishes frontend publishable keys from server-only RLS-bypassing secret keys." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable key secret key vs legacy anon service_role API keys RLS\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/quickstarts/refine", + "title": "Use Supabase with Refine" + }, { "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", "title": "Migrating to publishable and secret API keys" @@ -60631,19 +50774,91 @@ }, { "source": "web_fetch", - "query": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys.md", + "query": "https://supabase.com/docs/guides/getting-started/api-keys.md", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys.md" + "url": "https://supabase.com/docs/guides/getting-started/api-keys.md" } ], - "resultChars": 13042 + "resultChars": 22207 } ] }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "run": 2, + "sourcePath": "opencode-kimi-k3/investigate-auth-001-deleted-user-access/run-2/result.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", + "product": [ + "auth" + ], + "topic": [ + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "victim session active before delete-account", + "passed": true + }, + { + "name": "delete_account flow ran for the victim", + "passed": true + }, + { + "name": "delete-account revokes the user's sessions", + "passed": false, + "notes": "sessions left: 1" + }, + { + "name": "deleted user's refresh token is rejected", + "passed": false, + "notes": "refresh token still produces a session" + }, + { + "name": "deleted user cannot sign back in", + "passed": false, + "notes": "deleted account can still sign in" + }, + { + "name": "other users keep their sessions and access", + "passed": true + }, + { + "name": "diagnosed and explained session revocation", + "passed": false, + "judgeNotes": "No diagnosis, fix, JWT revocation-window explanation, or publishable-vs-secret key clarification was provided; the response only initiated database inspection." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", "run": 3, "sourcePath": "opencode-kimi-k3/investigate-auth-001-deleted-user-access/run-3/result.json" }, @@ -60692,7 +50907,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified the missing orders table in supabase_realtime, added it with ALTER PUBLICATION, and preserved RLS, policies, and courier_locations." + "judgeNotes": "Correctly identified the missing orders publication membership, added public.orders to supabase_realtime, and preserved RLS, policies, and courier_locations." } ], "skills": { @@ -60701,8 +50916,7 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { @@ -60758,7 +50972,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified the missing orders table in supabase_realtime, added it without altering existing tables, RLS, or policies, and verified both feeds remain published." + "judgeNotes": "Correctly identified the missing orders table in supabase_realtime and added it with ALTER PUBLICATION, preserving courier_locations, RLS, and existing policies." } ], "skills": { @@ -60824,7 +51038,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified the missing orders table in supabase_realtime, added it with ALTER PUBLICATION, and preserved RLS, policies, and courier_locations." + "judgeNotes": "Correctly identified the missing orders table in supabase_realtime, added it with ALTER PUBLICATION, verified both feeds remain published, and preserved RLS and policies." } ], "skills": { @@ -60867,17 +51081,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform and clearly described the recurring intermittent HTTP 503 pattern: 8 failures from 07:00Z through 12:00Z on 2026-04-28." + "judgeNotes": "Identified `image-transform` as the affected function and reported 8 recurring HTTP 503 failures from 07:00–12:00 UTC on 2026-04-28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": false, - "judgeNotes": "It correctly attributes the 503s to the gateway/platform and cites missing invocation metadata, but explicitly recommends redeploying the function as remediation, which the rubric defines as a failure." + "judgeNotes": "It attributes the 503s to the gateway layer, but provides no valid supporting observation such as absent invocation/runtime rows, unchanged deployment ID, or contrast with a function-level error." }, { "name": "recommended a concrete next step", - "passed": true, - "judgeNotes": "Provides multiple concrete actions, including redeploying the function and opening a support ticket with specific gateway request IDs and the affected time window." + "passed": false, + "judgeNotes": "The response identifies a specific 503 pattern and time window but does not recommend any concrete next action, escalation, configuration review, or correlated infrastructure investigation." } ], "skills": { @@ -60920,17 +51134,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified `image-transform` and clearly described the recurring eight HTTP 503 gateway failures from 07:00Z–12:00Z on 2026-04-28." + "judgeNotes": "Identified `image-transform` and the recurring eight HTTP 503 gateway failures across 07:00Z–12:00Z on 2026-04-28; correctly treated older `billing-webhook` errors as unrelated." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": false, - "judgeNotes": "Correctly attributes 503s to the gateway/platform and cites missing runtime logs plus unchanged deployment, but explicitly recommends redeploying the function as remediation, which the rubric defines as a failure." + "judgeNotes": "Although it notes the 503s exist only in gateway logs with no invocation rows, it ultimately blames function boot/dependency resolution and recommends pinning dependencies and redeploying. The rubric requires attribution to the gateway/platform layer, not the function runtime or code." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Recommended concrete actions: redeploy the function, verify via repeated invocations and edge logs, and escalate to Supabase with gateway request IDs, time window, and region if failures persist." + "judgeNotes": "The assistant gave multiple concrete actions: pin dependencies, add lock/import configuration, test locally, redeploy, monitor specific Edge logs, add retries, and instrument the separate 500 error." } ], "skills": { @@ -60943,7 +51157,36 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function 503 error cold start boot failure npm dependencies pin version\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/status-codes", + "title": "Status codes" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", + "title": "Edge Function 503 error response" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" + }, + { + "url": "https://supabase.com/docs/guides/security/npm-security", + "title": "Securing npm installs" + }, + { + "url": "https://supabase.com/docs/guides/functions/routing", + "title": "Handling Routing in Functions" + } + ], + "resultChars": 53812 + } + ] }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", @@ -60973,17 +51216,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified `image-transform` as affected and documented all eight recurring HTTP 503 gateway failures from 07:00Z through 12:00Z on 2026-04-28, while correctly distinguishing older `billing-webhook` errors." + "judgeNotes": "Identified `image-transform` as the affected function and explicitly documented all 8 recurring gateway HTTP 503 failures from 07:00Z through 12:00Z on 2026-04-28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": false, - "judgeNotes": "Although it correctly identifies gateway-only 503s with no matching function executions, it then blames an unpinned function dependency and recommends fixing/redeploying the function as the primary remediation, which violates the rubric." + "judgeNotes": "Although it cites gateway-only 503s with no invocations, it speculates that an unpinned function dependency caused worker boot failures and recommends pinning/redeploying the function. This contradicts the required platform-layer attribution and prohibited remediation." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Recommended concrete actions: pin and redeploy the dependency, verify correlated gateway/function logs, and escalate persistent 503s with specific request IDs." + "judgeNotes": "The assistant provided multiple concrete next steps, including pinning and redeploying the dependency, testing afterward, escalating to Supabase with exact gateway request IDs and dates, checking regional platform status, and adding targeted alerting." } ], "skills": { @@ -61053,7 +51296,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and added authenticated-only owner-scoped SELECT and INSERT policies using auth.uid()." + "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and added authenticated-only SELECT and INSERT policies scoped by user_id = auth.uid() using USING and WITH CHECK." } ], "skills": { @@ -61124,7 +51367,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly diagnosed deny-all RLS with no policies and added authenticated-only SELECT and INSERT policies scoped by user_id = auth.uid(), while keeping RLS enabled." + "judgeNotes": "Correctly diagnosed deny-all RLS, kept RLS enabled, and added authenticated-only SELECT and INSERT policies scoped to auth.uid()." } ], "skills": { @@ -61195,7 +51438,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and added authenticated owner-scoped SELECT and INSERT policies using auth.uid() with USING and WITH CHECK." + "judgeNotes": "Correctly diagnosed deny-all RLS, kept RLS enabled, and added authenticated owner-scoped SELECT and INSERT policies using auth.uid() with USING and WITH CHECK." } ], "skills": { @@ -61259,7 +51502,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "`supabase db push` (#21) applied `20240220000000_add_avatar_url.sql` and finished successfully. History was reconciled by adding the missing local `20240115000000_add_profile_bio.sql` (#19), after which `supabase migration list` matched and the successful push proceeded. No prohibited mutation workaround was used; the direct `psql` commands were read-only inspections." + "judgeNotes": "`supabase migration fetch --db-url ...` plus adding the local bio migration reconciled history; `supabase db push` then applied `20240220000000_add_avatar_url.sql`, showing both “Applying migration” and “Finished supabase db push.” The `psql` commands were read-only; no prohibited workaround occurred." } ], "skills": { @@ -61322,7 +51565,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "`supabase db push` applied `20240220000000_add_avatar_url.sql`, showing both “Applying migration” and “Finished supabase db push.” History was reconciled by adding `20240115000000_add_profile_bio.sql` locally and then running the successful push; the final migration list matched local and remote. The `psql` commands were read-only inspections; no prohibited workaround was used." + "judgeNotes": "`supabase db push` applied `20240220000000_add_avatar_url.sql`, showing both “Applying migration” and “Finished supabase db push.” History was reconciled by adding the matching local `20240115000000_add_profile_bio.sql`, after which `supabase migration list` matched and the push succeeded. The psql commands were read-only; no prohibited workaround occurred." } ], "skills": { @@ -61385,7 +51628,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "`supabase db push --yes` applied `20240220000000_add_avatar_url.sql` and finished successfully. The orphan bio history was reconciled by fetching/adding `20240115000000_add_profile_bio.sql` locally, then pushing. The `psql` commands were read-only inspections; no prohibited workaround was used." + "judgeNotes": "`supabase db push` applied `20240220000000_add_avatar_url.sql`, showing both “Applying migration” and “Finished supabase db push.” History was reconciled by adding `20240115000000_add_profile_bio.sql` locally, then pushing; final migration list matched. All `psql` usage was read-only; no workaround was used." } ], "skills": { @@ -61502,7 +51745,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_recent (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -61563,7 +51806,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_recent (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -61840,7 +52083,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "auth module loads and the driver completes", @@ -61850,7 +52093,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 06221c92-fac6-4682-b99e-100c67d4cabd, signUp returned {\"userId\":\"06221c92-fac6-4682-b99e-100c67d4cabd\"}" + "notes": "db user 173dadf9-d3ca-4ced-9b23-d939a60abe71, signUp returned {\"userId\":\"173dadf9-d3ca-4ced-9b23-d939a60abe71\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -61865,7 +52108,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"06221c92-fac6-4682-b99e-100c67d4cabd\"}" + "notes": "{\"userId\":\"173dadf9-d3ca-4ced-9b23-d939a60abe71\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -61879,8 +52122,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/src/auth.mjs" + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" } ], "skills": { @@ -61916,7 +52159,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "auth module loads and the driver completes", @@ -61926,7 +52169,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 35a7a6fe-1c08-4b37-aab7-5f3122131828, signUp returned {\"userId\":\"35a7a6fe-1c08-4b37-aab7-5f3122131828\"}" + "notes": "db user 110ee48f-ab75-4db5-8665-b1100b26a2c0, signUp returned {\"userId\":\"110ee48f-ab75-4db5-8665-b1100b26a2c0\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -61941,7 +52184,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"35a7a6fe-1c08-4b37-aab7-5f3122131828\"}" + "notes": "{\"userId\":\"110ee48f-ab75-4db5-8665-b1100b26a2c0\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -61955,8 +52198,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "passed": true, + "notes": "imports found in: app/src/auth.mjs" } ], "skills": { @@ -61964,7 +52207,26 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"createClient publishable key signUp signInWithPassword getUser\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", + "title": "signInWithPassword()" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" + } + ], + "resultChars": 2458 + } + ] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", @@ -62002,7 +52264,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 38d00280-5649-4cdc-b90c-27d9e52bee69, signUp returned {\"userId\":\"38d00280-5649-4cdc-b90c-27d9e52bee69\"}" + "notes": "db user 16409f55-c6b0-4afd-97fc-d93e2100cb2b, signUp returned {\"userId\":\"16409f55-c6b0-4afd-97fc-d93e2100cb2b\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -62017,7 +52279,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"38d00280-5649-4cdc-b90c-27d9e52bee69\"}" + "notes": "{\"userId\":\"16409f55-c6b0-4afd-97fc-d93e2100cb2b\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -62080,7 +52342,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 3 rows" + "notes": "found 2 rows" }, { "name": "row level security is enabled on todos", @@ -62098,7 +52360,7 @@ { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "3 rows" + "notes": "2 rows" } ], "skills": { @@ -62146,7 +52408,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 2 rows" + "notes": "found 3 rows" }, { "name": "row level security is enabled on todos", @@ -62164,7 +52426,7 @@ { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "2 rows" + "notes": "3 rows" } ], "skills": { @@ -62212,7 +52474,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 3 rows" + "notes": "found 2 rows" }, { "name": "row level security is enabled on todos", @@ -62230,7 +52492,7 @@ { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "3 rows" + "notes": "2 rows" } ], "skills": { @@ -62432,59 +52694,7 @@ { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 37) from the queue" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "run": 1, - "sourcePath": "opencode-kimi-k3-no-skills/build-cli-003-pg-cron-queue-workflow/run-1/result.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" - }, - { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" - }, - { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 40) from the queue" + "notes": "function removed the seeded message (id 37) from the queue" } ], "skills": { @@ -62495,38 +52705,38 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron schedule job pgmq queue send every minute edge function read delete messages\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pgmq queues edge function read delete messages cron schedule\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, { "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", "title": "Automatic embeddings" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" }, { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" }, { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" } ], - "resultChars": 58555 + "resultChars": 77267 } ] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "run": 2, - "sourcePath": "opencode-kimi-k3-no-skills/build-cli-003-pg-cron-queue-workflow/run-2/result.json" + "run": 1, + "sourcePath": "opencode-kimi-k3-no-skills/build-cli-003-pg-cron-queue-workflow/run-1/result.json" }, { "experiment": "opencode-kimi-k3-no-skills", @@ -62560,12 +52770,12 @@ { "name": "cron command enqueues to the 'tasks' queue", "passed": true, - "notes": "queue depth 1 -> 2" + "notes": "queue depth 0 -> 1" }, { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 41) from the queue" + "notes": "function removed the seeded message (id 37) from the queue" } ], "skills": { @@ -62576,7 +52786,7 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron schedule job every minute send message to pgmq queue cron.schedule pgmq.send\") { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pg_cron schedule job every minute pgmq queue send message\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -62591,28 +52801,93 @@ "url": "https://supabase.com/docs/guides/cron", "title": "Cron" }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, { "url": "https://supabase.com/docs/guides/functions/schedule-functions", "title": "Scheduling Edge Functions" + } + ], + "resultChars": 68390 + } + ] + }, + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "run": 2, + "sourcePath": "opencode-kimi-k3-no-skills/build-cli-003-pg-cron-queue-workflow/run-2/result.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-cli-003-pg-cron-queue-workflow", + "stage": "build", + "product": [ + "database", + "edge-functions", + "cron", + "queues" + ], + "topic": [ + "sql", + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "passed": true, + "notes": "schedule='* * * * *', active=true" + }, + { + "name": "cron command enqueues to the 'tasks' queue", + "passed": true, + "notes": "queue depth 0 -> 1" + }, + { + "name": "process-tasks function drains the queue", + "passed": true, + "notes": "function removed the seeded message (id 37) from the queue" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"pgmq queues cron schedule send message\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" }, { "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", "title": "Automatic embeddings" }, { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart", + "url": "https://supabase.com/docs/guides/queues/quickstart", "title": "Quickstart" } ], - "resultChars": 84995 + "resultChars": 75189 } ] }, @@ -62761,7 +53036,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "report runs and prints JSON", @@ -62780,8 +53055,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/report.mjs" + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" }, { "name": "report queries via the Data API, not raw SQL", @@ -63162,7 +53437,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "rejects missing auth", @@ -63176,17 +53451,17 @@ }, { "name": "reads only with the caller's JWT", - "passed": false, - "notes": "bearer_tokens=1, all_match=false" + "passed": true, + "notes": "bearer_tokens=2, all_match=true" }, { "name": "user A cannot force-read user B note", - "passed": false, + "passed": true, "notes": "status=200" }, { "name": "user B cannot force-read user A note", - "passed": false, + "passed": true, "notes": "status=200" } ], @@ -63257,28 +53532,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions verify JWT get authenticated user RLS authorization header\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - } - ], - "resultChars": 16547 - } - ] + "calls": [] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", @@ -63340,13 +53594,116 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function get authenticated user from Authorization header JWT auth.getUser\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-getuser" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + } + ], + "resultChars": 9611 + } + ] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", "run": 3, "sourcePath": "opencode-kimi-k3-no-skills/build-functions-004-service-role-bypass/run-3/result.json" }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-functions-005-dual-auth-user-secret", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "sdk", + "rls", + "security" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" + }, + { + "name": "rejects request with no credentials", + "passed": true, + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"MISSING_CREDENTIALS\",\"message\":\"[@supabase/server] No credentials found on the request. This endpoint accepts auth mode(s):" + }, + { + "name": "user with JWT reads only their own rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"1bc290d5-eb39-492a-82a6-09f115338b88\",\"metric\":\"steps_a_mtsfyzsv\",\"value\":111}]" + }, + { + "name": "user cannot read another user's rows by passing user_id", + "passed": true, + "notes": "status 200: [{\"user_id\":\"1bc290d5-eb39-492a-82a6-09f115338b88\",\"metric\":\"steps_a_mtsfyzsv\",\"value\":111}]" + }, + { + "name": "service key bypasses RLS to read the target user's rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"c89c480c-31c1-46b3-9347-b2c8c2a2afe6\",\"metric\":\"steps_b_mtsfyzsv\",\"value\":222}]" + }, + { + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" + }, + { + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_JWT\",\"message\":\"[@supabase/server] The JWT in the Authorization header failed verification: its header is missing \\" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 401: {\"source\":\"@supabase/server\",\"code\":\"INVALID_API_KEY\",\"message\":\"[@supabase/server] The apikey header matched no key configured for auth mode(s): \\\"user\\\", \\\"se" + }, + { + "name": "implementation uses @supabase/server", + "passed": true, + "notes": "imports @supabase/server / withSupabase" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + "run": 1, + "sourcePath": "opencode-kimi-k3-no-skills/build-functions-005-dual-auth-user-secret/run-1/result.json" + }, { "experiment": "opencode-kimi-k3-no-skills", "experimentSuite": "no-skills", @@ -63380,27 +53737,27 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"error\":\"missing bearer token\"}" + "notes": "status 401: {\"error\":\"authentication required\"}" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: {\"stats\":[{\"user_id\":\"43f024d8-2428-4c7a-a922-6a2ef72843a6\",\"metric\":\"steps_a_mta4f0ow\",\"value\":111}]}" + "notes": "status 200: [{\"user_id\":\"ff8ed979-26ed-43fd-9fb0-6d02e7c41ab3\",\"metric\":\"steps_a_mtsfxjkv\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: {\"stats\":[{\"user_id\":\"43f024d8-2428-4c7a-a922-6a2ef72843a6\",\"metric\":\"steps_a_mta4f0ow\",\"value\":111}]}" + "notes": "status 200: [{\"user_id\":\"ff8ed979-26ed-43fd-9fb0-6d02e7c41ab3\",\"metric\":\"steps_a_mtsfxjkv\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: {\"stats\":[{\"user_id\":\"4a37ac0e-5396-490f-9b0a-a25a35e9bb47\",\"metric\":\"steps_b_mta4f0ow\",\"value\":222}]}" + "notes": "status 200: [{\"user_id\":\"9996a82a-8c8f-422a-8702-c748b7b22a58\",\"metric\":\"steps_b_mtsfxjkv\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"error\":\"missing bearer token\"}" + "notes": "status 401: {\"error\":\"authentication required\"}" }, { "name": "rejects an unverified (forged) user token", @@ -63410,7 +53767,7 @@ { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"error\":\"missing bearer token\"}" + "notes": "status 401: {\"error\":\"authentication required\"}" }, { "name": "implementation uses @supabase/server", @@ -63426,38 +53783,38 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"edge functions environment variables SUPABASE_SECRET_KEY SUPABASE_PUBLISHABLE_KEY new API keys\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"edge function verify apikey secret key service role new API keys sb_secret\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" } ], - "resultChars": 65829 + "resultChars": 98556 } ] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "run": 1, - "sourcePath": "opencode-kimi-k3-no-skills/build-functions-005-dual-auth-user-secret/run-1/result.json" + "run": 2, + "sourcePath": "opencode-kimi-k3-no-skills/build-functions-005-dual-auth-user-secret/run-2/result.json" }, { "experiment": "opencode-kimi-k3-no-skills", @@ -63492,27 +53849,27 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"error\":\"missing credentials\"}" + "notes": "status 401: {\"error\":\"missing or malformed Authorization: Bearer \"}" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"3dac710b-1cf0-4e0b-8f09-cc7fe2a0da04\",\"metric\":\"steps_a_mta4le54\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"6d72e91f-c31a-4b29-a094-7553431b7fdf\",\"metric\":\"steps_a_mtsg04fw\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"3dac710b-1cf0-4e0b-8f09-cc7fe2a0da04\",\"metric\":\"steps_a_mta4le54\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"6d72e91f-c31a-4b29-a094-7553431b7fdf\",\"metric\":\"steps_a_mtsg04fw\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"5992a7cd-1953-4efc-ba12-c7a332521689\",\"metric\":\"steps_b_mta4le54\",\"value\":222}]" + "notes": "status 200: [{\"user_id\":\"b4c9a327-022b-4127-bbc1-f2fca67cc03f\",\"metric\":\"steps_b_mtsg04fw\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"error\":\"missing credentials\"}" + "notes": "status 401: {\"error\":\"missing or malformed Authorization: Bearer \"}" }, { "name": "rejects an unverified (forged) user token", @@ -63522,7 +53879,7 @@ { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"error\":\"missing credentials\"}" + "notes": "status 401: {\"error\":\"missing or malformed Authorization: Bearer \"}" }, { "name": "implementation uses @supabase/server", @@ -63539,230 +53896,6 @@ }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "run": 2, - "sourcePath": "opencode-kimi-k3-no-skills/build-functions-005-dual-auth-user-secret/run-2/result.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "sdk", - "rls", - "security" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"792893eb-e24c-4380-ab46-a61160c0a41e\",\"metric\":\"steps_a_mta4e9ip\",\"value\":111}]" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: [{\"user_id\":\"792893eb-e24c-4380-ab46-a61160c0a41e\",\"metric\":\"steps_a_mta4e9ip\",\"value\":111}]" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"38663d6c-cdcc-4b7e-8eb7-01991a59505c\",\"metric\":\"steps_b_mta4e9ip\",\"value\":222}]" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge functions environment variables secret key publishable key SUPABASE_SECRET_KEY\") { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/functions/storage-caching", - "title": "Integrating with Supabase Storage" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" - }, - { - "url": "https://supabase.com/docs/guides/ai/examples/nextjs-vector-search", - "title": "Vector search with Next.js and OpenAI" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/integrations/vercel-marketplace", - "title": "Vercel Marketplace" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", - "title": "Transcription Telegram Bot" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/architecture", - "title": "Edge Functions Architecture" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/unit-test", - "title": "Testing your Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" - }, - { - "url": "https://supabase.com/docs/guides/functions/ephemeral-storage", - "title": "File Storage" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/auth-send-email-hook-react-email-resend", - "title": "Custom Auth Emails with React Email and Resend" - }, - { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" - } - ], - "resultChars": 405734 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function verify apikey secret key sb_secret authorization server\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - } - ], - "resultChars": 53424 - } - ] - }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", "run": 3, "sourcePath": "opencode-kimi-k3-no-skills/build-functions-005-dual-auth-user-secret/run-3/result.json" }, @@ -64088,7 +54221,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 01a03e35-f4f6-73c9-9480-bc04bef773c3/receipt-alpha.pdf, 01a03e35-f4f6-73c9-9480-bc04bef773c3/receipt-beta.pdf" + "notes": "saw: 01a0803c-c505-770c-a2c7-dac5ddce3996/receipt-alpha.pdf, 01a0803c-c505-770c-a2c7-dac5ddce3996/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -64109,7 +54242,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT policies, no RLS disablement, and short-lived createSignedUrl usage are all present." + "judgeNotes": "Creates a private user-files bucket, authenticated owner-scoped SELECT/INSERT policies, retains RLS, and uses createSignedUrl with a 10-minute expiry." } ], "skills": { @@ -64117,7 +54250,36 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"storage RLS policy user own folder auth.uid foldername signed URL\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" + }, + { + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" + }, + { + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/storage/debugging/error-codes", + "title": "Error Codes" + } + ], + "resultChars": 30145 + } + ] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", @@ -64161,7 +54323,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 01a03e35-6598-772f-95b3-969f93ca7819/receipt-alpha.pdf, 01a03e35-6598-772f-95b3-969f93ca7819/receipt-beta.pdf" + "notes": "saw: 01a0803c-f2a0-709a-b106-753ae58c0997/receipt-alpha.pdf, 01a0803c-f2a0-709a-b106-753ae58c0997/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -64182,7 +54344,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Creates a private user-files bucket, authenticated owner-scoped SELECT and INSERT policies (plus UPDATE/DELETE), retains RLS, and uses createSignedUrl with an expiry." + "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT policies, RLS retained, and expiring createSignedUrl sharing code are all provided." } ], "skills": { @@ -64193,7 +54355,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"storage access control RLS policies private bucket folder per user\", limit: 3) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"storage access control RLS policy foldername auth.uid own files private bucket\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -64204,12 +54366,20 @@ "url": "https://supabase.com/docs/guides/storage/security/access-control", "title": "Storage Access Control" }, + { + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" + }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", "title": "Configure S3 Storage" + }, + { + "url": "https://supabase.com/docs/guides/storage/schema/design", + "title": "The Storage Schema" } ], - "resultChars": 16943 + "resultChars": 20574 } ] }, @@ -64255,7 +54425,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 01a03e38-8c1f-756f-91d3-248060ad989f/receipt-alpha.pdf, 01a03e38-8c1f-756f-91d3-248060ad989f/receipt-beta.pdf" + "notes": "saw: 01a0803c-c66c-73fb-8d5e-7dedeab02119/receipt-alpha.pdf, 01a0803c-c66c-73fb-8d5e-7dedeab02119/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -64276,7 +54446,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT policies with RLS enforced, and short-lived createSignedUrl sharing code are all provided." + "judgeNotes": "Private bucket, authenticated owner-scoped SELECT/INSERT policies, RLS retained, and short-lived createSignedUrl sharing are all provided." } ], "skills": { @@ -64287,23 +54457,31 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"storage createSignedUrl supabase-js expiresIn\", limit: 3) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"storage RLS policy user own folder auth.uid createSignedUrl expires\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", - "title": "Storage Image Transformations" + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" }, { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", - "title": "Manage Storage Image Transformations usage" + "url": "https://supabase.com/docs/guides/storage/debugging/error-codes", + "title": "Error Codes" } ], - "resultChars": 28966 + "resultChars": 30145 } ] }, @@ -64341,12 +54519,12 @@ { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "8 passed, 0 failed" + "notes": "8 passed, 3 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies `posts` as leaking cross-tenant data, grounds this in the failing pgTAP tests, confirms `notes` isolation held, and reports all tests passing after the policy fix." + "judgeNotes": "Correctly identifies the `posts` SELECT policy as the tenant-isolation flaw and grounds it in the failing pgTAP cross-tenant read test. It also correctly states that `notes` read isolation passes." } ], "skills": { @@ -64385,17 +54563,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation.sql" + "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "6 passed, 1 failed" + "notes": "8 passed, 0 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies the `posts` SELECT policy as the cross-tenant isolation flaw and grounds the conclusion in failing pgTAP test 5, while correctly recognizing `notes` as isolated." + "judgeNotes": "Correctly identifies `posts` as the broken tenant-isolation policy, cites the 3/8 pgTAP failures showing cross-org reads, and notes that `notes` tests pass." } ], "skills": { @@ -64429,22 +54607,22 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" + "notes": "1 file(s): supabase/tests/rls_tenant_isolation.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "3 passed, 5 failed" + "notes": "6 passed, 3 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", - "passed": false, - "judgeNotes": "The agent never concludes that the `posts` tenant-isolation policy is broken or grounds that finding in pgTAP results; it only sets up and troubleshoots tests." + "passed": true, + "judgeNotes": "Correctly identifies `posts` as leaking cross-tenant data due to the missing org correlation and grounds this conclusion in pgTAP failures. It also correctly reports `notes` as isolated." } ], "skills": { @@ -64518,28 +54696,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"gte-small embedding dimensions vector 384 semantic search match_document_sections\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", - "title": "Choosing your Compute Add-on" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/ai/concepts", - "title": "Concepts" - } - ], - "resultChars": 37288 - } - ] + "calls": [] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", @@ -64576,12 +54733,12 @@ { "name": "HNSW index on the embedding column", "passed": true, - "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { "name": "index operator class matches the search operator", "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { "name": "user A search returns only own sections, best match first", @@ -64605,36 +54762,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"semantic search pgvector embeddings row level security match documents edge function\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-columns", - "title": "Vector columns" - } - ], - "resultChars": 70138 - } - ] + "calls": [] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", @@ -64700,7 +54828,15 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"gte-small embedding dimensions vector semantic search match documents function\") { nodes { title content } } }", + "hasContent": true, + "pages": [], + "resultChars": 247721 + } + ] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", @@ -64725,7 +54861,7 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { "name": "preserved existing app scrape job", @@ -64733,13 +54869,13 @@ }, { "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "Meets all requirements: HTTPS Supabase target, correct metrics path, Basic Auth with password_file, matching mounted secret directory, and preserved app scrape." + "passed": false, + "judgeNotes": "Supabase scrape is not deployable as required: it uses HTTP and host.docker.internal instead of HTTPS with a .supabase.co/.red target, hardcodes a placeholder Secret API key, lacks basic_auth.password_file, and docker-compose does not mount the password file. The app scrape is preserved and the metrics path is correct." }, { "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README documents creating a Supabase secret API key, placing it in the matching mounted secret file, starting/reloading Compose/Prometheus, and verifying the scrape via Prometheus targets and a direct authenticated endpoint check." + "passed": false, + "judgeNotes": "README uses a key directly in prometheus.yml; it does not require creating/placing a matching secret file or configure Compose to mount one. This fails the required secret setup despite adequate reload and target verification steps." } ], "skills": { @@ -64750,39 +54886,31 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"metrics endpoint prometheus scrape project metrics\", limit: 10) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"metrics endpoint prometheus scrape privileged metrics authentication\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", - "title": "Scrape a project's metrics" + "url": "https://supabase.com/docs/guides/observability/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-self-hosted", "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-cloud", "title": "Metrics API with Grafana Cloud" }, { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" + "url": "https://supabase.com/docs/guides/observability/metrics", + "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", - "title": "How to View Database Metrics" + "url": "https://supabase.com/docs/guides/observability/access-data", + "title": "Observe the data" } ], - "resultChars": 29660 + "resultChars": 21684 } ] }, @@ -64817,13 +54945,13 @@ }, { "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "HTTPS Supabase metrics scrape is correctly configured with the required path, Basic Auth password_file, project target, preserved app job, and matching secret volume mount." + "passed": false, + "judgeNotes": "Supabase uses a hardcoded basic_auth.password placeholder instead of required password_file, and docker-compose.yml does not mount or provide that password file via a volume or Compose secret." }, { "name": "documented live deployment and verification steps", "passed": false, - "judgeNotes": "README.md does not explain Supabase integration setup: no Secret API key creation, matching secret file placement, Compose restart/reload, or concrete Prometheus/Grafana verification steps." + "judgeNotes": "README creates a Secret API key and provides reload/verification steps, but it does not require or explain placing a matching secret file or mounting it into Compose. Instead, it instructs users to put the key directly in prometheus.yml, so the required secret-file setup is missing." } ], "skills": { @@ -64834,31 +54962,55 @@ "calls": [ { "source": "search_docs", - "query": "{\n searchDocs(query: \"prometheus metrics endpoint observability\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", + "query": "query { searchDocs(query: \"prometheus metrics endpoint scrape project metrics\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", + "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", + "title": "Scrape a project's metrics" + }, + { + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" + }, + { + "url": "https://supabase.com/docs/guides/observability/metrics", "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", + "url": "https://supabase.com/docs/guides/observability/metrics/vendor-agnostic", "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-cloud", "title": "Metrics API with Grafana Cloud" + } + ], + "resultChars": 20237 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"create secret API key sb_secret management api\", limit: 4) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" }, { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" } ], - "resultChars": 28211 + "resultChars": 92231 } ] }, @@ -64885,7 +55037,7 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "preserved existing app scrape job", @@ -64894,12 +55046,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "HTTPS Supabase Metrics API scrape is correctly configured with Basic Auth via password_file, matching Compose secret wiring, and the existing app scrape is preserved." + "judgeNotes": "HTTPS Supabase scrape uses the correct metrics path and Basic Auth password_file; the mounted secrets volume matches, and the existing app job is preserved." }, { "name": "documented live deployment and verification steps", - "passed": false, - "judgeNotes": "README does not explain creating a Secret API key or placing a matching secret file; it instead exports an existing service_role key as an environment-backed Compose secret. Verification and restart/reload steps are present, but required secret setup is missing." + "passed": true, + "judgeNotes": "README covers secret key creation, matching secret file placement, stack apply/reload, and verification through Prometheus targets." } ], "skills": { @@ -64910,51 +55062,87 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"metrics endpoint Prometheus scrape\") { nodes { title href content } } }", + "query": "{ searchDocs(query: \"metrics endpoint prometheus scrape project metrics API\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", + "title": "Scrape a project's metrics" + }, + { + "url": "https://supabase.com/docs/guides/observability/metrics", + "title": "Metrics API" + }, + { + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-self-hosted", "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", + "url": "https://supabase.com/docs/guides/observability/metrics/vendor-agnostic", "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", - "title": "Scrape a project's metrics" + "url": "https://supabase.com/docs/guides/observability/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" + } + ], + "resultChars": 20237 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"create project api key secret sb_secret management api\", limit: 4) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", + "title": "Build a User Management App with Angular" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", - "title": "How to View Database Metrics" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + } + ], + "resultChars": 112040 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"v1 projects api-keys create a new api key for project\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/api/v1-create-project-api-key", + "title": "Creates a new API key for the project" }, { - "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", - "title": "Deleting data and dropping objects safely" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/database/replication/pipelines-monitoring", - "title": "Monitor pipeline status" + "url": "https://supabase.com/docs/reference/api/v1-update-project-api-key", + "title": "Updates an API key for the project" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/grafana-not-displaying-data-sXJrMj", - "title": "Grafana not displaying data" + "url": "https://supabase.com/docs/reference/api/v1-get-project-api-keys", + "title": "Get project api keys" }, { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" + "url": "https://supabase.com/docs/reference/api/v1-get-project-api-key", + "title": "Get API key" + }, + { + "url": "https://supabase.com/docs/reference/api/v1-delete-project-api-key", + "title": "Deletes an API key for the project" } ], - "resultChars": 59315 + "resultChars": 15622 } ] }, @@ -65046,7 +55234,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -65097,7 +55285,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment via Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -65109,36 +55297,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"deploy edge function CLI secrets set env file\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", - "title": "Transcription Telegram Bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - } - ], - "resultChars": 58240 - } - ] + "calls": [] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", @@ -65189,7 +55348,47 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"self-hosting with docker compose setup guide\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", + "title": "Configure social login (OAuth) providers" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-hooks", + "title": "Configure Auth Hooks" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + } + ], + "resultChars": 107291 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/self-hosting/docker", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/docker" + } + ], + "resultChars": 29564 + } + ] }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", @@ -65291,7 +55490,36 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"self-hosting with docker generate API keys JWT secret\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", + "title": "Configure social login (OAuth) providers" + } + ], + "resultChars": 103252 + } + ] }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", @@ -65347,7 +55575,204 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "Correctly diagnoses the ineffective soft delete, implements auth-user deletion with cascading session/refresh-token removal, accurately explains the remaining stateless JWT window and mitigation, and correctly distinguishes frontend publishable keys from server-only RLS-bypassing secret keys." + "judgeNotes": "Correctly diagnoses the soft-delete flaw, implements auth-user deletion with session/refresh-token revocation, accurately explains the remaining stateless JWT expiry window and mitigations, and correctly distinguishes frontend publishable keys from server-only RLS-bypassing secret keys." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable key secret key vs legacy anon service_role API keys RLS\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/quickstarts/refine", + "title": "Use Supabase with Refine" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" + } + ], + "resultChars": 65078 + } + ] + }, + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "run": 1, + "sourcePath": "opencode-kimi-k3-no-skills/investigate-auth-001-deleted-user-access/run-1/result.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", + "product": [ + "auth" + ], + "topic": [ + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "victim session active before delete-account", + "passed": true + }, + { + "name": "delete_account flow ran for the victim", + "passed": true + }, + { + "name": "delete-account revokes the user's sessions", + "passed": true, + "notes": "sessions left: 0" + }, + { + "name": "deleted user's refresh token is rejected", + "passed": true + }, + { + "name": "deleted user cannot sign back in", + "passed": true + }, + { + "name": "other users keep their sessions and access", + "passed": true + }, + { + "name": "diagnosed and explained session revocation", + "passed": true, + "judgeNotes": "Correctly diagnoses the soft-delete flaw, implements real auth-user/session/refresh-token removal, accurately explains the stale JWT window and mitigations, and distinguishes frontend publishable keys from server-only RLS-bypassing secret keys." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable secret API keys vs legacy anon service_role RLS\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/quickstarts/refine", + "title": "Use Supabase with Refine" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" + } + ], + "resultChars": 66468 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/api/api-keys", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/api-keys" + } + ], + "resultChars": 22207 + } + ] + }, + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "run": 2, + "sourcePath": "opencode-kimi-k3-no-skills/investigate-auth-001-deleted-user-access/run-2/result.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", + "product": [ + "auth" + ], + "topic": [ + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "victim session active before delete-account", + "passed": true + }, + { + "name": "delete_account flow ran for the victim", + "passed": true + }, + { + "name": "delete-account revokes the user's sessions", + "passed": true, + "notes": "sessions left: 0" + }, + { + "name": "deleted user's refresh token is rejected", + "passed": true + }, + { + "name": "deleted user cannot sign back in", + "passed": true + }, + { + "name": "other users keep their sessions and access", + "passed": true + }, + { + "name": "diagnosed and explained session revocation", + "passed": true, + "judgeNotes": "Correctly identifies the soft-delete/session problem, implements auth-user deletion with cascading session and refresh-token revocation, closes the demonstrated Data API path via RLS existence checks, explains stateless JWT validity for local verification, and accurately distinguishes frontend publishable keys from server-only secret keys that bypass RLS." } ], "skills": { @@ -65358,193 +55783,36 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"publishable secret API keys vs anon service_role\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"publishable key secret key vs anon service_role JWT RLS\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, { "url": "https://supabase.com/docs/guides/auth/signing-keys", "title": "JWT Signing Keys" }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/quickstarts/refine", - "title": "Use Supabase with Refine" - } - ], - "resultChars": 91848 - } - ] - }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "run": 1, - "sourcePath": "opencode-kimi-k3-no-skills/investigate-auth-001-deleted-user-access/run-1/result.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", - "product": [ - "auth" - ], - "topic": [ - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true - }, - { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" - }, - { - "name": "deleted user's refresh token is rejected", - "passed": true - }, - { - "name": "deleted user cannot sign back in", - "passed": true - }, - { - "name": "other users keep their sessions and access", - "passed": true - }, - { - "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "Correctly diagnoses the soft-delete flaw, implements auth-user/session/refresh-token removal, accurately explains the residual stateless JWT window with mitigations, and correctly distinguishes publishable versus server-only secret keys and RLS behavior." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable secret API keys anon service_role migration RLS\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ { "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "API keys" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", - "title": "Build a User Management App with SvelteKit" + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" } ], - "resultChars": 169611 + "resultChars": 84448 } ] }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "run": 2, - "sourcePath": "opencode-kimi-k3-no-skills/investigate-auth-001-deleted-user-access/run-2/result.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", - "product": [ - "auth" - ], - "topic": [ - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true - }, - { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" - }, - { - "name": "deleted user's refresh token is rejected", - "passed": true - }, - { - "name": "deleted user cannot sign back in", - "passed": true - }, - { - "name": "other users keep their sessions and access", - "passed": true - }, - { - "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "Correctly diagnoses the soft-delete flaw, implements auth-user deletion with session/refresh-token revocation and cascading cleanup, accurately explains the residual stateless JWT expiry window and mitigation, and correctly distinguishes frontend publishable keys from server-only RLS-bypassing secret keys." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", "run": 3, "sourcePath": "opencode-kimi-k3-no-skills/investigate-auth-001-deleted-user-access/run-3/result.json" }, @@ -65593,7 +55861,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified orders missing from supabase_realtime, added only public.orders to the existing publication, and preserved RLS, policies, and courier_locations." + "judgeNotes": "Correctly diagnosed orders missing from supabase_realtime, added only public.orders to the existing publication, and preserved RLS, policies, and courier_locations." } ], "skills": { @@ -65653,7 +55921,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified the missing orders table in supabase_realtime and added only public.orders, preserving courier_locations, RLS, and policies." + "judgeNotes": "Correctly identified orders as missing from supabase_realtime and added only public.orders to the existing publication, preserving courier_locations, RLS, and policies." } ], "skills": { @@ -65713,7 +55981,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Correctly identified the missing orders table in supabase_realtime, added it to the existing publication, preserved courier_locations and RLS/policies, and verified the fix." + "judgeNotes": "Correctly identified the missing orders table in supabase_realtime, added only public.orders, and preserved courier_locations, RLS, and existing policies." } ], "skills": { @@ -65746,22 +56014,22 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified `image-transform` and accurately described all 8 recurring HTTP 503 failures from 07:00Z through 12:00Z on 2026-04-28." + "judgeNotes": "Identified `image-transform` as the affected function and documented all 8 recurring HTTP 503 failures from 07:00Z through 12:00Z on 2026-04-28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": true, - "judgeNotes": "Correctly attributes the recurring image-transform 503s to the gateway/platform layer, supported by their presence only in gateway logs without matching invocation/runtime records, while nearby requests on the same deployment succeeded. It also distinguishes the separate function-level avatar-upload 500." + "passed": false, + "judgeNotes": "Although it correctly identifies gateway-only 503s with no matching function invocations, it recommends redeploying and changing the function/dependencies as remediation, which the rubric explicitly marks as a failure." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant gave concrete next steps, including opening a Supabase support incident with the project, region, function, UTC timestamps, request IDs, and missing execution-log correlation." + "judgeNotes": "Provides multiple concrete actions, including redeploying the Edge Function, pinning dependencies, checking memory/resource limits, adding retries and instrumentation, and configuring alerts." } ], "skills": { @@ -65769,36 +56037,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{\n searchDocs(query: \"Edge Functions logs 503 edge_logs function_edge_logs troubleshooting\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/debugging", - "title": "Debugging guide" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", - "title": "Edge Function 503 error response" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", - "title": "Edge Function 504 error response" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-500-error-response", - "title": "Edge Function 500 error response" - } - ], - "resultChars": 64405 - } - ] + "calls": [] }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", @@ -65828,17 +56067,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform and explicitly documented all 8 recurring HTTP 503 gateway failures from 07:00Z through 12:00Z on 2026-04-28." + "judgeNotes": "Identified image-transform and the recurring eight HTTP 503 gateway failures across 07:00Z–12:00Z on 2026-04-28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": false, - "judgeNotes": "Although it correctly identifies gateway-layer 503s using missing invocation logs and nearby successes, it then blames a function dependency/cold start and recommends modifying and redeploying the function, which the rubric explicitly disallows." + "judgeNotes": "Although it notes the 503s occur at the gateway with no function executions, it ultimately blames the function’s dependency/cold-start path and recommends modifying and redeploying the function. The required conclusion is a gateway/Edge Functions platform-layer failure, not function code or runtime." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Recommended concrete actions: pin/vendor the dependency, redeploy the function, monitor path-specific 503s, add retries and alerts, and improve error logging." + "judgeNotes": "Provides multiple concrete actions, including pinning/vendoring and redeploying the dependency, adding boot logging, and escalating to Supabase support with specific gateway request IDs." } ], "skills": { @@ -65871,22 +56110,22 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified `image-transform` and documented all 8 recurring HTTP 503 gateway failures from 07:00Z through 12:00Z on 2026-04-28." + "judgeNotes": "The assistant correctly identified image-transform and explicitly described all eight recurring HTTP 503 failures from 07:00Z through 12:00Z on 2026-04-28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": true, - "judgeNotes": "Correctly attributes the 503s to the gateway/platform layer, supported by their presence only in edge_logs with no matching function invocations, while the unchanged deployment had nearby successful executions. It also distinguishes avatar-upload’s function-level 500." + "passed": false, + "judgeNotes": "Although it notes no matching execution logs, it ultimately attributes the 503s to unpinned function dependencies/runtime boot failures and recommends modifying and redeploying the functions, contrary to the required gateway/platform-layer attribution." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant gave specific actionable steps, including opening a Supabase support ticket with the project, region, function, timestamps, request IDs, and log-correlation evidence." + "judgeNotes": "Recommended concrete actions, including pinning dependency versions, redeploying, checking registry publishes, and adding targeted alerting." } ], "skills": { @@ -65894,36 +56133,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions logs 503 edge_logs function_edge_logs troubleshooting\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/debugging", - "title": "Debugging guide" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", - "title": "Edge Function 503 error response" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", - "title": "Edge Function 504 error response" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-500-error-response", - "title": "Edge Function 500 error response" - } - ], - "resultChars": 64405 - } - ] + "calls": [] }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", @@ -65980,7 +56190,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly diagnosed deny-all RLS with no policies and added authenticated-only SELECT and INSERT policies scoped by user_id = auth.uid(), while keeping RLS enabled." + "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and added authenticated-only SELECT and INSERT policies scoped by user_id = auth.uid() using USING and WITH CHECK." } ], "skills": { @@ -66016,36 +56226,12 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true - }, - { - "name": "user B cannot insert a bookmark as user A", - "passed": true - }, - { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "Correctly identifies deny-by-default RLS with no policies, keeps RLS enabled, and creates authenticated-only SELECT and INSERT policies scoped by user_id = auth.uid() using USING and WITH CHECK." + "name": "created auth sessions", + "passed": false, + "notes": "Internal server error" } ], "skills": { @@ -66110,7 +56296,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly diagnosed deny-all RLS with no policies, kept RLS enabled, and added authenticated-only SELECT and INSERT policies scoped by user_id = auth.uid() using USING and WITH CHECK." + "judgeNotes": "Correctly diagnosed RLS default-deny, kept RLS enabled, and added authenticated-only SELECT and INSERT policies scoped by user_id = auth.uid() using USING and WITH CHECK." } ], "skills": { @@ -66168,7 +56354,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "Added local 20240115000000_add_profile_bio.sql to reconcile the orphan history, then `supabase db push` applied 20240220000000_add_avatar_url.sql and finished successfully. Only read-only psql inspection was used; no prohibited workaround occurred." + "judgeNotes": "`supabase db push` in #20 applied `20240220000000_add_avatar_url.sql` and finished successfully. History was reconciled by adding local `20240115000000_add_profile_bio.sql` in #19, then letting that push align local/remote history; #22 confirms all versions match. The `psql` commands were read-only inspections, with no prohibited workaround." } ], "skills": { @@ -66226,7 +56412,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "`supabase db push` (#20) applied `20240220000000_add_avatar_url.sql`, showing both “Applying migration” and “Finished supabase db push.” The orphan bio history was reconciled by adding `20240115000000_add_profile_bio.sql` locally (#18), then pushing via the CLI. No prohibited workaround was used; the `psql` commands were read-only inspections." + "judgeNotes": "`supabase db push` (#15) applied `20240220000000_add_avatar_url.sql` and finished successfully. History was reconciled by adding the missing local `20240115000000_add_profile_bio.sql` (#13), after which the CLI matched it to the remote record and allowed the push. The `psql` commands were read-only inspections; no prohibited workaround was used." } ], "skills": { @@ -66284,7 +56470,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "`supabase db push` (#11) applied `20240220000000_add_avatar_url.sql` and finished successfully. History was reconciled by adding local `20240115000000_add_profile_bio.sql` (#9), then letting the CLI push workflow confirm alignment. No prohibited mutation workaround was used; `psql` was read-only." + "judgeNotes": "`supabase db push` applied `20240220000000_add_avatar_url.sql` and finished successfully. History was reconciled by adding local `20240115000000_add_profile_bio.sql`, then letting the same push align migrations. No prohibited workaround was used; `psql` was read-only inspection." } ], "skills": { @@ -66335,7 +56521,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_id_created_at (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_id_created_at (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -66390,7 +56576,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_created_at (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -66445,7 +56631,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_created_at (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_id_created_at (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", diff --git a/experiments/claude-haiku-4.5.ts b/experiments/claude-haiku-4.5.ts deleted file mode 100644 index 5b4ceae9..00000000 --- a/experiments/claude-haiku-4.5.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { anthropic } from '@ai-sdk/anthropic'; -import { - aiSdkAgent, - defineExperiment, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - agent: aiSdkAgent({ - model: anthropic('claude-haiku-4-5'), - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: ['supabase', 'supabase-postgres-best-practices'], -}); diff --git a/experiments/claude-opus-4.7.ts b/experiments/claude-opus-4.7.ts deleted file mode 100644 index 0efb5ed3..00000000 --- a/experiments/claude-opus-4.7.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { anthropic } from '@ai-sdk/anthropic'; -import { - aiSdkAgent, - defineExperiment, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - agent: aiSdkAgent({ - model: anthropic('claude-opus-4-7'), - providerOptions: { - anthropic: { effort: 'max' }, - }, - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: ['supabase', 'supabase-postgres-best-practices'], -}); diff --git a/experiments/claude-sonnet-5-executor.ts b/experiments/claude-sonnet-5-executor.ts deleted file mode 100644 index 29aeaea9..00000000 --- a/experiments/claude-sonnet-5-executor.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { anthropic } from '@ai-sdk/anthropic'; -import { - aiSdkAgent, - defineExperiment, - executorMcpServer, - platformLiteRuntime, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - agent: aiSdkAgent({ - model: anthropic('claude-sonnet-5'), - providerOptions: { - anthropic: { effort: 'max' }, - }, - }), - runtime: platformLiteRuntime({ - mcpServers: [executorMcpServer()], - }), - localStack: localStackRuntime(), - skills: ['supabase', 'supabase-postgres-best-practices'], -}); diff --git a/experiments/claude-sonnet-5.ts b/experiments/claude-sonnet-5.ts deleted file mode 100644 index 7ddfc926..00000000 --- a/experiments/claude-sonnet-5.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { anthropic } from '@ai-sdk/anthropic'; -import { - aiSdkAgent, - defineExperiment, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - agent: aiSdkAgent({ - model: anthropic('claude-sonnet-5'), - providerOptions: { - anthropic: { effort: 'max' }, - }, - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: ['supabase', 'supabase-postgres-best-practices'], -}); diff --git a/experiments/codex-gpt-5.4-mini-no-skills.ts b/experiments/codex-gpt-5.6-luna-no-skills.ts similarity index 94% rename from experiments/codex-gpt-5.4-mini-no-skills.ts rename to experiments/codex-gpt-5.6-luna-no-skills.ts index 07b82d67..40ccfbaf 100644 --- a/experiments/codex-gpt-5.4-mini-no-skills.ts +++ b/experiments/codex-gpt-5.6-luna-no-skills.ts @@ -9,7 +9,7 @@ import { localStackRuntime } from '@supabase-evals/sandbox'; export default defineExperiment({ suite: ['no-skills'], agent: codexAgent({ - model: 'gpt-5.4-mini', + model: 'gpt-5.6-luna', reasoningEffort: 'medium', }), runtime: platformLiteRuntime({ diff --git a/experiments/codex-gpt-5.4-mini.ts b/experiments/codex-gpt-5.6-luna.ts similarity index 94% rename from experiments/codex-gpt-5.4-mini.ts rename to experiments/codex-gpt-5.6-luna.ts index 8b22bbb0..141b1459 100644 --- a/experiments/codex-gpt-5.4-mini.ts +++ b/experiments/codex-gpt-5.6-luna.ts @@ -9,7 +9,7 @@ import { localStackRuntime } from '@supabase-evals/sandbox'; export default defineExperiment({ suite: ['benchmark'], agent: codexAgent({ - model: 'gpt-5.4-mini', + model: 'gpt-5.6-luna', reasoningEffort: 'medium', }), runtime: platformLiteRuntime({ diff --git a/experiments/codex-gpt-5.6-no-skills.ts b/experiments/codex-gpt-5.6-sol-no-skills.ts similarity index 100% rename from experiments/codex-gpt-5.6-no-skills.ts rename to experiments/codex-gpt-5.6-sol-no-skills.ts diff --git a/experiments/codex-gpt-5.6.ts b/experiments/codex-gpt-5.6-sol.ts similarity index 100% rename from experiments/codex-gpt-5.6.ts rename to experiments/codex-gpt-5.6-sol.ts diff --git a/experiments/openai-gpt-5.4-mini-executor.ts b/experiments/openai-gpt-5.4-mini-executor.ts deleted file mode 100644 index 8f2549b6..00000000 --- a/experiments/openai-gpt-5.4-mini-executor.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { openai } from '@ai-sdk/openai'; -import { - aiSdkAgent, - defineExperiment, - executorMcpServer, - platformLiteRuntime, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - agent: aiSdkAgent({ - model: openai('gpt-5.4-mini'), - providerOptions: { - openai: { - reasoningEffort: 'low', - textVerbosity: 'low', - }, - }, - }), - runtime: platformLiteRuntime({ - mcpServers: [executorMcpServer()], - }), - localStack: localStackRuntime(), - skills: ['supabase', 'supabase-postgres-best-practices'], -}); diff --git a/experiments/openai-gpt-5.4-mini.ts b/experiments/openai-gpt-5.4-mini.ts deleted file mode 100644 index e066c034..00000000 --- a/experiments/openai-gpt-5.4-mini.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { openai } from '@ai-sdk/openai'; -import { - aiSdkAgent, - defineExperiment, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - agent: aiSdkAgent({ - model: openai('gpt-5.4-mini'), - providerOptions: { - openai: { - reasoningEffort: 'low', - textVerbosity: 'low', - }, - }, - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: ['supabase', 'supabase-postgres-best-practices'], -}); diff --git a/experiments/openai-gpt-5.4-nano.ts b/experiments/openai-gpt-5.4-nano.ts deleted file mode 100644 index 0aef5122..00000000 --- a/experiments/openai-gpt-5.4-nano.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { openai } from '@ai-sdk/openai'; -import { - aiSdkAgent, - defineExperiment, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - agent: aiSdkAgent({ - model: openai('gpt-5.4-nano'), - providerOptions: { - openai: { - reasoningEffort: 'low', - textVerbosity: 'low', - }, - }, - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: ['supabase', 'supabase-postgres-best-practices'], -}); diff --git a/experiments/openai-gpt-5.6.ts b/experiments/openai-gpt-5.6.ts deleted file mode 100644 index 313cabec..00000000 --- a/experiments/openai-gpt-5.6.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { openai } from '@ai-sdk/openai'; -import { - aiSdkAgent, - defineExperiment, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - agent: aiSdkAgent({ - model: openai('gpt-5.6'), - providerOptions: { - openai: { - reasoningEffort: 'medium', - textVerbosity: 'low', - }, - }, - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: ['supabase', 'supabase-postgres-best-practices'], -});