From 35c0710d678291bfdc55f7cd16536ad52adb338b Mon Sep 17 00:00:00 2001 From: Miranda Limonczenko Date: Fri, 4 Sep 2026 16:42:45 -0700 Subject: [PATCH 1/3] test(evals): point build-docs-006 at the tables access-control preview Scratch, not for merge. Three changes, all reverted rather than landed. PROMPT.md's REFERENCE points at the preview for supabase/supabase#50024. EVAL.ts matches the preview host rather than the path alone, so fetching the published page cannot satisfy the guide-read check. docs-results.ts accepts docs-git-*-supabase.vercel.app, without which the preview fetch never reaches docsCalls and the check fails a run that did read the page. --- evals/build-docs-006-database-tables/EVAL.ts | 3 ++- evals/build-docs-006-database-tables/PROMPT.md | 2 +- packages/core/src/docs-results.ts | 6 +++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/evals/build-docs-006-database-tables/EVAL.ts b/evals/build-docs-006-database-tables/EVAL.ts index b7a3fd13..0c28ed8e 100644 --- a/evals/build-docs-006-database-tables/EVAL.ts +++ b/evals/build-docs-006-database-tables/EVAL.ts @@ -21,7 +21,8 @@ import { type Fixtures, } from './access.js'; -const GUIDE_PATH = 'guides/database/tables'; +const GUIDE_PATH = + 'docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables'; const scorer: LocalStackScorer = async (ctx) => { try { diff --git a/evals/build-docs-006-database-tables/PROMPT.md b/evals/build-docs-006-database-tables/PROMPT.md index 9e33d5b2..40f9b4fa 100644 --- a/evals/build-docs-006-database-tables/PROMPT.md +++ b/evals/build-docs-006-database-tables/PROMPT.md @@ -35,4 +35,4 @@ Read the guide below before you start and rely on it for how to set this up, rather than on what you already know. REFERENCE -https://supabase.com/docs/guides/database/tables.md +https://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md diff --git a/packages/core/src/docs-results.ts b/packages/core/src/docs-results.ts index c8e9af39..f2610b7f 100644 --- a/packages/core/src/docs-results.ts +++ b/packages/core/src/docs-results.ts @@ -71,7 +71,11 @@ function extractGraphqlQuery( */ function isSupabaseApexUrl(value: string): boolean { try { - return new URL(value).hostname === 'supabase.com'; + const { hostname } = new URL(value); + return ( + hostname === 'supabase.com' || + /^docs-git-[a-z0-9-]+-supabase\.vercel\.app$/.test(hostname) + ); } catch { return false; } From 8c1bd012e00de5a1f2c78ad3a3cd164cc15b2327 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 23:53:42 +0000 Subject: [PATCH 2/3] chore: refresh eval results --- .../web/src/data/regression-eval-results.json | 74 ++++++++++--------- 1 file changed, 40 insertions(+), 34 deletions(-) diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index ffd37cf2..e4157b8c 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -1236,7 +1236,7 @@ { "name": "the app's tables exist and accept the rows its queries write", "passed": false, - "notes": "Stopped services: [supabase_inbucket_sandbox-database-tables supabase_realtime_sandbox-database-tables supabase_storage_sandbox-database-tables supabase_imgproxy_sandbox-database-tables supabase_pg_meta_sandbox-database-tables supabase_studio_sandbox-database-tables supabase_edge_runtime_sandbox-database-tables supabase_analytics_sandbox-database-tables supabase_vector_sandbox-database-tables supabase_pooler_sandbox-database-tables]\nA new version of Supabase CLI is available: v2.116.0 (currently installed v2.67.1)\nWe recommend updating regularly for new features and bug fixes: https://supabase.com/docs/guides/cli/getting-started#updating-the-supabase-cli\nERROR: relation \"routine_library\" does not exist\nLINE 1: INSERT INTO routine_library (title, category)\n ^\n" + "notes": "ERROR: relation \"routine_library\" does not exist; LINE 1: INSERT INTO routine_library (title, category); ^" }, { "name": "row level security is enabled on every table in the public schema", @@ -1287,18 +1287,18 @@ "calls": [ { "source": "web_fetch", - "query": "Extract all guidance about creating tables in Supabase: syntax, primary keys, data types, columns, schemas, relationships (foreign keys), RLS enabling, GUI vs SQL, naming conventions, and best practices mentioned in this specific page.", + "query": "Extract the full guide content verbatim/in detail: all sections about creating tables, columns, data types, primary keys, foreign keys, RLS (row level security) setup, policies, best practices, and any code examples. I need the complete guidance for designing a schema with RLS.", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/tables.md" + "url": "https://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md" } ], - "resultChars": 1735 + "resultChars": 6809 } ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 1, "sourcePath": "claude-code-sonnet-5/build-docs-006-database-tables/run-1/result.json" @@ -1330,12 +1330,12 @@ { "name": "the app's tables exist and accept the rows its queries write", "passed": false, - "notes": "Stopped services: [supabase_inbucket_sandbox-database-tables supabase_realtime_sandbox-database-tables supabase_storage_sandbox-database-tables supabase_imgproxy_sandbox-database-tables supabase_pg_meta_sandbox-database-tables supabase_studio_sandbox-database-tables supabase_edge_runtime_sandbox-database-tables supabase_analytics_sandbox-database-tables supabase_vector_sandbox-database-tables supabase_pooler_sandbox-database-tables]\nA new version of Supabase CLI is available: v2.116.0 (currently installed v2.67.1)\nWe recommend updating regularly for new features and bug fixes: https://supabase.com/docs/guides/cli/getting-started#updating-the-supabase-cli\nERROR: relation \"routine_library\" does not exist\nLINE 1: INSERT INTO routine_library (title, category)\n ^\n" + "notes": "ERROR: relation \"routine_library\" does not exist; LINE 1: INSERT INTO routine_library (title, category); ^" }, { "name": "row level security is enabled on every table in the public schema", "passed": false, - "notes": "the app's tables are missing: routine_library" + "notes": "the app's tables are missing: routine_logs, routine_library" }, { "name": "every table with row level security enabled carries at least one policy", @@ -1381,18 +1381,18 @@ "calls": [ { "source": "web_fetch", - "query": "Extract all guidance about how to create tables in Supabase: syntax, column types, primary keys, schemas (public vs custom), foreign key relationships, enabling RLS, naming conventions, and any complete example table creation code shown on the page. Include everything verbatim where possible.", + "query": "Extract the full content of this guide verbatim, including all guidance on table design, schema conventions, RLS policies, and any code examples.", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/tables.md" + "url": "https://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md" } ], - "resultChars": 2365 + "resultChars": 2968 } ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 2, "sourcePath": "claude-code-sonnet-5/build-docs-006-database-tables/run-2/result.json" @@ -1419,15 +1419,17 @@ ], "suite": "regression", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { "name": "the app's tables exist and accept the rows its queries write", - "passed": true + "passed": false, + "notes": "ERROR: relation \"routine_library\" does not exist; LINE 1: INSERT INTO routine_library (title, category); ^" }, { "name": "row level security is enabled on every table in the public schema", - "passed": true + "passed": false, + "notes": "the app's tables are missing: routine_logs, routine_library" }, { "name": "every table with row level security enabled carries at least one policy", @@ -1435,19 +1437,23 @@ }, { "name": "a signed-out visitor can browse the starter routine library", - "passed": true + "passed": false, + "notes": "not run: the app's tables did not accept the rows its queries write" }, { "name": "a signed-out visitor cannot read anyone's routines", - "passed": true + "passed": false, + "notes": "not run: the app's tables did not accept the rows its queries write" }, { "name": "the signed-in owner reads their own routines and nobody else's", - "passed": true + "passed": false, + "notes": "not run: the app's tables did not accept the rows its queries write" }, { "name": "a signed-out visitor cannot create a routine", - "passed": true + "passed": false, + "notes": "not run: the app's tables did not accept the rows its queries write" }, { "name": "the agent read the Tables and Data guide the prompt referenced", @@ -1469,18 +1475,18 @@ "calls": [ { "source": "web_fetch", - "query": "Summarize the guidance on creating tables in Supabase: naming conventions, column types, primary keys, foreign keys, schemas, RLS considerations mentioned on this page. Include any code examples for creating tables with the Supabase dashboard/SQL editor, especially around identity columns, timestamps, relationships.", + "query": "Return the full raw content of this markdown document verbatim, including all headings, code blocks, and text.", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/tables.md" + "url": "https://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md" } ], - "resultChars": 1570 + "resultChars": 27792 } ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 3, "sourcePath": "claude-code-sonnet-5/build-docs-006-database-tables/run-3/result.json" @@ -5327,18 +5333,18 @@ "calls": [ { "source": "web_fetch", - "query": "Extract the full guide content about creating tables in Supabase: syntax, best practices, primary keys, foreign keys/relationships, RLS enabling, schemas, data types, naming conventions, and any recommended workflow (e.g., using migrations).", + "query": "Extract the full content of this guide verbatim/in detail: all guidance about creating tables, columns, data types, primary keys, foreign keys, relationships, naming conventions, schemas, and especially anything about Row Level Security (RLS) policies. Include code examples exactly as written.", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/tables.md" + "url": "https://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md" } ], - "resultChars": 4585 + "resultChars": 6349 } ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 1, "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-006-database-tables/run-1/result.json" @@ -5409,18 +5415,18 @@ "calls": [ { "source": "web_fetch", - "query": "Summarize all guidance on creating tables: syntax, primary keys, foreign keys/relationships, data types, nullability, schemas, naming conventions, RLS notes, and any best practices mentioned.", + "query": "Extract the full guide content: all instructions, best practices, SQL conventions, naming rules, RLS policy guidance, examples of table creation, primary keys, foreign keys, schema design recommendations, and anything about Row Level Security setup. Return as much verbatim detail as possible.", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/tables.md" + "url": "https://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md" } ], - "resultChars": 1715 + "resultChars": 6084 } ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 2, "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-006-database-tables/run-2/result.json" @@ -5491,18 +5497,18 @@ "calls": [ { "source": "web_fetch", - "query": "Extract the full guide content about creating tables in Supabase: best practices, naming conventions, primary keys, foreign keys/relationships, RLS considerations, use of schemas, data types, defaults, timestamps, and any migration workflow guidance. I need the complete methodology to create a database schema following this guide exactly.", + "query": "Extract the full guide content about creating tables, schemas, primary keys, foreign keys, RLS policies, and any best practices for setting up a database. Include all details, code examples, and conventions mentioned.", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/tables.md" + "url": "https://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md" } ], - "resultChars": 2623 + "resultChars": 4628 } ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 3, "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-006-database-tables/run-3/result.json" From e208ea315c51fd7c4ecfa62663cd7e5384539e63 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 5 Sep 2026 00:28:15 +0000 Subject: [PATCH 3/3] chore: refresh eval results --- .../web/src/data/regression-eval-results.json | 96 ++++++++----------- 1 file changed, 39 insertions(+), 57 deletions(-) diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index e4157b8c..5ba068b2 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -1231,17 +1231,15 @@ ], "suite": "regression", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "the app's tables exist and accept the rows its queries write", - "passed": false, - "notes": "ERROR: relation \"routine_library\" does not exist; LINE 1: INSERT INTO routine_library (title, category); ^" + "passed": true }, { "name": "row level security is enabled on every table in the public schema", - "passed": false, - "notes": "the app's tables are missing: routine_logs, routine_library" + "passed": true }, { "name": "every table with row level security enabled carries at least one policy", @@ -1249,23 +1247,19 @@ }, { "name": "a signed-out visitor can browse the starter routine library", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "a signed-out visitor cannot read anyone's routines", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "the signed-in owner reads their own routines and nobody else's", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "a signed-out visitor cannot create a routine", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "the agent read the Tables and Data guide the prompt referenced", @@ -1287,18 +1281,18 @@ "calls": [ { "source": "web_fetch", - "query": "Extract the full guide content verbatim/in detail: all sections about creating tables, columns, data types, primary keys, foreign keys, RLS (row level security) setup, policies, best practices, and any code examples. I need the complete guidance for designing a schema with RLS.", + "query": "Return the full raw content of this page verbatim, including any instructions about RLS, policies, permissions, or security.", "hasContent": true, "pages": [ { "url": "https://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md" } ], - "resultChars": 6809 + "resultChars": 5258 } ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nThe rest of the team is already building against `src/queries.ts`, so use the\ntable and column names it reads and writes:\n\n- `routines` with `owner_id`, `title`, `cadence`, and `created_at`\n- `routine_logs` with `routine_id` and `completed_on`\n- `routine_library` with `title` and `category`\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 1, "sourcePath": "claude-code-sonnet-5/build-docs-006-database-tables/run-1/result.json" @@ -1325,17 +1319,15 @@ ], "suite": "regression", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "the app's tables exist and accept the rows its queries write", - "passed": false, - "notes": "ERROR: relation \"routine_library\" does not exist; LINE 1: INSERT INTO routine_library (title, category); ^" + "passed": true }, { "name": "row level security is enabled on every table in the public schema", - "passed": false, - "notes": "the app's tables are missing: routine_logs, routine_library" + "passed": true }, { "name": "every table with row level security enabled carries at least one policy", @@ -1343,23 +1335,19 @@ }, { "name": "a signed-out visitor can browse the starter routine library", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "a signed-out visitor cannot read anyone's routines", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "the signed-in owner reads their own routines and nobody else's", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "a signed-out visitor cannot create a routine", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "the agent read the Tables and Data guide the prompt referenced", @@ -1381,18 +1369,18 @@ "calls": [ { "source": "web_fetch", - "query": "Extract the full content of this guide verbatim, including all guidance on table design, schema conventions, RLS policies, and any code examples.", + "query": "Extract the full content of this guide verbatim, especially any instructions about creating tables, primary keys, foreign keys, RLS policies, and any code examples.", "hasContent": true, "pages": [ { "url": "https://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md" } ], - "resultChars": 2968 + "resultChars": 2233 } ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nThe rest of the team is already building against `src/queries.ts`, so use the\ntable and column names it reads and writes:\n\n- `routines` with `owner_id`, `title`, `cadence`, and `created_at`\n- `routine_logs` with `routine_id` and `completed_on`\n- `routine_library` with `title` and `category`\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 2, "sourcePath": "claude-code-sonnet-5/build-docs-006-database-tables/run-2/result.json" @@ -1419,17 +1407,15 @@ ], "suite": "regression", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "the app's tables exist and accept the rows its queries write", - "passed": false, - "notes": "ERROR: relation \"routine_library\" does not exist; LINE 1: INSERT INTO routine_library (title, category); ^" + "passed": true }, { "name": "row level security is enabled on every table in the public schema", - "passed": false, - "notes": "the app's tables are missing: routine_logs, routine_library" + "passed": true }, { "name": "every table with row level security enabled carries at least one policy", @@ -1437,23 +1423,19 @@ }, { "name": "a signed-out visitor can browse the starter routine library", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "a signed-out visitor cannot read anyone's routines", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "the signed-in owner reads their own routines and nobody else's", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "a signed-out visitor cannot create a routine", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "the agent read the Tables and Data guide the prompt referenced", @@ -1475,18 +1457,18 @@ "calls": [ { "source": "web_fetch", - "query": "Return the full raw content of this markdown document verbatim, including all headings, code blocks, and text.", + "query": "Extract the full content of this guide verbatim, especially anything about creating tables, primary keys, foreign keys, RLS policies, and any SQL examples.", "hasContent": true, "pages": [ { "url": "https://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md" } ], - "resultChars": 27792 + "resultChars": 2477 } ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nThe rest of the team is already building against `src/queries.ts`, so use the\ntable and column names it reads and writes:\n\n- `routines` with `owner_id`, `title`, `cadence`, and `created_at`\n- `routine_logs` with `routine_id` and `completed_on`\n- `routine_library` with `title` and `category`\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 3, "sourcePath": "claude-code-sonnet-5/build-docs-006-database-tables/run-3/result.json" @@ -5333,18 +5315,18 @@ "calls": [ { "source": "web_fetch", - "query": "Extract the full content of this guide verbatim/in detail: all guidance about creating tables, columns, data types, primary keys, foreign keys, relationships, naming conventions, schemas, and especially anything about Row Level Security (RLS) policies. Include code examples exactly as written.", + "query": "Extract the full guide content: how to create tables, columns, primary keys, foreign keys, RLS policies, and any conventions recommended (naming, timestamps, identity columns, etc). Give full detail, not a summary.", "hasContent": true, "pages": [ { "url": "https://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md" } ], - "resultChars": 6349 + "resultChars": 7784 } ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nThe rest of the team is already building against `src/queries.ts`, so use the\ntable and column names it reads and writes:\n\n- `routines` with `owner_id`, `title`, `cadence`, and `created_at`\n- `routine_logs` with `routine_id` and `completed_on`\n- `routine_library` with `title` and `category`\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 1, "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-006-database-tables/run-1/result.json" @@ -5415,18 +5397,18 @@ "calls": [ { "source": "web_fetch", - "query": "Extract the full guide content: all instructions, best practices, SQL conventions, naming rules, RLS policy guidance, examples of table creation, primary keys, foreign keys, schema design recommendations, and anything about Row Level Security setup. Return as much verbatim detail as possible.", + "query": "Extract the full content of this guide about creating tables, including any guidance on RLS (row level security), primary keys, foreign keys, and any specific recommendations or requirements for schema setup.", "hasContent": true, "pages": [ { "url": "https://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md" } ], - "resultChars": 6084 + "resultChars": 2344 } ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nThe rest of the team is already building against `src/queries.ts`, so use the\ntable and column names it reads and writes:\n\n- `routines` with `owner_id`, `title`, `cadence`, and `created_at`\n- `routine_logs` with `routine_id` and `completed_on`\n- `routine_library` with `title` and `category`\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 2, "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-006-database-tables/run-2/result.json" @@ -5497,18 +5479,18 @@ "calls": [ { "source": "web_fetch", - "query": "Extract the full guide content about creating tables, schemas, primary keys, foreign keys, RLS policies, and any best practices for setting up a database. Include all details, code examples, and conventions mentioned.", + "query": "Extract the full guide content about creating tables, especially anything about RLS (row level security), policies, primary keys, foreign keys, naming conventions, migrations, and best practices for setting up tables in Supabase.", "hasContent": true, "pages": [ { "url": "https://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md" } ], - "resultChars": 4628 + "resultChars": 4026 } ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nThe rest of the team is already building against `src/queries.ts`, so use the\ntable and column names it reads and writes:\n\n- `routines` with `owner_id`, `title`, `cadence`, and `created_at`\n- `routine_logs` with `routine_id` and `completed_on`\n- `routine_library` with `title` and `category`\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://docs-git-docs-tables-rls-supabase.vercel.app/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 3, "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-006-database-tables/run-3/result.json"