Merge pull request #3737 from liam-hq/supabase-repository-schema-bench

✨(schema-bench): Migrate to SupabaseRepositories for database persistence

Author: NoritakaIkeda

frontend/internal-packages/schema-bench/README.md

@@ -170,11 +170,22 @@ benchmark-workspace/
 
 ## Environment Setup
 
-For OpenAI executor, set your API key:
+### Required Environment Variables
+
+For **LiamDB executor**, you need Supabase connection details:
+```bash
+export NEXT_PUBLIC_SUPABASE_URL="your-supabase-url"
+export SUPABASE_SERVICE_ROLE_KEY="your-service-role-key"
+export OPENAI_API_KEY="your-api-key"  # Also required by LiamDB for AI operations
+```
+
+For **OpenAI executor**, you only need:
 ```bash
 export OPENAI_API_KEY="your-api-key"
 ```
 
+These variables should be set in a `.env` file at the repository root.
+
 ## Example Workflow
 
 1) Clean and setup: `rm -rf benchmark-workspace && pnpm --filter @liam-hq/schema-bench setupWorkspace`

frontend/internal-packages/schema-bench/package.json

@@ -6,6 +6,7 @@
   "dependencies": {
     "@huggingface/transformers": "3.3.3",
     "@liam-hq/agent": "workspace:*",
+    "@liam-hq/db": "workspace:*",
     "@liam-hq/neverthrow": "workspace:*",
     "@liam-hq/schema": "workspace:*",
     "dotenv": "16.5.0",

frontend/internal-packages/schema-bench/src/executors/liamDb/liamDbExecutor.ts

@@ -1,39 +1,46 @@
-import { deepModeling, InMemoryRepository } from '@liam-hq/agent'
+import { deepModeling } from '@liam-hq/agent'
 import { aSchema } from '@liam-hq/schema'
 import { err, ok, type Result } from 'neverthrow'
 import { handleExecutionResult, logInputProcessing } from '../utils.ts'
+import { setupRepositories } from './setupRepositories.ts'
 import type { LiamDbExecutorInput, LiamDbExecutorOutput } from './types.ts'
 
 export async function execute(
   input: LiamDbExecutorInput,
 ): Promise<Result<LiamDbExecutorOutput, Error>> {
   logInputProcessing(input.input)
 
-  // Setup InMemory repository
-  const repositories = {
-    schema: new InMemoryRepository({
-      schemas: {
-        'demo-design-session': aSchema({
-          tables: {},
-        }),
-      },
-    }),
+  // Setup Supabase repositories
+  const setupResult = await setupRepositories()
+  if (setupResult.isErr()) {
+    return err(
+      new Error(`Failed to setup repositories: ${setupResult.error.message}`),
+    )
   }
 
+  const {
+    repositories,
+    organizationId,
+    buildingSchemaId,
+    designSessionId,
+    userId,
+  } = setupResult.value
+
   // Create workflow params
   const workflowParams = {
     userInput: input.input,
     schemaData: aSchema({ tables: {} }),
-    organizationId: 'demo-org-id',
-    designSessionId: 'demo-design-session',
-    userId: 'demo-user-id',
+    organizationId,
+    buildingSchemaId,
+    designSessionId,
+    userId,
     signal: new AbortController().signal,
   }
 
   const config = {
     configurable: {
       repositories,
-      thread_id: 'demo-design-session',
+      thread_id: designSessionId,
     },
   }
 

frontend/internal-packages/schema-bench/src/executors/liamDb/setupRepositories.ts

@@ -0,0 +1,214 @@
+import { createSupabaseRepositories } from '@liam-hq/agent'
+import {
+  createClient,
+  type SupabaseClientType,
+  toResultAsync,
+} from '@liam-hq/db'
+import { err, ok, type Result, type ResultAsync } from 'neverthrow'
+
+type Repositories = ReturnType<typeof createSupabaseRepositories>
+
+type SetupResult = {
+  repositories: Repositories
+  organizationId: string
+  buildingSchemaId: string
+  designSessionId: string
+  userId: string
+}
+
+let cachedSetup: SetupResult | null = null
+
+/**
+ * Create Supabase client
+ */
+const createDatabaseConnection = (): Result<SupabaseClientType, Error> => {
+  const supabaseUrl = process.env['NEXT_PUBLIC_SUPABASE_URL']
+  const supabaseKey = process.env['SUPABASE_SERVICE_ROLE_KEY']
+
+  if (!supabaseUrl || !supabaseKey) {
+    return err(
+      new Error(
+        'Missing required environment variables: NEXT_PUBLIC_SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY',
+      ),
+    )
+  }
+
+  const supabaseClient = createClient(supabaseUrl, supabaseKey)
+  return ok(supabaseClient)
+}
+
+/**
+ * Get the first organization from database
+ */
+const getFirstOrganization = (
+  supabaseClient: SupabaseClientType,
+): ResultAsync<{ id: string }, Error> => {
+  return toResultAsync(
+    supabaseClient.from('organizations').select('id').limit(1).single(),
+  )
+}
+
+/**
+ * Get the first user from database
+ */
+const getUser = (
+  supabaseClient: SupabaseClientType,
+): ResultAsync<{ id: string }, Error> => {
+  return toResultAsync(
+    supabaseClient.from('users').select('id').limit(1).single(),
+  )
+}
+
+const getDesignSession = (
+  supabaseClient: SupabaseClientType,
+  organizationId: string,
+): ResultAsync<{ id: string }, Error> => {
+  return toResultAsync(
+    supabaseClient
+      .from('design_sessions')
+      .select('id')
+      .eq('organization_id', organizationId)
+      .limit(1)
+      .single(),
+  )
+}
+
+const createDesignSession = (
+  supabaseClient: SupabaseClientType,
+  organizationId: string,
+  userId: string,
+): ResultAsync<{ id: string }, Error> => {
+  return toResultAsync(
+    supabaseClient
+      .from('design_sessions')
+      .insert({
+        name: 'Schema Benchmark Session',
+        project_id: null,
+        organization_id: organizationId,
+        created_by_user_id: userId,
+        parent_design_session_id: null,
+      })
+      .select('id')
+      .single(),
+  )
+}
+
+/**
+ * Get or create a design session for benchmarking
+ */
+const getOrCreateDesignSession = (
+  supabaseClient: SupabaseClientType,
+  organizationId: string,
+  userId: string,
+): ResultAsync<{ id: string }, Error> => {
+  return getDesignSession(supabaseClient, organizationId).orElse(() =>
+    createDesignSession(supabaseClient, organizationId, userId),
+  )
+}
+
+const getBuildingSchema = (
+  supabaseClient: SupabaseClientType,
+  designSessionId: string,
+): ResultAsync<{ id: string }, Error> => {
+  return toResultAsync(
+    supabaseClient
+      .from('building_schemas')
+      .select('id')
+      .eq('design_session_id', designSessionId)
+      .limit(1)
+      .single(),
+  )
+}
+
+const createBuildingSchema = (
+  supabaseClient: SupabaseClientType,
+  designSessionId: string,
+  organizationId: string,
+): ResultAsync<{ id: string }, Error> => {
+  const initialSchema = { tables: {}, enums: {}, extensions: {} }
+  return toResultAsync(
+    supabaseClient
+      .from('building_schemas')
+      .insert({
+        design_session_id: designSessionId,
+        organization_id: organizationId,
+        schema: structuredClone(initialSchema),
+        initial_schema_snapshot: structuredClone(initialSchema),
+        schema_file_path: null,
+        git_sha: null,
+      })
+      .select('id')
+      .single(),
+  )
+}
+
+/**
+ * Get or create a building schema for the design session
+ */
+const getOrCreateBuildingSchema = (
+  supabaseClient: SupabaseClientType,
+  designSessionId: string,
+  organizationId: string,
+): ResultAsync<{ id: string }, Error> => {
+  return getBuildingSchema(supabaseClient, designSessionId).orElse(() =>
+    createBuildingSchema(supabaseClient, designSessionId, organizationId),
+  )
+}
+
+/**
+ * Setup Supabase repositories for schema benchmark
+ * Uses singleton pattern to reuse connections across benchmark runs
+ */
+export const setupRepositories = async (): Promise<
+  Result<SetupResult, Error>
+> => {
+  // Return cached setup if available
+  if (cachedSetup) {
+    return ok(cachedSetup)
+  }
+
+  // Create database connection
+  const connectionResult = createDatabaseConnection()
+  if (connectionResult.isErr()) {
+    return err(connectionResult.error)
+  }
+  const supabaseClient = connectionResult.value
+
+  return await getFirstOrganization(supabaseClient)
+    .andThen((organization) =>
+      getUser(supabaseClient).andThen((user) =>
+        getOrCreateDesignSession(
+          supabaseClient,
+          organization.id,
+          user.id,
+        ).andThen((designSession) =>
+          getOrCreateBuildingSchema(
+            supabaseClient,
+            designSession.id,
+            organization.id,
+          ).map((buildingSchema) => ({
+            organization,
+            user,
+            designSession,
+            buildingSchema,
+          })),
+        ),
+      ),
+    )
+    .map(({ organization, user, designSession, buildingSchema }) => {
+      const repositories = createSupabaseRepositories(
+        supabaseClient,
+        organization.id,
+      )
+
+      cachedSetup = {
+        repositories,
+        organizationId: organization.id,
+        buildingSchemaId: buildingSchema.id,
+        designSessionId: designSession.id,
+        userId: user.id,
+      }
+
+      return cachedSetup
+    })
+}