diff --git a/frontend/internal-packages/agent/src/db-agent/nodes/designSchemaNode.ts b/frontend/internal-packages/agent/src/db-agent/nodes/designSchemaNode.ts index 9820da4697..b6d47fbbc0 100644 --- a/frontend/internal-packages/agent/src/db-agent/nodes/designSchemaNode.ts +++ b/frontend/internal-packages/agent/src/db-agent/nodes/designSchemaNode.ts @@ -1,5 +1,6 @@ import type { RunnableConfig } from '@langchain/core/runnables' -import { convertSchemaToText } from '../../utils/convertSchemaToText' +import { yamlSchemaDeparser } from '@liam-hq/schema' +import { Result } from 'neverthrow' import { WorkflowTerminationError } from '../../utils/errorHandling' import { getConfigurable } from '../../utils/getConfigurable' import { removeReasoningFromMessages } from '../../utils/messageCleanup' @@ -14,16 +15,15 @@ export async function designSchemaNode( state: DbAgentState, config: RunnableConfig, ): Promise { - const configurableResult = getConfigurable(config) - if (configurableResult.isErr()) { - throw new WorkflowTerminationError( - configurableResult.error, - 'designSchemaNode', - ) - } - const { repositories } = configurableResult.value + const combinedResult = Result.combine([ + getConfigurable(config), + yamlSchemaDeparser(state.schemaData), + ]) - const schemaText = convertSchemaToText(state.schemaData) + if (combinedResult.isErr()) { + throw new WorkflowTerminationError(combinedResult.error, 'designSchemaNode') + } + const [{ repositories }, schemaText] = combinedResult.value // Remove reasoning field from AIMessages to avoid API issues // This prevents the "reasoning without required following item" error diff --git a/frontend/internal-packages/agent/src/pm-agent/nodes/analyzeRequirementsNode.ts b/frontend/internal-packages/agent/src/pm-agent/nodes/analyzeRequirementsNode.ts index 97d9dfc342..484452e5c2 100644 --- a/frontend/internal-packages/agent/src/pm-agent/nodes/analyzeRequirementsNode.ts +++ b/frontend/internal-packages/agent/src/pm-agent/nodes/analyzeRequirementsNode.ts @@ -1,5 +1,6 @@ import type { RunnableConfig } from '@langchain/core/runnables' -import { convertSchemaToText } from '../../utils/convertSchemaToText' +import { yamlSchemaDeparser } from '@liam-hq/schema' +import { Result } from 'neverthrow' import { WorkflowTerminationError } from '../../utils/errorHandling' import { getConfigurable } from '../../utils/getConfigurable' import { invokePmAnalysisAgent } from '../invokePmAnalysisAgent' @@ -13,20 +14,24 @@ export async function analyzeRequirementsNode( state: PmAgentState, config: RunnableConfig, ): Promise> { - const configurableResult = getConfigurable(config) - if (configurableResult.isErr()) { + const combinedResult = Result.combine([ + getConfigurable(config), + yamlSchemaDeparser(state.schemaData), + ]) + + if (combinedResult.isErr()) { throw new WorkflowTerminationError( - configurableResult.error, + combinedResult.error, 'analyzeRequirementsNode', ) } - const schemaText = convertSchemaToText(state.schemaData) + const [configurable, schemaText] = combinedResult.value const analysisResult = await invokePmAnalysisAgent( { schemaText }, state.messages, - configurableResult.value, + configurable, ) if (analysisResult.isErr()) { diff --git a/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/generateTestcaseNode.ts b/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/generateTestcaseNode.ts index f71a53fdde..be78f5d84f 100644 --- a/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/generateTestcaseNode.ts +++ b/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/generateTestcaseNode.ts @@ -5,7 +5,7 @@ import { } from '@langchain/core/messages' import { ChatOpenAI } from '@langchain/openai' import { fromAsyncThrowable } from '@liam-hq/neverthrow' -import { convertSchemaToText } from '../../utils/convertSchemaToText' +import { yamlSchemaDeparser } from '@liam-hq/schema' import { removeReasoningFromMessages } from '../../utils/messageCleanup' import { streamLLMResponse } from '../../utils/streamingLlmUtils' import { saveTestcaseTool } from '../tools/saveTestcaseTool' @@ -36,7 +36,11 @@ export async function generateTestcaseNode( ): Promise<{ messages: BaseMessage[] }> { const { currentTestcase, schemaData, goal, messages } = state - const schemaContext = convertSchemaToText(schemaData) + const schemaContextResult = yamlSchemaDeparser(schemaData) + if (schemaContextResult.isErr()) { + throw schemaContextResult.error + } + const schemaContext = schemaContextResult.value const contextMessage = await humanPromptTemplateForTestcaseGeneration.format({ schemaContext, diff --git a/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/validateSchemaRequirementsNode.ts b/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/validateSchemaRequirementsNode.ts index f8e0026ee7..6bc9aff471 100644 --- a/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/validateSchemaRequirementsNode.ts +++ b/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/validateSchemaRequirementsNode.ts @@ -2,8 +2,8 @@ import { HumanMessage, SystemMessage } from '@langchain/core/messages' import { Command, END } from '@langchain/langgraph' import { ChatOpenAI } from '@langchain/openai' import { fromPromise } from '@liam-hq/neverthrow' +import { yamlSchemaDeparser } from '@liam-hq/schema' import * as v from 'valibot' -import { convertSchemaToText } from '../../utils/convertSchemaToText' import { toJsonSchema } from '../../utils/jsonSchema' import type { testcaseAnnotation } from './testcaseAnnotation' @@ -47,7 +47,11 @@ export async function validateSchemaRequirementsNode( ): Promise { const { currentTestcase, schemaData, goal } = state - const schemaContext = convertSchemaToText(schemaData) + const schemaContextResult = yamlSchemaDeparser(schemaData) + if (schemaContextResult.isErr()) { + throw schemaContextResult.error + } + const schemaContext = schemaContextResult.value const contextMessage = ` # Database Schema Context diff --git a/frontend/internal-packages/agent/src/utils/convertSchemaToText.ts b/frontend/internal-packages/agent/src/utils/convertSchemaToText.ts deleted file mode 100644 index 87b63fe765..0000000000 --- a/frontend/internal-packages/agent/src/utils/convertSchemaToText.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { isPrimaryKey, type Schema } from '@liam-hq/schema' - -const tableToDocument = ( - tableName: string, - tableData: Schema['tables'][string], -): string => { - const tableDescription = `Table: ${tableName}\nDescription: ${tableData.comment || 'No description'}\n` - - let columnsText = 'Columns:\n' - if (tableData.columns) { - for (const [columnName, columnData] of Object.entries(tableData.columns)) { - columnsText += `- ${columnName}: ${columnData.type || 'unknown type'} ${!columnData.notNull ? '(nullable)' : '(not nullable)'}\n` - if (columnData.comment) { - columnsText += ` Description: ${columnData.comment}\n` - } - } - } - - let primaryKeyText = '' - const primaryKeyColumns = Object.entries(tableData.columns || {}) - .filter(([name]) => isPrimaryKey(name, tableData.constraints || {})) - .map(([name]) => name) - - if (primaryKeyColumns.length > 0) { - primaryKeyText = `Primary Key: ${primaryKeyColumns.join(', ')}\n` - } - - return `${tableDescription}${columnsText}${primaryKeyText}` -} - -export const convertSchemaToText = (schema: Schema): string => { - let schemaText = 'FULL DATABASE SCHEMA:\n\n' - - if (schema.tables) { - schemaText += 'TABLES:\n\n' - for (const [tableName, tableData] of Object.entries(schema.tables)) { - const tableDoc = tableToDocument(tableName, tableData) - schemaText = `${schemaText}${tableDoc}\n\n` - } - } - - return schemaText -} diff --git a/frontend/packages/schema/package.json b/frontend/packages/schema/package.json index e0cca454db..f848a54122 100644 --- a/frontend/packages/schema/package.json +++ b/frontend/packages/schema/package.json @@ -23,6 +23,7 @@ "neverthrow": "8.2.0", "pg-query-emscripten": "5.1.0", "valibot": "1.1.0", + "yaml": "2.8.1", "zod": "3.25.76" }, "devDependencies": { diff --git a/frontend/packages/schema/src/deparser/postgresql/operationDeparser.ts b/frontend/packages/schema/src/deparser/postgresql/operationDeparser.ts index 3d36933fea..55fe94c44f 100644 --- a/frontend/packages/schema/src/deparser/postgresql/operationDeparser.ts +++ b/frontend/packages/schema/src/deparser/postgresql/operationDeparser.ts @@ -53,7 +53,7 @@ import { isReplaceTableCommentOperation, isReplaceTableNameOperation, } from '../../operation/schema/table.js' -import type { OperationDeparser } from '../type.js' +import type { LegacyOperationDeparser } from '../type.js' import { generateAddCheckConstraintStatement, generateAddColumnStatement, @@ -605,7 +605,12 @@ function generateAlterConstraintUpdateFromOperation( ) } -export const postgresqlOperationDeparser: OperationDeparser = ( +/** + * PostgreSQL operation deparser + * @deprecated This implementation uses LegacyOperationDeparser type. + * TODO: Migrate to new OperationDeparser type (Result) for better error handling. + */ +export const postgresqlOperationDeparser: LegacyOperationDeparser = ( operation: Operation, // biome-ignore lint/complexity/noExcessiveCognitiveComplexity: TODO: Refactor to reduce complexity ) => { diff --git a/frontend/packages/schema/src/deparser/postgresql/schemaDeparser.ts b/frontend/packages/schema/src/deparser/postgresql/schemaDeparser.ts index 6d2bd3e58f..a0e99fad8a 100644 --- a/frontend/packages/schema/src/deparser/postgresql/schemaDeparser.ts +++ b/frontend/packages/schema/src/deparser/postgresql/schemaDeparser.ts @@ -5,7 +5,7 @@ import type { Schema, Table, } from '../../schema/index.js' -import type { SchemaDeparser } from '../type.js' +import type { LegacySchemaDeparser } from '../type.js' import { generateAddConstraintStatement, generateCreateEnumStatement, @@ -14,7 +14,14 @@ import { generateCreateTableStatement, } from './utils.js' -export const postgresqlSchemaDeparser: SchemaDeparser = (schema: Schema) => { +/** + * PostgreSQL schema deparser + * @deprecated This implementation uses LegacySchemaDeparser type. + * TODO: Migrate to new SchemaDeparser type (Result) for better error handling. + */ +export const postgresqlSchemaDeparser: LegacySchemaDeparser = ( + schema: Schema, +) => { const ddlStatements: string[] = [] const errors: { message: string }[] = [] diff --git a/frontend/packages/schema/src/deparser/type.ts b/frontend/packages/schema/src/deparser/type.ts index ababe3b2e4..a6b3d3b2ff 100644 --- a/frontend/packages/schema/src/deparser/type.ts +++ b/frontend/packages/schema/src/deparser/type.ts @@ -1,14 +1,31 @@ +import type { Result } from 'neverthrow' import type { Operation } from '../operation/schema/index.js' import type { Schema } from '../schema/index.js' +// Legacy types - TODO: Migrate all implementations to use the new types type DeparserError = { message: string } -type DeparserResult = { +type LegacyDeparserResult = { value: string errors: DeparserError[] } -export type SchemaDeparser = (schema: Schema) => DeparserResult -export type OperationDeparser = (operation: Operation) => DeparserResult +/** + * @deprecated Use SchemaDeparser instead. This type is kept for backward compatibility. + * TODO: Migrate existing implementations to use the new SchemaDeparser type. + */ +export type LegacySchemaDeparser = (schema: Schema) => LegacyDeparserResult + +/** + * @deprecated Use OperationDeparser instead. This type is kept for backward compatibility. + * TODO: Migrate existing implementations to use the new OperationDeparser type. + */ +export type LegacyOperationDeparser = ( + operation: Operation, +) => LegacyDeparserResult + +// New types using neverthrow +export type SchemaDeparser = (schema: Schema) => Result +export type OperationDeparser = (operation: Operation) => Result diff --git a/frontend/packages/schema/src/deparser/yaml/index.ts b/frontend/packages/schema/src/deparser/yaml/index.ts new file mode 100644 index 0000000000..19288234d0 --- /dev/null +++ b/frontend/packages/schema/src/deparser/yaml/index.ts @@ -0,0 +1 @@ +export { yamlSchemaDeparser } from './schemaDeparser.js' diff --git a/frontend/packages/schema/src/deparser/yaml/schemaDeparser.test.ts b/frontend/packages/schema/src/deparser/yaml/schemaDeparser.test.ts new file mode 100644 index 0000000000..1d79ad82f0 --- /dev/null +++ b/frontend/packages/schema/src/deparser/yaml/schemaDeparser.test.ts @@ -0,0 +1,464 @@ +import { describe, expect, it } from 'vitest' +import { + aColumn, + aForeignKeyConstraint, + anEnum, + anIndex, + aPrimaryKeyConstraint, + aSchema, + aTable, + aUniqueConstraint, +} from '../../schema/factories.js' +import { yamlSchemaDeparser } from './schemaDeparser.js' + +describe('yamlSchemaDeparser', () => { + it('should convert basic schema to YAML', () => { + const schema = aSchema({ + tables: { + users: aTable({ + name: 'users', + columns: { + id: aColumn({ + name: 'id', + type: 'bigint', + notNull: true, + }), + email: aColumn({ + name: 'email', + type: 'varchar(255)', + notNull: true, + }), + }, + constraints: { + users_pkey: aPrimaryKeyConstraint({ + name: 'users_pkey', + columnNames: ['id'], + }), + }, + }), + }, + }) + + const result = yamlSchemaDeparser(schema)._unsafeUnwrap() + + expect(result).toMatchInlineSnapshot(` + "tables: + users: + name: users + columns: + id: + name: id + type: bigint + notNull: true + email: + name: email + type: varchar(255) + notNull: true + constraints: + users_pkey: + type: PRIMARY KEY + name: users_pkey + columnNames: + - id + indexes: {} + enums: {} + extensions: {} + " + `) + }) + + it('should handle schema with comments', () => { + const schema = aSchema({ + tables: { + products: aTable({ + name: 'products', + comment: 'Product table', + columns: { + id: aColumn({ + name: 'id', + type: 'bigint', + notNull: true, + comment: 'Product ID', + }), + }, + constraints: { + products_pkey: aPrimaryKeyConstraint({ + name: 'products_pkey', + columnNames: ['id'], + }), + }, + }), + }, + }) + + const result = yamlSchemaDeparser(schema)._unsafeUnwrap() + + expect(result).toMatchInlineSnapshot(` + "tables: + products: + name: products + comment: Product table + columns: + id: + name: id + type: bigint + comment: Product ID + notNull: true + constraints: + products_pkey: + type: PRIMARY KEY + name: products_pkey + columnNames: + - id + indexes: {} + enums: {} + extensions: {} + " + `) + }) + + it('should handle schema with enums', () => { + const schema = aSchema({ + enums: { + status: anEnum({ + name: 'status', + values: ['active', 'inactive', 'pending'], + }), + }, + tables: {}, + }) + + const result = yamlSchemaDeparser(schema)._unsafeUnwrap() + + expect(result).toMatchInlineSnapshot(` + "tables: {} + enums: + status: + name: status + values: + - active + - inactive + - pending + extensions: {} + " + `) + }) + + it('should handle schema with indexes', () => { + const schema = aSchema({ + tables: { + users: aTable({ + name: 'users', + columns: { + id: aColumn({ + name: 'id', + type: 'bigint', + notNull: true, + }), + email: aColumn({ + name: 'email', + type: 'varchar(255)', + notNull: true, + }), + }, + indexes: { + idx_users_email: anIndex({ + name: 'idx_users_email', + columns: ['email'], + type: 'BTREE', + }), + }, + }), + }, + }) + + const result = yamlSchemaDeparser(schema)._unsafeUnwrap() + + expect(result).toMatchInlineSnapshot(` + "tables: + users: + name: users + columns: + id: + name: id + type: bigint + notNull: true + email: + name: email + type: varchar(255) + notNull: true + indexes: + idx_users_email: + name: idx_users_email + unique: false + columns: + - email + type: BTREE + constraints: {} + enums: {} + extensions: {} + " + `) + }) + + it('should handle schema with foreign key constraints', () => { + const schema = aSchema({ + tables: { + users: aTable({ + name: 'users', + columns: { + id: aColumn({ + name: 'id', + type: 'bigint', + notNull: true, + }), + }, + }), + orders: aTable({ + name: 'orders', + columns: { + id: aColumn({ + name: 'id', + type: 'bigint', + notNull: true, + }), + user_id: aColumn({ + name: 'user_id', + type: 'bigint', + notNull: true, + }), + }, + constraints: { + fk_orders_user_id: aForeignKeyConstraint({ + name: 'fk_orders_user_id', + columnNames: ['user_id'], + targetTableName: 'users', + targetColumnNames: ['id'], + updateConstraint: 'CASCADE', + deleteConstraint: 'SET_NULL', + }), + }, + }), + }, + }) + + const result = yamlSchemaDeparser(schema)._unsafeUnwrap() + + expect(result).toMatchInlineSnapshot(` + "tables: + users: + name: users + columns: + id: + name: id + type: bigint + notNull: true + indexes: {} + constraints: {} + orders: + name: orders + columns: + id: + name: id + type: bigint + notNull: true + user_id: + name: user_id + type: bigint + notNull: true + constraints: + fk_orders_user_id: + type: FOREIGN KEY + name: fk_orders_user_id + columnNames: + - user_id + targetTableName: users + targetColumnNames: + - id + updateConstraint: CASCADE + deleteConstraint: SET_NULL + indexes: {} + enums: {} + extensions: {} + " + `) + }) + + it('should handle schema with unique constraints', () => { + const schema = aSchema({ + tables: { + users: aTable({ + name: 'users', + columns: { + id: aColumn({ + name: 'id', + type: 'bigint', + notNull: true, + }), + email: aColumn({ + name: 'email', + type: 'varchar(255)', + notNull: true, + }), + }, + constraints: { + uk_users_email: aUniqueConstraint({ + name: 'uk_users_email', + columnNames: ['email'], + }), + }, + }), + }, + }) + + const result = yamlSchemaDeparser(schema)._unsafeUnwrap() + + expect(result).toMatchInlineSnapshot(` + "tables: + users: + name: users + columns: + id: + name: id + type: bigint + notNull: true + email: + name: email + type: varchar(255) + notNull: true + constraints: + uk_users_email: + type: UNIQUE + name: uk_users_email + columnNames: + - email + indexes: {} + enums: {} + extensions: {} + " + `) + }) + + it('should handle empty schema', () => { + const schema = aSchema({ tables: {} }) + + const result = yamlSchemaDeparser(schema)._unsafeUnwrap() + + expect(result).toMatchInlineSnapshot(` + "tables: {} + enums: {} + extensions: {} + " + `) + }) + + it('should handle complex schema with multiple tables, enums, and constraints', () => { + const schema = aSchema({ + enums: { + user_status: anEnum({ + name: 'user_status', + values: ['active', 'inactive'], + }), + }, + tables: { + users: aTable({ + name: 'users', + comment: 'Users table', + columns: { + id: aColumn({ + name: 'id', + type: 'bigint', + notNull: true, + comment: 'User ID', + }), + email: aColumn({ + name: 'email', + type: 'varchar(255)', + notNull: true, + }), + status: aColumn({ + name: 'status', + type: 'user_status', + notNull: true, + }), + }, + indexes: { + idx_users_email: anIndex({ + name: 'idx_users_email', + unique: true, + columns: ['email'], + type: 'BTREE', + }), + }, + }), + products: aTable({ + name: 'products', + columns: { + id: aColumn({ + name: 'id', + type: 'bigint', + notNull: true, + }), + name: aColumn({ + name: 'name', + type: 'varchar(100)', + notNull: true, + }), + }, + }), + }, + }) + + const result = yamlSchemaDeparser(schema)._unsafeUnwrap() + + expect(result).toMatchInlineSnapshot(` + "tables: + users: + name: users + comment: Users table + columns: + id: + name: id + type: bigint + comment: User ID + notNull: true + email: + name: email + type: varchar(255) + notNull: true + status: + name: status + type: user_status + notNull: true + indexes: + idx_users_email: + name: idx_users_email + unique: true + columns: + - email + type: BTREE + constraints: {} + products: + name: products + columns: + id: + name: id + type: bigint + notNull: true + name: + name: name + type: varchar(100) + notNull: true + indexes: {} + constraints: {} + enums: + user_status: + name: user_status + values: + - active + - inactive + extensions: {} + " + `) + }) +}) diff --git a/frontend/packages/schema/src/deparser/yaml/schemaDeparser.ts b/frontend/packages/schema/src/deparser/yaml/schemaDeparser.ts new file mode 100644 index 0000000000..ac5f422e5e --- /dev/null +++ b/frontend/packages/schema/src/deparser/yaml/schemaDeparser.ts @@ -0,0 +1,35 @@ +import { fromThrowable } from '@liam-hq/neverthrow' +import yaml from 'yaml' +import type { Schema } from '../../schema/index.js' +import type { SchemaDeparser } from '../type.js' + +const removeNullValues = (obj: unknown): unknown => { + if (Array.isArray(obj)) { + return obj.map(removeNullValues) + } + if (obj && typeof obj === 'object') { + return Object.fromEntries( + Object.entries(obj) + .filter(([, v]) => v !== null) + .map(([k, v]) => [k, removeNullValues(v)]), + ) + } + return obj +} + +export const yamlSchemaDeparser: SchemaDeparser = (schema: Schema) => { + const cleanedSchema = removeNullValues(schema) + return fromThrowable( + () => + yaml.stringify(cleanedSchema, { + defaultStringType: 'PLAIN', + defaultKeyType: 'PLAIN', + lineWidth: 0, + minContentWidth: 0, + }), + (error) => + error instanceof Error + ? error + : new Error(`Failed to stringify YAML: ${String(error)}`), + )() +} diff --git a/frontend/packages/schema/src/index.ts b/frontend/packages/schema/src/index.ts index ce960e1174..416eafe6ed 100644 --- a/frontend/packages/schema/src/index.ts +++ b/frontend/packages/schema/src/index.ts @@ -2,7 +2,13 @@ export { postgresqlOperationDeparser, postgresqlSchemaDeparser, } from './deparser/postgresql/index.js' -export type { OperationDeparser, SchemaDeparser } from './deparser/type.js' +export type { + LegacyOperationDeparser, + LegacySchemaDeparser, + OperationDeparser, + SchemaDeparser, +} from './deparser/type.js' +export { yamlSchemaDeparser } from './deparser/yaml/index.js' export { PATH_PATTERNS } from './operation/constants.js' export { applyPatchOperations, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 82e7a6cef9..272fd03b30 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -961,6 +961,9 @@ importers: valibot: specifier: 1.1.0 version: 1.1.0(typescript@5.9.2) + yaml: + specifier: 2.8.1 + version: 2.8.1 zod: specifier: 3.25.76 version: 3.25.76