diff --git a/frontend/internal-packages/agent/src/qa-agent/distributeRequirements/index.ts b/frontend/internal-packages/agent/src/qa-agent/distributeRequirements/index.ts index 74b474a17a..d359db3859 100644 --- a/frontend/internal-packages/agent/src/qa-agent/distributeRequirements/index.ts +++ b/frontend/internal-packages/agent/src/qa-agent/distributeRequirements/index.ts @@ -1,5 +1,4 @@ import { Send } from '@langchain/langgraph' -import { convertSchemaToText } from '../../utils/convertSchemaToText' import type { QaAgentState } from '../shared/qaAgentAnnotation' import { getUnprocessedRequirements } from './getUnprocessedRequirements' @@ -11,7 +10,6 @@ export type { RequirementData } from './types' */ export function continueToRequirements(state: QaAgentState) { const targetRequirements = getUnprocessedRequirements(state) - const schemaContext = convertSchemaToText(state.schemaData) // Use Send API to distribute each requirement for parallel processing // Each requirement will be processed by testcaseGeneration with isolated state @@ -20,7 +18,7 @@ export function continueToRequirements(state: QaAgentState) { new Send('testcaseGeneration', { // Each subgraph gets its own isolated state currentRequirement: reqData, - schemaContext, + schemaData: state.schemaData, messages: [], // Start with empty messages for isolation testcases: [], // Will be populated by the subgraph }), diff --git a/frontend/internal-packages/agent/src/qa-agent/shared/qaAgentAnnotation.ts b/frontend/internal-packages/agent/src/qa-agent/shared/qaAgentAnnotation.ts index 79f8920b4f..58e80ab80b 100644 --- a/frontend/internal-packages/agent/src/qa-agent/shared/qaAgentAnnotation.ts +++ b/frontend/internal-packages/agent/src/qa-agent/shared/qaAgentAnnotation.ts @@ -10,7 +10,12 @@ import type { Testcase } from '../types' */ export const qaAgentAnnotation = Annotation.Root({ ...MessagesAnnotation.spec, - schemaData: Annotation, + schemaData: Annotation({ + // Read-only field: QA agent should not modify schema data, only read it + // Using identity reducer to maintain existing value and avoid InvalidUpdateError + // when multiple subgraphs pass the same schemaData back to parent state + reducer: (x) => x, + }), analyzedRequirements: Annotation({ reducer: (x, y) => y ?? x, default: () => ({ diff --git a/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/executeSingleTestNode.ts b/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/executeSingleTestNode.ts new file mode 100644 index 0000000000..a0c8f5167c --- /dev/null +++ b/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/executeSingleTestNode.ts @@ -0,0 +1,21 @@ +import { AIMessage, type BaseMessage } from '@langchain/core/messages' +import { v4 as uuidv4 } from 'uuid' +import type { testcaseAnnotation } from './testcaseAnnotation' + +export async function executeSingleTestNode( + _state: typeof testcaseAnnotation.State, +): Promise<{ messages: BaseMessage[] }> { + const toolCallId = uuidv4() + const aiMessage = new AIMessage({ + content: 'Running single test case to validate the generated testcase.', + tool_calls: [ + { + id: toolCallId, + name: 'runSingleTestTool', + args: {}, + }, + ], + }) + + return { messages: [aiMessage] } +} diff --git a/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/generateTestcaseNode.ts b/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/generateTestcaseNode.ts index 9208eeec1e..a85e779ede 100644 --- a/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/generateTestcaseNode.ts +++ b/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/generateTestcaseNode.ts @@ -5,6 +5,7 @@ import { } from '@langchain/core/messages' import { ChatOpenAI } from '@langchain/openai' import { fromAsyncThrowable } from '@liam-hq/neverthrow' +import { convertSchemaToText } from '../../utils/convertSchemaToText' import { removeReasoningFromMessages } from '../../utils/messageCleanup' import { streamLLMResponse } from '../../utils/streamingLlmUtils' import { saveTestcaseTool } from '../tools/saveTestcaseTool' @@ -32,7 +33,9 @@ const model = new ChatOpenAI({ export async function generateTestcaseNode( state: typeof testcaseAnnotation.State, ): Promise<{ messages: BaseMessage[] }> { - const { currentRequirement, schemaContext, messages } = state + const { currentRequirement, schemaData, messages } = state + + const schemaContext = convertSchemaToText(schemaData) const contextMessage = await humanPromptTemplateForTestcaseGeneration.format({ schemaContext, diff --git a/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/index.integration.test.ts b/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/index.integration.test.ts index 3e55ac9cb2..ae35154be6 100644 --- a/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/index.integration.test.ts +++ b/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/index.integration.test.ts @@ -1,4 +1,5 @@ import { END, START, StateGraph } from '@langchain/langgraph' +import { aColumn, aSchema, aTable } from '@liam-hq/schema' import { describe, it } from 'vitest' import { getTestConfig, @@ -20,6 +21,27 @@ describe('testcaseGeneration Integration', () => { type TestcaseState = typeof testcaseAnnotation.State + const mockSchema = aSchema({ + tables: { + users: aTable({ + name: 'users', + columns: { + id: aColumn({ name: 'id', type: 'uuid', notNull: true }), + email: aColumn({ name: 'email', type: 'varchar', notNull: true }), + }, + }), + tasks: aTable({ + name: 'tasks', + columns: { + id: aColumn({ name: 'id', type: 'uuid', notNull: true }), + user_id: aColumn({ name: 'user_id', type: 'uuid', notNull: true }), + title: aColumn({ name: 'title', type: 'varchar', notNull: true }), + status: aColumn({ name: 'status', type: 'varchar', notNull: true }), + }, + }), + }, + }) + const state: TestcaseState = { messages: [], currentRequirement: { @@ -30,17 +52,7 @@ describe('testcaseGeneration Integration', () => { 'A task management system where users create projects and tasks', requirementId: '550e8400-e29b-41d4-a716-446655440000', }, - schemaContext: ` -Table: users -- id: uuid (not null) -- email: varchar (not null) - -Table: tasks -- id: uuid (not null) -- user_id: uuid (not null) -- title: varchar (not null) -- status: varchar (not null) - `, + schemaData: mockSchema, testcases: [], schemaIssues: [], } diff --git a/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/index.ts b/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/index.ts index 3ccf951726..438b331eb9 100644 --- a/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/index.ts +++ b/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/index.ts @@ -1,6 +1,8 @@ import { END, START, StateGraph } from '@langchain/langgraph' import { RETRY_POLICY } from '../../utils/errorHandling' +import { executeSingleTestNode } from './executeSingleTestNode' import { generateTestcaseNode } from './generateTestcaseNode' +import { invokeSingleTestToolNode } from './invokeSingleTestToolNode' import { routeAfterGenerate } from './routeAfterGenerate' import { routeAfterSave } from './routeAfterSave' import { saveToolNode } from './saveToolNode' @@ -20,6 +22,12 @@ graph .addNode('invokeSaveTool', saveToolNode, { retryPolicy: RETRY_POLICY, }) + .addNode('executeSingleTest', executeSingleTestNode, { + retryPolicy: RETRY_POLICY, + }) + .addNode('invokeSingleTestTool', invokeSingleTestToolNode, { + retryPolicy: RETRY_POLICY, + }) .addEdge(START, 'validateSchemaRequirements') .addConditionalEdges('generateTestcase', routeAfterGenerate, { invokeSaveTool: 'invokeSaveTool', @@ -27,7 +35,10 @@ graph }) .addConditionalEdges('invokeSaveTool', routeAfterSave, { generateTestcase: 'generateTestcase', + executeSingleTest: 'executeSingleTest', [END]: END, }) + .addEdge('executeSingleTest', 'invokeSingleTestTool') + .addEdge('invokeSingleTestTool', END) export const testcaseGeneration = graph.compile() diff --git a/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/invokeSingleTestToolNode.ts b/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/invokeSingleTestToolNode.ts new file mode 100644 index 0000000000..bd48d3c507 --- /dev/null +++ b/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/invokeSingleTestToolNode.ts @@ -0,0 +1,27 @@ +import type { RunnableConfig } from '@langchain/core/runnables' +import { ToolNode } from '@langchain/langgraph/prebuilt' +import { runSingleTestTool } from '../../tools/runSingleTestTool' +import { generateDdlFromSchema } from '../../utils/generateDdl' +import type { testcaseAnnotation } from './testcaseAnnotation' + +const toolNode = new ToolNode([runSingleTestTool]) + +export const invokeSingleTestToolNode = async ( + state: typeof testcaseAnnotation.State, + config: RunnableConfig, +): Promise> => { + const ddlStatements = generateDdlFromSchema(state.schemaData) + const requiredExtensions = Object.keys(state.schemaData.extensions).sort() + + const enhancedConfig: RunnableConfig = { + ...config, + configurable: { + ...config.configurable, + testcases: state.testcases, + ddlStatements, + requiredExtensions, + }, + } + + return await toolNode.invoke(state, enhancedConfig) +} diff --git a/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/routeAfterSave.ts b/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/routeAfterSave.ts index ea37dabb67..87fca23e86 100644 --- a/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/routeAfterSave.ts +++ b/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/routeAfterSave.ts @@ -1,4 +1,4 @@ -import { END } from '@langchain/langgraph' +import type { END } from '@langchain/langgraph' import type { testcaseAnnotation } from './testcaseAnnotation' /** @@ -6,11 +6,11 @@ import type { testcaseAnnotation } from './testcaseAnnotation' */ export const routeAfterSave = ( state: typeof testcaseAnnotation.State, -): 'generateTestcase' | typeof END => { +): 'generateTestcase' | 'executeSingleTest' | typeof END => { const { testcases } = state if (testcases.length > 0) { - return END + return 'executeSingleTest' } return 'generateTestcase' diff --git a/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/testcaseAnnotation.ts b/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/testcaseAnnotation.ts index d31fdff226..634f16bab7 100644 --- a/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/testcaseAnnotation.ts +++ b/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/testcaseAnnotation.ts @@ -1,4 +1,5 @@ import { Annotation, MessagesAnnotation } from '@langchain/langgraph' +import type { Schema } from '@liam-hq/schema' import type { RequirementData } from '../distributeRequirements' import type { Testcase } from '../types' @@ -15,7 +16,7 @@ export const schemaIssuesAnnotation = Annotation>({ export const testcaseAnnotation = Annotation.Root({ ...MessagesAnnotation.spec, currentRequirement: Annotation, - schemaContext: Annotation, + schemaData: Annotation, testcases: Annotation({ reducer: (prev, next) => prev.concat(next), }), diff --git a/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/validateSchemaRequirementsNode.integration.test.ts b/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/validateSchemaRequirementsNode.integration.test.ts index ed35481bd4..f8b262df71 100644 --- a/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/validateSchemaRequirementsNode.integration.test.ts +++ b/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/validateSchemaRequirementsNode.integration.test.ts @@ -1,4 +1,5 @@ import { END } from '@langchain/langgraph' +import type { Schema } from '@liam-hq/schema' import { v4 as uuidv4 } from 'uuid' import { describe, expect, it } from 'vitest' import type { testcaseAnnotation } from './testcaseAnnotation' @@ -9,6 +10,77 @@ describe('validateSchemaRequirementsNode Integration', () => { // Arrange type TestcaseState = typeof testcaseAnnotation.State + const mockSchema: Schema = { + tables: { + users: { + name: 'users', + columns: { + id: { + name: 'id', + type: 'uuid', + notNull: true, + default: null, + check: null, + comment: null, + }, + email: { + name: 'email', + type: 'varchar', + notNull: true, + default: null, + check: null, + comment: null, + }, + }, + comment: null, + indexes: {}, + constraints: {}, + }, + tasks: { + name: 'tasks', + columns: { + id: { + name: 'id', + type: 'uuid', + notNull: true, + default: null, + check: null, + comment: null, + }, + user_id: { + name: 'user_id', + type: 'uuid', + notNull: true, + default: null, + check: null, + comment: null, + }, + title: { + name: 'title', + type: 'varchar', + notNull: true, + default: null, + check: null, + comment: null, + }, + status: { + name: 'status', + type: 'varchar', + notNull: true, + default: null, + check: null, + comment: null, + }, + }, + comment: null, + indexes: {}, + constraints: {}, + }, + }, + enums: {}, + extensions: {}, + } + const state: TestcaseState = { messages: [], currentRequirement: { @@ -19,17 +91,7 @@ describe('validateSchemaRequirementsNode Integration', () => { 'A task management system where users create projects and tasks', requirementId: uuidv4(), }, - schemaContext: ` -Table: users -- id: uuid (not null) -- email: varchar (not null) - -Table: tasks -- id: uuid (not null) -- user_id: uuid (not null) -- title: varchar (not null) -- status: varchar (not null) - `, + schemaData: mockSchema, testcases: [], schemaIssues: [], } @@ -46,6 +108,78 @@ Table: tasks // Arrange type TestcaseState = typeof testcaseAnnotation.State + // Limited schema - missing projects/clients tables and deadline/priority columns + const mockSchema: Schema = { + tables: { + users: { + name: 'users', + columns: { + id: { + name: 'id', + type: 'uuid', + notNull: true, + default: null, + check: null, + comment: null, + }, + email: { + name: 'email', + type: 'varchar', + notNull: true, + default: null, + check: null, + comment: null, + }, + }, + comment: null, + indexes: {}, + constraints: {}, + }, + tasks: { + name: 'tasks', + columns: { + id: { + name: 'id', + type: 'uuid', + notNull: true, + default: null, + check: null, + comment: null, + }, + user_id: { + name: 'user_id', + type: 'uuid', + notNull: true, + default: null, + check: null, + comment: null, + }, + title: { + name: 'title', + type: 'varchar', + notNull: true, + default: null, + check: null, + comment: null, + }, + status: { + name: 'status', + type: 'varchar', + notNull: true, + default: null, + check: null, + comment: null, + }, + }, + comment: null, + indexes: {}, + constraints: {}, + }, + }, + enums: {}, + extensions: {}, + } + const state: TestcaseState = { messages: [], currentRequirement: { @@ -57,18 +191,7 @@ Table: tasks 'A comprehensive project management system where users manage multiple client projects with detailed task assignment', requirementId: uuidv4(), }, - // Limited schema - missing projects/clients tables and deadline/priority columns - schemaContext: ` -Table: users -- id: uuid (not null) -- email: varchar (not null) - -Table: tasks -- id: uuid (not null) -- user_id: uuid (not null) -- title: varchar (not null) -- status: varchar (not null) - `, + schemaData: mockSchema, testcases: [], schemaIssues: [], } diff --git a/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/validateSchemaRequirementsNode.ts b/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/validateSchemaRequirementsNode.ts index 8465023c7f..cdc68cfc75 100644 --- a/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/validateSchemaRequirementsNode.ts +++ b/frontend/internal-packages/agent/src/qa-agent/testcaseGeneration/validateSchemaRequirementsNode.ts @@ -3,6 +3,7 @@ import { Command, END } from '@langchain/langgraph' import { ChatOpenAI } from '@langchain/openai' import { fromPromise } from '@liam-hq/neverthrow' import * as v from 'valibot' +import { convertSchemaToText } from '../../utils/convertSchemaToText' import { toJsonSchema } from '../../utils/jsonSchema' import type { testcaseAnnotation } from './testcaseAnnotation' @@ -44,7 +45,9 @@ Be decisive and focus on what is missing, not how to design it. Do not suggest s export async function validateSchemaRequirementsNode( state: typeof testcaseAnnotation.State, ): Promise { - const { currentRequirement, schemaContext } = state + const { currentRequirement, schemaData } = state + + const schemaContext = convertSchemaToText(schemaData) const contextMessage = ` # Database Schema Context diff --git a/frontend/internal-packages/agent/src/tools/runSingleTestTool.ts b/frontend/internal-packages/agent/src/tools/runSingleTestTool.ts new file mode 100644 index 0000000000..e7ea91da23 --- /dev/null +++ b/frontend/internal-packages/agent/src/tools/runSingleTestTool.ts @@ -0,0 +1,130 @@ +import { dispatchCustomEvent } from '@langchain/core/callbacks/dispatch' +import { ToolMessage } from '@langchain/core/messages' +import type { RunnableConfig } from '@langchain/core/runnables' +import type { StructuredTool } from '@langchain/core/tools' +import { tool } from '@langchain/core/tools' +import { Command } from '@langchain/langgraph' +import { v4 as uuidv4 } from 'uuid' +import * as v from 'valibot' +import type { Testcase } from '../qa-agent/types' +import { SSE_EVENTS } from '../streaming/constants' +import { WorkflowTerminationError } from '../utils/errorHandling' +import { executeTestcase } from '../utils/executeTestcase' +import { getToolConfigurable } from './getToolConfigurable' + +const toolSchema = v.object({ + testcaseId: v.optional(v.string()), +}) + +export const runSingleTestTool: StructuredTool = tool( + async (input, config: RunnableConfig): Promise => { + const parsedInput = v.parse(toolSchema, input) + const toolConfigurableResult = getToolConfigurable(config) + if (toolConfigurableResult.isErr()) { + throw new WorkflowTerminationError( + toolConfigurableResult.error, + 'runSingleTestTool', + ) + } + + const { testcases, ddlStatements, requiredExtensions, toolCallId } = + toolConfigurableResult.value + + if (testcases.length === 0) { + const toolMessage = new ToolMessage({ + id: uuidv4(), + content: 'No test cases to execute.', + tool_call_id: toolCallId, + }) + await dispatchCustomEvent(SSE_EVENTS.MESSAGES, toolMessage) + + return new Command({ + update: { + messages: [toolMessage], + }, + }) + } + + // Select testcase - use specific ID if provided, otherwise use first testcase + const targetTestcase = parsedInput.testcaseId + ? testcases.find((tc) => tc.id === parsedInput.testcaseId) + : testcases[0] + + if (!targetTestcase) { + const errorMsg = parsedInput.testcaseId + ? `Test case with ID "${parsedInput.testcaseId}" not found.` + : 'No test case available to execute.' + + const toolMessage = new ToolMessage({ + id: uuidv4(), + content: errorMsg, + tool_call_id: toolCallId, + }) + await dispatchCustomEvent(SSE_EVENTS.MESSAGES, toolMessage) + + return new Command({ + update: { + messages: [toolMessage], + }, + }) + } + + // Execute single test case + const testResult = await executeTestcase( + ddlStatements, + targetTestcase, + requiredExtensions, + ) + + // Update testcase with execution result + const executionLog = { + executed_at: testResult.executedAt.toISOString(), + success: testResult.success, + result_summary: testResult.success + ? `Test Case "${testResult.testCaseTitle}" completed successfully` + : `Test Case "${testResult.testCaseTitle}" failed: ${testResult.failedOperation?.error ?? 'Unknown error'}`, + } + + const updatedDmlOperation = { + ...targetTestcase.dmlOperation, + dml_execution_logs: [executionLog], + } + + const updatedTestcase: Testcase = { + ...targetTestcase, + dmlOperation: updatedDmlOperation, + } + + // Update testcases array + const updatedTestcases = testcases.map((tc) => + tc.id === targetTestcase.id ? updatedTestcase : tc, + ) + + // Create tool success message + const summary = testResult.success + ? `Test case "${testResult.testCaseTitle}" passed successfully` + : `Test case "${testResult.testCaseTitle}" failed: ${testResult.failedOperation?.error ?? 'Unknown error'}` + + const toolMessage = new ToolMessage({ + id: uuidv4(), + content: summary, + tool_call_id: toolCallId, + }) + await dispatchCustomEvent(SSE_EVENTS.MESSAGES, toolMessage) + + const updateData = { + testcases: updatedTestcases, + messages: [toolMessage], + } + + return new Command({ + update: updateData, + }) + }, + { + name: 'runSingleTestTool', + description: + 'Execute a single test case with its DML operation to validate database schema. Runs DDL setup followed by single test case execution.', + schema: toolSchema, + }, +) diff --git a/frontend/internal-packages/agent/src/tools/runTestTool.ts b/frontend/internal-packages/agent/src/tools/runTestTool.ts index 7e83ece3a5..ae584125b2 100644 --- a/frontend/internal-packages/agent/src/tools/runTestTool.ts +++ b/frontend/internal-packages/agent/src/tools/runTestTool.ts @@ -5,7 +5,6 @@ import type { StructuredTool } from '@langchain/core/tools' import { tool } from '@langchain/core/tools' import { Command } from '@langchain/langgraph' import type { DmlOperation } from '@liam-hq/artifact' -import { executeQuery } from '@liam-hq/pglite-server' import type { SqlResult } from '@liam-hq/pglite-server/src/types' import { v4 as uuidv4 } from 'uuid' import * as v from 'valibot' @@ -14,54 +13,10 @@ import { formatValidationErrors } from '../qa-agent/validateSchema/formatValidat import type { TestcaseDmlExecutionResult } from '../qa-agent/validateSchema/types' import { SSE_EVENTS } from '../streaming/constants' import { WorkflowTerminationError } from '../utils/errorHandling' +import { executeTestcase } from '../utils/executeTestcase' import { getToolConfigurable } from './getToolConfigurable' import { transformStateToArtifact } from './transformStateToArtifact' -function isErrorResult(value: unknown): value is { error: unknown } { - return typeof value === 'object' && value !== null && 'error' in value -} - -/** - * Build combined SQL for DDL and testcase DML - */ -function buildCombinedSql(ddlStatements: string, testcase: Testcase): string { - const sqlParts = [] - - if (ddlStatements.trim()) { - sqlParts.push('-- DDL Statements', ddlStatements, '') - } - - const op: DmlOperation = testcase.dmlOperation - const header = op.description - ? `-- ${op.description}` - : `-- ${op.operation_type} operation` - sqlParts.push( - `-- Test Case: ${testcase.id}`, - `-- ${testcase.title}`, - `${header}\n${op.sql};`, - ) - - return sqlParts.filter(Boolean).join('\n') -} - -/** - * Extract failed operation from SQL results - */ -function extractFailedOperation( - sqlResults: SqlResult[], -): { sql: string; error: string } | undefined { - const firstFailed = sqlResults.find((r) => !r.success) - if (!firstFailed) { - return undefined - } - - const error = isErrorResult(firstFailed.result) - ? String(firstFailed.result.error) - : String(firstFailed.result) - - return { sql: firstFailed.sql, error } -} - /** * Execute DML operations by testcase with DDL statements * Combines DDL and testcase-specific DML into single execution units @@ -71,39 +26,11 @@ async function executeDmlOperationsByTestcase( testcases: Testcase[], requiredExtensions: string[], ): Promise { - const results: TestcaseDmlExecutionResult[] = [] - - for (const testcase of testcases) { - const combinedSql = buildCombinedSql(ddlStatements, testcase) - const startTime = new Date() - - const sqlResults = await executeQuery(combinedSql, requiredExtensions) - const hasErrors = sqlResults.some((result) => !result.success) - const failedOperation = hasErrors - ? extractFailedOperation(sqlResults) - : undefined - - const baseResult = { - testCaseId: testcase.id, - testCaseTitle: testcase.title, - executedAt: startTime, - } - - if (hasErrors && failedOperation) { - results.push({ - ...baseResult, - success: false, - failedOperation, - }) - } else { - results.push({ - ...baseResult, - success: true, - }) - } - } - - return results + return Promise.all( + testcases.map((testcase) => + executeTestcase(ddlStatements, testcase, requiredExtensions), + ), + ) } const toolSchema = v.object({}) diff --git a/frontend/internal-packages/agent/src/utils/executeTestcase.ts b/frontend/internal-packages/agent/src/utils/executeTestcase.ts new file mode 100644 index 0000000000..993dea806f --- /dev/null +++ b/frontend/internal-packages/agent/src/utils/executeTestcase.ts @@ -0,0 +1,87 @@ +import type { DmlOperation } from '@liam-hq/artifact' +import { executeQuery } from '@liam-hq/pglite-server' +import type { SqlResult } from '@liam-hq/pglite-server/src/types' +import type { Testcase } from '../qa-agent/types' +import type { TestcaseDmlExecutionResult } from '../qa-agent/validateSchema/types' + +function isErrorResult(value: unknown): value is { error: unknown } { + return typeof value === 'object' && value !== null && 'error' in value +} + +/** + * Build combined SQL for DDL and testcase DML + */ +function buildCombinedSql(ddlStatements: string, testcase: Testcase): string { + const sqlParts = [] + + if (ddlStatements.trim()) { + sqlParts.push('-- DDL Statements', ddlStatements, '') + } + + const op: DmlOperation = testcase.dmlOperation + const header = op.description + ? `-- ${op.description}` + : `-- ${op.operation_type} operation` + sqlParts.push( + `-- Test Case: ${testcase.id}`, + `-- ${testcase.title}`, + `${header}\n${op.sql};`, + ) + + return sqlParts.filter(Boolean).join('\n') +} + +/** + * Extract failed operation from SQL results + */ +function extractFailedOperation( + sqlResults: SqlResult[], +): { sql: string; error: string } | undefined { + const firstFailed = sqlResults.find((r) => !r.success) + if (!firstFailed) { + return undefined + } + + const error = isErrorResult(firstFailed.result) + ? String(firstFailed.result.error) + : String(firstFailed.result) + + return { sql: firstFailed.sql, error } +} + +/** + * Execute a single testcase with DDL statements + */ +export async function executeTestcase( + ddlStatements: string, + testcase: Testcase, + requiredExtensions: string[], +): Promise { + const combinedSql = buildCombinedSql(ddlStatements, testcase) + const startTime = new Date() + + const sqlResults = await executeQuery(combinedSql, requiredExtensions) + const hasErrors = sqlResults.some((result) => !result.success) + const failedOperation = hasErrors + ? extractFailedOperation(sqlResults) + : undefined + + const baseResult = { + testCaseId: testcase.id, + testCaseTitle: testcase.title, + executedAt: startTime, + } + + if (hasErrors && failedOperation) { + return { + ...baseResult, + success: false, + failedOperation, + } + } + + return { + ...baseResult, + success: true, + } +}