diff --git a/src/__tests__/assertion.test.ts b/src/__tests__/assertion.test.ts index 636b1e8..760fee2 100644 --- a/src/__tests__/assertion.test.ts +++ b/src/__tests__/assertion.test.ts @@ -70,13 +70,17 @@ function makeGenerateTextImpl(opts: { return { output: opts.claude } as any; } if (model.includes("gemini-3-flash")) { - const g = typeof opts.gemini === "function" ? (opts.gemini as () => AssertionObj)() : opts.gemini; + const g = + typeof opts.gemini === "function" ? (opts.gemini as () => AssertionObj)() : opts.gemini; return { output: g } as any; } if (model.includes("3.1-pro-preview")) { return { - output: - opts.arbiter ?? { assertionPassed: false, confidenceScore: 0, reasoning: "no arbiter set" }, + output: opts.arbiter ?? { + assertionPassed: false, + confidenceScore: 0, + reasoning: "no arbiter set", + }, } as any; } return { output: { assertionPassed: false, confidenceScore: 0, reasoning: "unknown" } } as any; @@ -140,7 +144,11 @@ describe("assert consensus logic", () => { makeGenerateTextImpl({ claude: { assertionPassed: true, confidenceScore: 95, reasoning: "Claude: yes" }, gemini: { assertionPassed: false, confidenceScore: 30, reasoning: "Gemini: no" }, - arbiter: { assertionPassed: true, confidenceScore: 70, reasoning: "Arbiter: I side with Claude" }, + arbiter: { + assertionPassed: true, + confidenceScore: 70, + reasoning: "Arbiter: I side with Claude", + }, }) as any, ); @@ -163,7 +171,11 @@ describe("assert consensus logic", () => { makeGenerateTextImpl({ claude: { assertionPassed: true, confidenceScore: 60, reasoning: "Claude: yes" }, gemini: { assertionPassed: false, confidenceScore: 40, reasoning: "Gemini: no" }, - arbiter: { assertionPassed: false, confidenceScore: 45, reasoning: "Arbiter: I disagree, it fails" }, + arbiter: { + assertionPassed: false, + confidenceScore: 45, + reasoning: "Arbiter: I disagree, it fails", + }, }) as any, ); @@ -191,7 +203,11 @@ describe("assert consensus logic", () => { if (geminiCalls === 1) { throw new Error("transient model error"); } - return { assertionPassed: true, confidenceScore: 80, reasoning: "Gemini: ok after retry" }; + return { + assertionPassed: true, + confidenceScore: 80, + reasoning: "Gemini: ok after retry", + }; }, }) as any, ); @@ -213,7 +229,9 @@ describe("assert consensus logic", () => { const page = createMockPage(); // Make withTimeout reject once to simulate timeout - vi.mocked(withTimeout).mockImplementationOnce(() => Promise.reject(new Error("timed out")) as any); + vi.mocked(withTimeout).mockImplementationOnce( + () => Promise.reject(new Error("timed out")) as any, + ); vi.mocked(generateText).mockImplementation( makeGenerateTextImpl({ diff --git a/src/__tests__/cua-config.test.ts b/src/__tests__/cua-config.test.ts index 83027f1..247317b 100644 --- a/src/__tests__/cua-config.test.ts +++ b/src/__tests__/cua-config.test.ts @@ -27,9 +27,9 @@ describe("cua config", () => { }); it("configure throws when user tries to override cua model", () => { - expect(() => - configure({ ai: { models: { cua: "custom-cua-model" } } }), - ).toThrow(/cua.*not user-configurable/); + expect(() => configure({ ai: { models: { cua: "custom-cua-model" } } })).toThrow( + /cua.*not user-configurable/, + ); // Default still wins. expect(getModelId("cua")).toBe("gpt-5.5"); }); @@ -83,9 +83,9 @@ describe("resolveAI", () => { }); it("throws when an override sets models.cua (lock applies per-layer)", () => { - expect(() => - resolveAI({ models: { cua: "custom-cua" } }), - ).toThrow(/cua.*not user-configurable/); + expect(() => resolveAI({ models: { cua: "custom-cua" } })).toThrow( + /cua.*not user-configurable/, + ); }); it("undefined override layers are ignored", () => { diff --git a/src/__tests__/integration/run-steps.test.ts b/src/__tests__/integration/run-steps.test.ts index 5dd590b..dd29026 100644 --- a/src/__tests__/integration/run-steps.test.ts +++ b/src/__tests__/integration/run-steps.test.ts @@ -328,10 +328,7 @@ describe("runSteps", () => { it("call-level ai option applies to all steps without per-step override", async () => { const page = createMockPage(); - const steps: Step[] = [ - { description: "Step A" }, - { description: "Step B" }, - ]; + const steps: Step[] = [{ description: "Step A" }, { description: "Step B" }]; await runSteps({ page, diff --git a/src/assertion.ts b/src/assertion.ts index 0173b4a..b7f0d29 100644 --- a/src/assertion.ts +++ b/src/assertion.ts @@ -10,9 +10,7 @@ import { assertVideoFile, deleteGeminiFile, uploadVideoToGemini } from "./video" const assertionSchema = z.object({ assertionPassed: z.boolean().describe("Indicates whether the assertion passed or not."), - confidenceScore: z - .number() - .describe("Confidence score of the assertion, between 0 and 100."), + confidenceScore: z.number().describe("Confidence score of the assertion, between 0 and 100."), reasoning: z .string() .describe( @@ -57,7 +55,7 @@ export const assert = async ({ images, failSilently, maxRetries = 1, - onRetry = (retryCount: number, previousResult: AssertionResult) => { }, + onRetry = (retryCount: number, previousResult: AssertionResult) => {}, video, videoFilePath, }: AssertionOptions): Promise => { @@ -106,31 +104,33 @@ export const assert = async ({ const imageContent = images ? images.map((image) => ({ type: "image" as const, image })) : [ - { - type: "image" as const, - image: (await resolvePage(page).screenshot({ fullPage: false })).toString("base64"), - }, - ]; + { + type: "image" as const, + image: (await resolvePage(page).screenshot({ fullPage: false })).toString("base64"), + }, + ]; const basePrompt = ` You are an AI-powered QA Agent designed to test web applications. You have access to the following information. Based on this information, you'll tell us whether the assertion provided below should pass or not. -${!images - ? ` +${ + !images + ? ` - An accessibility snapshot of the current page, which provides a detailed structure of the DOM - A screenshot of the current page` - : "- Screenshots from various stages of the user flow" - } + : "- Screenshots from various stages of the user flow" +} -${!images - ? ` +${ + !images + ? ` ${snapshot} ` - : "" - } + : "" +} ${assertion} @@ -176,13 +176,13 @@ Never hallucinate. Be truthful and if you are not sure, use a low confidence sco temperature: 0, providerOptions: thinkingEnabled ? { - anthropic: { - thinking: { type: "enabled", budgetTokens: THINKING_BUDGET_DEFAULT }, - }, - openrouter: { - reasoning: { max_tokens: THINKING_BUDGET_DEFAULT }, - }, - } + anthropic: { + thinking: { type: "enabled", budgetTokens: THINKING_BUDGET_DEFAULT }, + }, + openrouter: { + reasoning: { max_tokens: THINKING_BUDGET_DEFAULT }, + }, + } : undefined, messages, }); @@ -205,15 +205,15 @@ Never hallucinate. Be truthful and if you are not sure, use a low confidence sco temperature: 0, providerOptions: thinkingEnabled ? { - google: { - thinkingConfig: { - thinkingBudget: THINKING_BUDGET_DEFAULT, + google: { + thinkingConfig: { + thinkingBudget: THINKING_BUDGET_DEFAULT, + }, }, - }, - openrouter: { - reasoning: { max_tokens: THINKING_BUDGET_DEFAULT }, - }, - } + openrouter: { + reasoning: { max_tokens: THINKING_BUDGET_DEFAULT }, + }, + } : undefined, messages, output: Output.object({ schema: assertionSchema }), @@ -240,14 +240,15 @@ Gemini's Assessment: - Confidence: ${geminiResult.confidenceScore}% - Reasoning: ${geminiResult.reasoning} -${!images - ? ` +${ + !images + ? ` ${snapshot} ` - : "" - } + : "" +} ${assertion} diff --git a/src/config.ts b/src/config.ts index dffa078..1ed9053 100644 --- a/src/config.ts +++ b/src/config.ts @@ -126,7 +126,7 @@ export function configure(config: Config) { if (config.ai?.models?.cua !== undefined) { throw new Error( `[passmark] ai.models.cua is not user-configurable — CUA mode is locked to "${DEFAULT_MODELS.cua}". ` + - `Remove the "cua" field from configure({ ai: { models } }).`, + `Remove the "cua" field from configure({ ai: { models } }).`, ); } globalConfig = { ...globalConfig, ...config }; diff --git a/src/cua/loop.ts b/src/cua/loop.ts index 463d2a2..f8c1fdd 100644 --- a/src/cua/loop.ts +++ b/src/cua/loop.ts @@ -114,16 +114,16 @@ export async function runCUALoop({ const e = err as OpenAIErrorLike; logger.error( `[cua] initial request failed: status=${e?.status ?? "?"} msg=${e?.message ?? err} ` + - `model=${model} tool=${JSON.stringify(tool)} ` + - `body=${JSON.stringify(e?.error ?? e?.response?.data ?? e?.body ?? {})}`, + `model=${model} tool=${JSON.stringify(tool)} ` + + `body=${JSON.stringify(e?.error ?? e?.response?.data ?? e?.body ?? {})}`, ); // A generic 400 with no `param` usually means the account lacks access to // the CUA model or to the built-in `computer` tool on the Responses API. if (e?.status === 400) { logger.error( `[cua] if no "param" detail is shown above, verify your OpenAI API key has access to "${model}" ` + - `and the built-in "computer" tool on the Responses API ` + - `(https://platform.openai.com/settings/organization/limits).`, + `and the built-in "computer" tool on the Responses API ` + + `(https://platform.openai.com/settings/organization/limits).`, ); } throw err; @@ -248,7 +248,9 @@ function isAddressBarFocus(action: ComputerAction | undefined): boolean { if (!action || action.type !== "keypress") return false; const keys = ((action as { keys?: string[] }).keys ?? []).map((k) => k.toUpperCase()); if (keys.length !== 2 || !keys.includes("L")) return false; - return keys.some((k) => k === "CTRL" || k === "CONTROL" || k === "META" || k === "CMD" || k === "COMMAND"); + return keys.some( + (k) => k === "CTRL" || k === "CONTROL" || k === "META" || k === "CMD" || k === "COMMAND", + ); } function isUrlType(action: ComputerAction | undefined): boolean { diff --git a/src/cua/prompts.ts b/src/cua/prompts.ts index a266b03..e0e15a7 100644 --- a/src/cua/prompts.ts +++ b/src/cua/prompts.ts @@ -35,28 +35,31 @@ ${step.description} Current Step Index: ${stepIndex + 1} out of ${steps.length} steps. -${stepIndex + 1 < steps.length - ? ` +${ + stepIndex + 1 < steps.length + ? ` (For context only — DO NOT execute.) Next step: "${steps[stepIndex + 1].description}" ` - : "" - } + : "" +} -${step.data - ? ` +${ + step.data + ? ` Use this data for the current step: ${JSON.stringify(step.data)} ` - : "" - } + : "" +} -${auth - ? ` +${ + auth + ? ` If a login screen appears, use: - Email: ${auth.email} - Password: ${auth.password} ` - : "" - } + : "" +} - Look at the current screenshot before acting. If the page is still loading, use the wait action. @@ -85,17 +88,19 @@ You are an expert QA agent testing a web application using computer-use capabili ${userFlow} -${steps - ? ` +${ + steps + ? ` Follow these steps in order: ${steps} Stop once all steps are complete. ` - : "" - } + : "" +} -${assertion - ? ` +${ + assertion + ? ` ${assertion} @@ -103,8 +108,8 @@ When the flow is complete, evaluate the assertion and report: - assertionPassed: boolean - confidenceScore: 0-100 - reasoning: short explanation` - : "" - } + : "" +} - Inspect each screenshot before acting. diff --git a/src/data-cache.ts b/src/data-cache.ts index 3295fa8..996b878 100644 --- a/src/data-cache.ts +++ b/src/data-cache.ts @@ -354,7 +354,7 @@ export function replacePlaceholders( for (const placeholder of dynamicEmailPlaceholders) { if (result.includes(placeholder) && !getConfig().email) { throw new ConfigurationError( - `Email provider not configured. Call configure({ email: ... }) before using ${placeholder}.`, + `Email provider not configured. Call configure({ email: ... }) before using ${placeholder}.`, ); } } @@ -409,7 +409,7 @@ export async function processPlaceholders( if (hasGlobalPlaceholders && !executionId) { throw new ValidationError( "{{global.*}} placeholders require an executionId. " + - "Please provide executionId in runSteps options to use global placeholders.", + "Please provide executionId in runSteps options to use global placeholders.", ); } @@ -419,7 +419,7 @@ export async function processPlaceholders( if (hasProjectDataPlaceholders && !projectId) { throw new ValidationError( "{{data.*}} placeholders require a projectId. " + - "Please provide projectId in runSteps options to use project data placeholders.", + "Please provide projectId in runSteps options to use project data placeholders.", ); } diff --git a/src/email.ts b/src/email.ts index ecec9a8..d018967 100644 --- a/src/email.ts +++ b/src/email.ts @@ -7,9 +7,8 @@ function getEmailProvider() { const provider = getConfig().email; if (!provider) { throw new ConfigurationError( - "Email provider not configured. Call configure({ email: ... }) before using email features.", + "Email provider not configured. Call configure({ email: ... }) before using email features.", ); - } return provider; } @@ -95,6 +94,6 @@ export async function extractEmailContent({ } throw new AIModelError( - `Failed to extract email content after ${maxRetries} attempts. Email: ${email}, Prompt: ${prompt}`, + `Failed to extract email content after ${maxRetries} attempts. Email: ${email}, Prompt: ${prompt}`, ); } diff --git a/src/errors.ts b/src/errors.ts index f10ced4..e96df52 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -89,4 +89,4 @@ export class ValidationError extends PassmarkError { constructor(message: string) { super(message, "VALIDATION_ERROR"); } -} \ No newline at end of file +} diff --git a/src/index.ts b/src/index.ts index 5a95521..3d6f7c5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -131,7 +131,8 @@ export const runSteps = async ({ const isPlaywrightRetry = test ? test.info().retry > 0 : false; if (isPlaywrightRetry) { logger.debug( - `Playwright retry detected (retry #${test!.info().retry + `Playwright retry detected (retry #${ + test!.info().retry }). Bypassing cache and using AI only.`, ); } @@ -303,25 +304,21 @@ export const runSteps = async ({ } try { - await maybeWithSpan( - { capability: "step_execution", step: "cua_loop" }, - () => - runCUALoop({ - page: tabManager.active(), - instruction: buildRunStepsPromptCUA({ - auth, - steps: processedSteps, - step, - userFlow, - stepIndex: i, - }), - maxSteps: STEP_EXECUTION_MAX_STEPS, - abortSignal: AbortSignal.timeout(STEP_EXECUTION_TIMEOUT), - onReasoning: onReasoning - ? (reasoning) => onReasoning({ id, reasoning }) - : undefined, - gateway: effectiveAi.gateway, + await maybeWithSpan({ capability: "step_execution", step: "cua_loop" }, () => + runCUALoop({ + page: tabManager.active(), + instruction: buildRunStepsPromptCUA({ + auth, + steps: processedSteps, + step, + userFlow, + stepIndex: i, }), + maxSteps: STEP_EXECUTION_MAX_STEPS, + abortSignal: AbortSignal.timeout(STEP_EXECUTION_TIMEOUT), + onReasoning: onReasoning ? (reasoning) => onReasoning({ id, reasoning }) : undefined, + gateway: effectiveAi.gateway, + }), ); } catch (error: unknown) { logger.error({ err: error }, `CUA step execution failed: ${step.description}`); @@ -478,9 +475,9 @@ export const runSteps = async ({ let pageScreenshotBeforeApplyingAction: string = ""; if (step.waitUntil) { - pageScreenshotBeforeApplyingAction = (await tabManager.active().screenshot({ fullPage: false })).toString( - "base64", - ); + pageScreenshotBeforeApplyingAction = ( + await tabManager.active().screenshot({ fullPage: false }) + ).toString("base64"); } const stepModelId = effectiveAi.getModelId("stepExecution"); @@ -508,7 +505,7 @@ export const runSteps = async ({ openrouter: { reasoning: { effort: "medium", - exclude: true + exclude: true, }, }, }, @@ -742,7 +739,9 @@ export const runUserFlow = async ({ prompt: `Convert the following text output into a valid JSON object with the specified properties:\n\n${text}`, output: Output.object({ schema: z.object({ - assertionPassed: z.boolean().describe("Indicates whether the assertion passed or not."), + assertionPassed: z + .boolean() + .describe("Indicates whether the assertion passed or not."), confidenceScore: z .number() .describe("Confidence score of the assertion, between 0 and 100."), @@ -890,4 +889,11 @@ export { extractEmailContent, generateEmail } from "./email"; export { assert } from "./assertion"; export type { AssertionResult } from "./types"; -export { PassmarkError, StepExecutionError, ValidationError, AIModelError, CacheError, ConfigurationError } from "./errors"; +export { + PassmarkError, + StepExecutionError, + ValidationError, + AIModelError, + CacheError, + ConfigurationError, +} from "./errors"; diff --git a/src/prompts/index.ts b/src/prompts/index.ts index b2643b2..bdc43ed 100644 --- a/src/prompts/index.ts +++ b/src/prompts/index.ts @@ -30,34 +30,37 @@ export const buildRunStepsPrompt = ({ Current Step Index: ${stepIndex + 1} out of ${steps.length} steps. - ${stepIndex + 1 < steps.length - ? ` + ${ + stepIndex + 1 < steps.length + ? ` The next step (DO NOT EXECUTE THIS) is: "${steps[stepIndex + 1].description}" This is provided for context only. Stop immediately after completing the current step given above. ` - : "" + : "" } Remember we're only interested in executing the current step right now. We'll have a separate run for the next step. So, do not execute any steps other than the current step mentioned above. Stop right after executing the current step. - ${step.data - ? ` + ${ + step.data + ? ` Use the following data for the current step: "${JSON.stringify(step.data)}". `.trim() - : "" + : "" } - ${auth - ? ` + ${ + auth + ? ` If presented with login screen, log in to the website using the following credentials: - Email: ${auth.email} - Password: ${auth.password} ` - : "" + : "" } @@ -93,23 +96,26 @@ export const buildRunUserFlowPrompt = ({ ${userFlow} - ${steps - ? `Follow these steps **exactly** to test the user flow:\n\n\n${steps}\n- STOP user flow by calling \`browser_stop\` tool exactly once.\n` - : "" + ${ + steps + ? `Follow these steps **exactly** to test the user flow:\n\n\n${steps}\n- STOP user flow by calling \`browser_stop\` tool exactly once.\n` + : "" } - ${assertion - ? `\n\n${assertion}\n\n\n\n Double check your assertion analysis to ensure it's accurate.` - : "" + ${ + assertion + ? `\n\n${assertion}\n\n\n\n Double check your assertion analysis to ensure it's accurate.` + : "" } - ${assertion - ? ` + ${ + assertion + ? ` The output should contain the following information: - \`assertionPassed\`: A boolean indicating whether the assertion passed or not. - \`confidenceScore\`: A number between 0 and 100 indicating the confidence score of the assertion. - \`reasoning\`: A brief string explaining the reasoning behind the assertion. ` - : "" + : "" } Follow these instructions carefully while testing the website: diff --git a/src/tools.ts b/src/tools.ts index df205e1..eb24cef 100644 --- a/src/tools.ts +++ b/src/tools.ts @@ -96,9 +96,7 @@ export function getAItools(page: Page, settings?: ToolSettings) { const base64 = (typeof result === "string" ? result : result.output) as string; return { type: "content", - value: [ - { type: "media", data: base64, mediaType: "image/png" }, - ], + value: [{ type: "media", data: base64, mediaType: "image/png" }], }; }, }), diff --git a/src/types.ts b/src/types.ts index 7f9d546..4e0dc80 100644 --- a/src/types.ts +++ b/src/types.ts @@ -148,9 +148,9 @@ export type RunStepsOptions = { */ ai?: AIOverride; } & ( - | { + | { assertions: Omit[]; expect: Expect<{}>; } - | { assertions?: never; expect?: never } - ); + | { assertions?: never; expect?: never } +); diff --git a/src/utils/index.ts b/src/utils/index.ts index a637ec1..8493f87 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -12,11 +12,7 @@ import { z } from "zod"; import { getModelId } from "../config"; import { logger } from "../logger"; import { resolveModel } from "../models"; -import { - PageInput, - WaitConditionResult, - WaitForConditionOptions, -} from "../types"; +import { PageInput, WaitConditionResult, WaitForConditionOptions } from "../types"; import type { TabManager } from "./tab-manager"; /** @@ -66,7 +62,7 @@ export const withTimeout = ( export const safeSnapshot = async (input: PageInput, timeout = SNAPSHOT_TIMEOUT) => { const attempt = async () => { return await resolvePage(input).ariaSnapshot({ mode: "ai", timeout }); - } + }; try { const snapshot = await attempt(); @@ -191,7 +187,9 @@ export async function waitForDOMStabilization( (error instanceof Error && error.message?.includes("navigation")) ) { // Navigation occurred - wait for the page to be ready - await resolvePage(input).waitForLoadState("domcontentloaded").catch(() => { }); + await resolvePage(input) + .waitForLoadState("domcontentloaded") + .catch(() => {}); return; } // Re-throw other errors @@ -263,9 +261,9 @@ export async function waitForCondition({ let currentInterval = initialInterval; const checkCondition = async (): Promise => { - const pageScreenshotAfterApplyingAction = (await resolvePage(page).screenshot({ fullPage: false })).toString( - "base64", - ); + const pageScreenshotAfterApplyingAction = ( + await resolvePage(page).screenshot({ fullPage: false }) + ).toString("base64"); const prompt = ` You are an AI-powered QA Agent designed to test web applications. @@ -273,15 +271,16 @@ You are an AI-powered QA Agent designed to test web applications. You are helping to determine if a wait condition has been met during a test flow. -${previousSteps.length > 0 - ? `Previous steps completed:\n${previousSteps - .map( - (s, i) => - `${i + 1}. ${s.description}\n${s.data ? ` Data: ${JSON.stringify(s.data)}` : ""}`, - ) - .join("\n")}` - : "No previous steps." - } +${ + previousSteps.length > 0 + ? `Previous steps completed:\n${previousSteps + .map( + (s, i) => + `${i + 1}. ${s.description}\n${s.data ? ` Data: ${JSON.stringify(s.data)}` : ""}`, + ) + .join("\n")}` + : "No previous steps." +} Last executed step: ${currentStep.description} ${nextStep ? `Next step: ${nextStep.description}` : ""} diff --git a/src/utils/tab-manager.ts b/src/utils/tab-manager.ts index b219f71..5cddafc 100644 --- a/src/utils/tab-manager.ts +++ b/src/utils/tab-manager.ts @@ -43,9 +43,7 @@ export const createTabManager = (initialPage: Page): TabManager => { else if (target === "latest") idx = pages.length - 1; else idx = target; if (idx < 0 || idx >= pages.length) { - throw new Error( - `switchToTab: invalid target ${target}; ${pages.length} tab(s) open.`, - ); + throw new Error(`switchToTab: invalid target ${target}; ${pages.length} tab(s) open.`); } activeIndex = idx; return pages[idx]; diff --git a/src/video.ts b/src/video.ts index 517ea8e..fc2b558 100644 --- a/src/video.ts +++ b/src/video.ts @@ -124,10 +124,7 @@ export async function uploadVideoToGemini(filePath: string): Promise<{ file: filePath, config: { mimeType: VIDEO_MIME_TYPE }, }); - logger.debug( - { name: uploaded.name, state: uploaded.state }, - "Gemini Files API upload accepted", - ); + logger.debug({ name: uploaded.name, state: uploaded.state }, "Gemini Files API upload accepted"); if (!uploaded.name) { throw new AIModelError("Gemini Files API did not return a file name after upload."); @@ -224,10 +221,7 @@ Never hallucinate. If unsure, use a low confidence score. contents: [ { role: "user", - parts: [ - { fileData: { fileUri, mimeType: fileMimeType } }, - { text: prompt }, - ], + parts: [{ fileData: { fileUri, mimeType: fileMimeType } }, { text: prompt }], }, ], config: {