Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 25 additions & 7 deletions src/__tests__/assertion.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,13 +70,17 @@ function makeGenerateTextImpl(opts: {
return { output: opts.claude } as any;
}
if (model.includes("gemini-3-flash")) {
const g = typeof opts.gemini === "function" ? (opts.gemini as () => AssertionObj)() : opts.gemini;
const g =
typeof opts.gemini === "function" ? (opts.gemini as () => AssertionObj)() : opts.gemini;
return { output: g } as any;
}
if (model.includes("3.1-pro-preview")) {
return {
output:
opts.arbiter ?? { assertionPassed: false, confidenceScore: 0, reasoning: "no arbiter set" },
output: opts.arbiter ?? {
assertionPassed: false,
confidenceScore: 0,
reasoning: "no arbiter set",
},
} as any;
}
return { output: { assertionPassed: false, confidenceScore: 0, reasoning: "unknown" } } as any;
Expand Down Expand Up @@ -140,7 +144,11 @@ describe("assert consensus logic", () => {
makeGenerateTextImpl({
claude: { assertionPassed: true, confidenceScore: 95, reasoning: "Claude: yes" },
gemini: { assertionPassed: false, confidenceScore: 30, reasoning: "Gemini: no" },
arbiter: { assertionPassed: true, confidenceScore: 70, reasoning: "Arbiter: I side with Claude" },
arbiter: {
assertionPassed: true,
confidenceScore: 70,
reasoning: "Arbiter: I side with Claude",
},
}) as any,
);

Expand All @@ -163,7 +171,11 @@ describe("assert consensus logic", () => {
makeGenerateTextImpl({
claude: { assertionPassed: true, confidenceScore: 60, reasoning: "Claude: yes" },
gemini: { assertionPassed: false, confidenceScore: 40, reasoning: "Gemini: no" },
arbiter: { assertionPassed: false, confidenceScore: 45, reasoning: "Arbiter: I disagree, it fails" },
arbiter: {
assertionPassed: false,
confidenceScore: 45,
reasoning: "Arbiter: I disagree, it fails",
},
}) as any,
);

Expand Down Expand Up @@ -191,7 +203,11 @@ describe("assert consensus logic", () => {
if (geminiCalls === 1) {
throw new Error("transient model error");
}
return { assertionPassed: true, confidenceScore: 80, reasoning: "Gemini: ok after retry" };
return {
assertionPassed: true,
confidenceScore: 80,
reasoning: "Gemini: ok after retry",
};
},
}) as any,
);
Expand All @@ -213,7 +229,9 @@ describe("assert consensus logic", () => {
const page = createMockPage();

// Make withTimeout reject once to simulate timeout
vi.mocked(withTimeout).mockImplementationOnce(() => Promise.reject(new Error("timed out")) as any);
vi.mocked(withTimeout).mockImplementationOnce(
() => Promise.reject(new Error("timed out")) as any,
);

vi.mocked(generateText).mockImplementation(
makeGenerateTextImpl({
Expand Down
12 changes: 6 additions & 6 deletions src/__tests__/cua-config.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ describe("cua config", () => {
});

it("configure throws when user tries to override cua model", () => {
expect(() =>
configure({ ai: { models: { cua: "custom-cua-model" } } }),
).toThrow(/cua.*not user-configurable/);
expect(() => configure({ ai: { models: { cua: "custom-cua-model" } } })).toThrow(
/cua.*not user-configurable/,
);
// Default still wins.
expect(getModelId("cua")).toBe("gpt-5.5");
});
Expand Down Expand Up @@ -83,9 +83,9 @@ describe("resolveAI", () => {
});

it("throws when an override sets models.cua (lock applies per-layer)", () => {
expect(() =>
resolveAI({ models: { cua: "custom-cua" } }),
).toThrow(/cua.*not user-configurable/);
expect(() => resolveAI({ models: { cua: "custom-cua" } })).toThrow(
/cua.*not user-configurable/,
);
});

it("undefined override layers are ignored", () => {
Expand Down
5 changes: 1 addition & 4 deletions src/__tests__/integration/run-steps.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -328,10 +328,7 @@ describe("runSteps", () => {
it("call-level ai option applies to all steps without per-step override", async () => {
const page = createMockPage();

const steps: Step[] = [
{ description: "Step A" },
{ description: "Step B" },
];
const steps: Step[] = [{ description: "Step A" }, { description: "Step B" }];

await runSteps({
page,
Expand Down
73 changes: 37 additions & 36 deletions src/assertion.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,7 @@ import { assertVideoFile, deleteGeminiFile, uploadVideoToGemini } from "./video"

const assertionSchema = z.object({
assertionPassed: z.boolean().describe("Indicates whether the assertion passed or not."),
confidenceScore: z
.number()
.describe("Confidence score of the assertion, between 0 and 100."),
confidenceScore: z.number().describe("Confidence score of the assertion, between 0 and 100."),
reasoning: z
.string()
.describe(
Expand Down Expand Up @@ -57,7 +55,7 @@ export const assert = async ({
images,
failSilently,
maxRetries = 1,
onRetry = (retryCount: number, previousResult: AssertionResult) => { },
onRetry = (retryCount: number, previousResult: AssertionResult) => {},
video,
videoFilePath,
}: AssertionOptions): Promise<string> => {
Expand Down Expand Up @@ -106,31 +104,33 @@ export const assert = async ({
const imageContent = images
? images.map((image) => ({ type: "image" as const, image }))
: [
{
type: "image" as const,
image: (await resolvePage(page).screenshot({ fullPage: false })).toString("base64"),
},
];
{
type: "image" as const,
image: (await resolvePage(page).screenshot({ fullPage: false })).toString("base64"),
},
];

const basePrompt = `
You are an AI-powered QA Agent designed to test web applications.

You have access to the following information. Based on this information, you'll tell us whether the assertion provided below should pass or not.
${!images
? `
${
!images
? `
- An accessibility snapshot of the current page, which provides a detailed structure of the DOM
- A screenshot of the current page`
: "- Screenshots from various stages of the user flow"
}
: "- Screenshots from various stages of the user flow"
}

${!images
? `
${
!images
? `
<Snapshot>
${snapshot}
</Snapshot>
`
: ""
}
: ""
}

<Assertion>
${assertion}
Expand Down Expand Up @@ -176,13 +176,13 @@ Never hallucinate. Be truthful and if you are not sure, use a low confidence sco
temperature: 0,
providerOptions: thinkingEnabled
? {
anthropic: {
thinking: { type: "enabled", budgetTokens: THINKING_BUDGET_DEFAULT },
},
openrouter: {
reasoning: { max_tokens: THINKING_BUDGET_DEFAULT },
},
}
anthropic: {
thinking: { type: "enabled", budgetTokens: THINKING_BUDGET_DEFAULT },
},
openrouter: {
reasoning: { max_tokens: THINKING_BUDGET_DEFAULT },
},
}
: undefined,
messages,
});
Expand All @@ -205,15 +205,15 @@ Never hallucinate. Be truthful and if you are not sure, use a low confidence sco
temperature: 0,
providerOptions: thinkingEnabled
? {
google: {
thinkingConfig: {
thinkingBudget: THINKING_BUDGET_DEFAULT,
google: {
thinkingConfig: {
thinkingBudget: THINKING_BUDGET_DEFAULT,
},
},
},
openrouter: {
reasoning: { max_tokens: THINKING_BUDGET_DEFAULT },
},
}
openrouter: {
reasoning: { max_tokens: THINKING_BUDGET_DEFAULT },
},
}
: undefined,
messages,
output: Output.object({ schema: assertionSchema }),
Expand All @@ -240,14 +240,15 @@ Gemini's Assessment:
- Confidence: ${geminiResult.confidenceScore}%
- Reasoning: ${geminiResult.reasoning}

${!images
? `
${
!images
? `
<Snapshot>
${snapshot}
</Snapshot>
`
: ""
}
: ""
}

<Assertion>
${assertion}
Expand Down
2 changes: 1 addition & 1 deletion src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ export function configure(config: Config) {
if (config.ai?.models?.cua !== undefined) {
throw new Error(
`[passmark] ai.models.cua is not user-configurable — CUA mode is locked to "${DEFAULT_MODELS.cua}". ` +
`Remove the "cua" field from configure({ ai: { models } }).`,
`Remove the "cua" field from configure({ ai: { models } }).`,
);
}
globalConfig = { ...globalConfig, ...config };
Expand Down
12 changes: 7 additions & 5 deletions src/cua/loop.ts
Original file line number Diff line number Diff line change
Expand Up @@ -114,16 +114,16 @@ export async function runCUALoop({
const e = err as OpenAIErrorLike;
logger.error(
`[cua] initial request failed: status=${e?.status ?? "?"} msg=${e?.message ?? err} ` +
`model=${model} tool=${JSON.stringify(tool)} ` +
`body=${JSON.stringify(e?.error ?? e?.response?.data ?? e?.body ?? {})}`,
`model=${model} tool=${JSON.stringify(tool)} ` +
`body=${JSON.stringify(e?.error ?? e?.response?.data ?? e?.body ?? {})}`,
);
// A generic 400 with no `param` usually means the account lacks access to
// the CUA model or to the built-in `computer` tool on the Responses API.
if (e?.status === 400) {
logger.error(
`[cua] if no "param" detail is shown above, verify your OpenAI API key has access to "${model}" ` +
`and the built-in "computer" tool on the Responses API ` +
`(https://platform.openai.com/settings/organization/limits).`,
`and the built-in "computer" tool on the Responses API ` +
`(https://platform.openai.com/settings/organization/limits).`,
);
}
throw err;
Expand Down Expand Up @@ -248,7 +248,9 @@ function isAddressBarFocus(action: ComputerAction | undefined): boolean {
if (!action || action.type !== "keypress") return false;
const keys = ((action as { keys?: string[] }).keys ?? []).map((k) => k.toUpperCase());
if (keys.length !== 2 || !keys.includes("L")) return false;
return keys.some((k) => k === "CTRL" || k === "CONTROL" || k === "META" || k === "CMD" || k === "COMMAND");
return keys.some(
(k) => k === "CTRL" || k === "CONTROL" || k === "META" || k === "CMD" || k === "COMMAND",
);
}

function isUrlType(action: ComputerAction | undefined): boolean {
Expand Down
45 changes: 25 additions & 20 deletions src/cua/prompts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,28 +35,31 @@ ${step.description}
Current Step Index: ${stepIndex + 1} out of ${steps.length} steps.
</StepIndex>

${stepIndex + 1 < steps.length
? `<NextStep>
${
stepIndex + 1 < steps.length
? `<NextStep>
(For context only — DO NOT execute.) Next step: "${steps[stepIndex + 1].description}"
</NextStep>`
: ""
}
: ""
}

${step.data
? `<Data>
${
step.data
? `<Data>
Use this data for the current step: ${JSON.stringify(step.data)}
</Data>`
: ""
}
: ""
}

${auth
? `<Auth>
${
auth
? `<Auth>
If a login screen appears, use:
- Email: ${auth.email}
- Password: ${auth.password}
</Auth>`
: ""
}
: ""
}

<Instructions>
- Look at the current screenshot before acting. If the page is still loading, use the wait action.
Expand Down Expand Up @@ -85,26 +88,28 @@ You are an expert QA agent testing a web application using computer-use capabili
${userFlow}
</UserFlow>

${steps
? `<Steps>
${
steps
? `<Steps>
Follow these steps in order:
${steps}
Stop once all steps are complete.
</Steps>`
: ""
}
: ""
}

${assertion
? `<Assertion>
${
assertion
? `<Assertion>
${assertion}
</Assertion>

When the flow is complete, evaluate the assertion and report:
- assertionPassed: boolean
- confidenceScore: 0-100
- reasoning: short explanation`
: ""
}
: ""
}

<Instructions>
- Inspect each screenshot before acting.
Expand Down
6 changes: 3 additions & 3 deletions src/data-cache.ts
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,7 @@ export function replacePlaceholders(
for (const placeholder of dynamicEmailPlaceholders) {
if (result.includes(placeholder) && !getConfig().email) {
throw new ConfigurationError(
`Email provider not configured. Call configure({ email: ... }) before using ${placeholder}.`,
`Email provider not configured. Call configure({ email: ... }) before using ${placeholder}.`,
);
}
}
Expand Down Expand Up @@ -409,7 +409,7 @@ export async function processPlaceholders(
if (hasGlobalPlaceholders && !executionId) {
throw new ValidationError(
"{{global.*}} placeholders require an executionId. " +
"Please provide executionId in runSteps options to use global placeholders.",
"Please provide executionId in runSteps options to use global placeholders.",
);
}

Expand All @@ -419,7 +419,7 @@ export async function processPlaceholders(
if (hasProjectDataPlaceholders && !projectId) {
throw new ValidationError(
"{{data.*}} placeholders require a projectId. " +
"Please provide projectId in runSteps options to use project data placeholders.",
"Please provide projectId in runSteps options to use project data placeholders.",
);
}

Expand Down
Loading