Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 63 additions & 1 deletion packages/warden/src/action/reporting/output.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { describe, expect, it } from 'vitest';
import type { EventContext, Finding, SkillReport } from '../../types/index.js';
import { buildFindingsOutput, FindingsOutputSchema } from './output.js';
import { buildConfiguredSkillsList, buildFindingsOutput, FindingsOutputSchema } from './output.js';

describe('findings output schema', () => {
it('builds a schema-valid public findings payload', () => {
Expand Down Expand Up @@ -227,6 +227,68 @@ describe('findings output schema', () => {
expect('failedExtractions' in serialized).toBe(false);
expect('error' in serialized).toBe(false);
});

it('includes the configured skills roster when provided', () => {
const output = buildFindingsOutput([createReport()], createContext(), [], {
timestamp: '2026-01-01T00:00:00.000Z',
runId: '123',
configuredSkills: [
{ name: 'test-skill', triggered: true },
{ name: 'idle-skill', triggered: false },
],
});

expect(FindingsOutputSchema.parse(output)).toEqual(output);
expect(output.configuredSkills).toEqual([
{ name: 'test-skill', triggered: true },
{ name: 'idle-skill', triggered: false },
]);
});

it('omits the configured skills roster when not provided', () => {
const output = buildFindingsOutput([createReport()], createContext(), [], {
timestamp: '2026-01-01T00:00:00.000Z',
runId: '123',
});

expect(output.configuredSkills).toBeUndefined();
});
});

describe('buildConfiguredSkillsList', () => {
it('marks matched skills as triggered and unmatched skills as not', () => {
const result = buildConfiguredSkillsList({
allTriggers: [{ name: 'matched-skill' }, { name: 'skipped-skill' }],
matchedTriggers: [{ name: 'matched-skill' }],
});

expect(result).toEqual([
{ name: 'matched-skill', triggered: true },
{ name: 'skipped-skill', triggered: false },
]);
});

it('deduplicates multiple trigger blocks for the same skill', () => {
const result = buildConfiguredSkillsList({
allTriggers: [{ name: 'multi-trigger-skill' }, { name: 'multi-trigger-skill' }],
matchedTriggers: [{ name: 'multi-trigger-skill' }],
});

expect(result).toEqual([{ name: 'multi-trigger-skill', triggered: true }]);
});

it('returns an empty list when nothing is configured', () => {
expect(buildConfiguredSkillsList({ allTriggers: [], matchedTriggers: [] })).toEqual([]);
});

it('includes a skill whose only trigger is neither matched nor a PR-check skip', () => {
const result = buildConfiguredSkillsList({
allTriggers: [{ name: 'nightly-sweep' }],
matchedTriggers: [],
});

expect(result).toEqual([{ name: 'nightly-sweep', triggered: false }]);
});
});

function createFinding(): Finding {
Expand Down
27 changes: 27 additions & 0 deletions packages/warden/src/action/reporting/output.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,10 @@ export const FindingsOutputSchema = z.object({
})),
triggerResults: z.array(TriggerRunResultSchema).optional(),
findingObservations: z.array(FindingObservationSchema),
configuredSkills: z.array(z.object({
name: z.string(),
triggered: z.boolean(),
})).optional(),
});

export type FindingsOutput = z.infer<typeof FindingsOutputSchema>;
Expand All @@ -116,6 +120,28 @@ interface BuildFindingsOutputOptions {
timestamp?: string;
runId?: string;
triggerResults?: ReplayTriggerResult[];
configuredSkills?: { name: string; triggered: boolean }[];
}

/** Build the configured-skills roster, deduping by name since a skill's multiple trigger blocks (e.g. PR + schedule) share one name. */
export function buildConfiguredSkillsList({
allTriggers,
matchedTriggers,
}: {
allTriggers: { name: string }[];
matchedTriggers: { name: string }[];
}): { name: string; triggered: boolean }[] {
const matchedNames = new Set(matchedTriggers.map((t) => t.name));
const seen = new Set<string>();
const result: { name: string; triggered: boolean }[] = [];

for (const trigger of allTriggers) {
if (seen.has(trigger.name)) continue;
seen.add(trigger.name);
result.push({ name: trigger.name, triggered: matchedNames.has(trigger.name) });
}

return result;
}

function serializeTriggerError(error: unknown): z.infer<typeof TriggerErrorSchema> {
Expand Down Expand Up @@ -222,6 +248,7 @@ export function buildFindingsOutput(
triggerResults: options.triggerResults.map(serializeTriggerResult),
}),
findingObservations,
...(options.configuredSkills && { configuredSkills: options.configuredSkills }),
};

return FindingsOutputSchema.parse(output);
Expand Down
6 changes: 5 additions & 1 deletion packages/warden/src/action/workflow/base.ts
Original file line number Diff line number Diff line change
Expand Up @@ -380,11 +380,15 @@ export function writeFindingsOutput(
reports: SkillReport[],
context: EventContext,
findingObservations: FindingObservation[] = [],
options: { triggerResults?: ReplayTriggerResult[] } = {}
options: {
triggerResults?: ReplayTriggerResult[];
configuredSkills?: { name: string; triggered: boolean }[];
} = {}
): string {
const filePath = getFindingsOutputPath(context.repoPath);
const output = buildFindingsOutput(reports, context, findingObservations, {
triggerResults: options.triggerResults,
configuredSkills: options.configuredSkills,
});

mkdirSync(dirname(filePath), { recursive: true });
Expand Down
30 changes: 29 additions & 1 deletion packages/warden/src/action/workflow/pr-workflow.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ const RUNTIME_CLAUDE_FIXTURES_DIR = join(FIXTURES_DIR, 'runtime-claude');
const EMPTY_AUXILIARY_MODEL_FIXTURES_DIR = join(FIXTURES_DIR, 'empty-auxiliary-model');
const LAYERED_AUXILIARY_MODEL_FIXTURES_DIR = join(FIXTURES_DIR, 'layered-auxiliary-model');
const NO_MATCH_EMPTY_AUXILIARY_MODEL_FIXTURES_DIR = join(FIXTURES_DIR, 'no-match-empty-auxiliary-model');
const SCHEDULE_ONLY_FIXTURES_DIR = join(FIXTURES_DIR, 'schedule');
const EVENT_PAYLOAD_PATH = join(FIXTURES_DIR, 'event-payloads/pull_request_opened.json');
const PR_HEAD_SHA = 'abc123def456';
const PREVIOUS_HEAD_SHA = 'previous123sha456';
Expand Down Expand Up @@ -381,6 +382,30 @@ describe('runPRWorkflow', () => {
report,
}),
],
configuredSkills: [{ name: 'test-skill', triggered: true }],
}
);
});

it('analyze mode lists a schedule-only skill as configured but not triggered on a PR run', async () => {
await runPRWorkflow(
mockOctokit,
createDefaultInputs({ mode: 'analyze' }),
'pull_request',
EVENT_PAYLOAD_PATH,
SCHEDULE_ONLY_FIXTURES_DIR
);

expect(mockRunSkillTask).not.toHaveBeenCalled();
expect(mockWriteFindingsOutput).toHaveBeenCalledWith(
[],
expect.objectContaining({
repository: expect.objectContaining({ fullName: 'test-owner/test-repo' }),
}),
[],
{
triggerResults: [],
configuredSkills: [{ name: 'test-skill', triggered: false }],
}
);
});
Expand Down Expand Up @@ -2410,7 +2435,10 @@ describe('runPRWorkflow', () => {
skill: undefined,
resolvedReason: 'fix_evaluation',
}),
]
],
{
configuredSkills: [{ name: 'test-skill', triggered: false }],
}
);
});

Expand Down
54 changes: 46 additions & 8 deletions packages/warden/src/action/workflow/pr-workflow.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ import {
import { renderSkillReport } from '../../output/renderer.js';
import {
FindingsOutputSchema,
buildConfiguredSkillsList,
type FindingsOutput,
type ReplayTriggerResult,
} from '../reporting/output.js';
Expand All @@ -93,6 +94,7 @@ interface InitResult {
context: EventContext;
runnerConcurrency?: number;
auxiliaryOptions: AuxiliaryWorkflowOptions;
resolvedTriggers: ResolvedTrigger[];
matchedTriggers: ResolvedTrigger[];
skippedTriggers: ResolvedTrigger[];
skipCoreCheck?: SkippedCoreCheck;
Expand Down Expand Up @@ -345,7 +347,14 @@ async function initializeWorkflow(
console.log('No triggers matched for this event');
}

return { context, runnerConcurrency, auxiliaryOptions, matchedTriggers, skippedTriggers };
return {
context,
runnerConcurrency,
auxiliaryOptions,
resolvedTriggers,
matchedTriggers,
skippedTriggers,
};
} catch (error) {
if (
error instanceof ConfigLoadError &&
Expand All @@ -358,6 +367,7 @@ async function initializeWorkflow(
context,
runnerConcurrency,
auxiliaryOptions,
resolvedTriggers: [],
matchedTriggers: [],
skippedTriggers: [],
skipCoreCheck: {
Expand Down Expand Up @@ -980,7 +990,9 @@ async function finalizeWorkflow(
failureReasons: string[],
canResolveStale: boolean,
gate: ReviewFeedbackGate,
triggerErrors: string[]
triggerErrors: string[],
matchedTriggers: ResolvedTrigger[],
resolvedTriggers: ResolvedTrigger[]
): Promise<void> {
await dismissPreviousReviewIfResolved(
octokit,
Expand All @@ -999,6 +1011,7 @@ async function finalizeWorkflow(
try {
const findingsPath = writeFindingsOutput(reports, context, findingObservations, {
triggerResults: toReplayTriggerResults(results),
configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }),
});
logAction(`Findings written to ${findingsPath}`);
} catch (error) {
Expand Down Expand Up @@ -1572,7 +1585,11 @@ async function finalizeReportWorkflow(
canResolveStale: boolean,
gate: ReviewFeedbackGate,
triggerErrors: string[],
options: { failOnWriteError?: boolean } = {}
options: {
failOnWriteError?: boolean;
matchedTriggers: ResolvedTrigger[];
resolvedTriggers: ResolvedTrigger[];
}
): Promise<void> {
await dismissPreviousReviewIfResolved(
octokit,
Expand All @@ -1590,6 +1607,10 @@ async function finalizeReportWorkflow(
try {
const findingsPath = writeFindingsOutput(reports, context, findingObservations, {
triggerResults: toReplayTriggerResults(results),
configuredSkills: buildConfiguredSkillsList({
allTriggers: options.resolvedTriggers,
matchedTriggers: options.matchedTriggers,
}),
});
logAction(`Findings written to ${findingsPath}`);
} catch (error) {
Expand Down Expand Up @@ -1712,6 +1733,7 @@ async function runAnalyzeMode(
const {
context,
runnerConcurrency,
resolvedTriggers,
matchedTriggers,
skipCoreCheck,
} = initResult;
Expand All @@ -1721,7 +1743,10 @@ async function runAnalyzeMode(
setOutput('high-count', 0);
setOutput('summary', skipCoreCheck?.title ?? 'No triggers matched');
try {
const findingsPath = writeFindingsOutput([], context, [], { triggerResults: [] });
const findingsPath = writeFindingsOutput([], context, [], {
triggerResults: [],
configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }),
});
logAction(`Findings written to ${findingsPath}`);
} catch (error) {
setFailed(`Failed to write findings output: ${error}`);
Expand All @@ -1747,6 +1772,7 @@ async function runAnalyzeMode(
try {
const findingsPath = writeFindingsOutput(reports, context, [], {
triggerResults: toReplayTriggerResults(results),
configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }),
});
logAction(`Findings written to ${findingsPath}`);
} catch (error) {
Expand All @@ -1771,6 +1797,7 @@ async function runReportMode(
const {
context,
auxiliaryOptions,
resolvedTriggers,
matchedTriggers,
skippedTriggers,
skipCoreCheck,
Expand All @@ -1792,7 +1819,10 @@ async function runReportMode(
const outputs = { findingsCount: 0, highCount: 0, summary: skipCoreCheck.title };
setWorkflowOutputs(outputs);
try {
const findingsPath = writeFindingsOutput([], context, [], { triggerResults: [] });
const findingsPath = writeFindingsOutput([], context, [], {
triggerResults: [],
configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }),
});
logAction(`Findings written to ${findingsPath}`);
} catch (error) {
warnAction(`Failed to write findings output: ${error}`);
Expand Down Expand Up @@ -1827,6 +1857,7 @@ async function runReportMode(
try {
const findingsPath = writeFindingsOutput([], context, cleanupFindingObservations, {
triggerResults: [],
configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }),
});
logAction(`Findings written to ${findingsPath}`);
} catch (error) {
Expand Down Expand Up @@ -1899,7 +1930,7 @@ async function runReportMode(
canResolveStale,
gate,
triggerErrors,
{ failOnWriteError: true },
{ failOnWriteError: true, matchedTriggers, resolvedTriggers },
);
} catch (error) {
if (error instanceof ActionFailedError) {
Expand Down Expand Up @@ -1938,6 +1969,7 @@ export async function runPRWorkflow(
context,
runnerConcurrency,
auxiliaryOptions,
resolvedTriggers,
matchedTriggers,
skippedTriggers,
skipCoreCheck,
Expand Down Expand Up @@ -1988,7 +2020,9 @@ export async function runPRWorkflow(
setOutput('high-count', 0);
setOutput('summary', skipCoreCheck.title);
try {
writeFindingsOutput([], context);
writeFindingsOutput([], context, [], {
configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }),
});
} catch (error) {
warnAction(`Failed to write findings output: ${error}`);
}
Expand All @@ -2008,7 +2042,9 @@ export async function runPRWorkflow(
setOutput('high-count', 0);
setOutput('summary', 'No triggers matched');
try {
writeFindingsOutput([], context, cleanupFindingObservations);
writeFindingsOutput([], context, cleanupFindingObservations, {
configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }),
});
} catch (error) {
warnAction(`Failed to write findings output: ${error}`);
}
Expand Down Expand Up @@ -2088,6 +2124,8 @@ export async function runPRWorkflow(
canResolveStale,
gate,
triggerErrors,
matchedTriggers,
resolvedTriggers,
);

handleTriggerErrors(triggerErrors, matchedTriggers.length);
Expand Down
Loading