From 50f05543ddeb078273cf390eaacd11c7913a48c9 Mon Sep 17 00:00:00 2001 From: Caleb Aston Date: Mon, 20 Jul 2026 15:36:20 -0600 Subject: [PATCH 1/5] feat(reporting): add configuredSkills roster to findings output --- .../src/action/reporting/output.test.ts | 64 ++++++++++++++++++- .../warden/src/action/reporting/output.ts | 26 ++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/packages/warden/src/action/reporting/output.test.ts b/packages/warden/src/action/reporting/output.test.ts index 7ced469bd..03808fead 100644 --- a/packages/warden/src/action/reporting/output.test.ts +++ b/packages/warden/src/action/reporting/output.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import type { EventContext, Finding, SkillReport } from '../../types/index.js'; -import { buildFindingsOutput, FindingsOutputSchema } from './output.js'; +import { buildConfiguredSkillsList, buildFindingsOutput, FindingsOutputSchema } from './output.js'; describe('findings output schema', () => { it('builds a schema-valid public findings payload', () => { @@ -227,6 +227,68 @@ describe('findings output schema', () => { expect('failedExtractions' in serialized).toBe(false); expect('error' in serialized).toBe(false); }); + + it('includes the configured skills roster when provided', () => { + const output = buildFindingsOutput([createReport()], createContext(), [], { + timestamp: '2026-01-01T00:00:00.000Z', + runId: '123', + configuredSkills: [ + { name: 'test-skill', triggered: true }, + { name: 'idle-skill', triggered: false }, + ], + }); + + expect(FindingsOutputSchema.parse(output)).toEqual(output); + expect(output.configuredSkills).toEqual([ + { name: 'test-skill', triggered: true }, + { name: 'idle-skill', triggered: false }, + ]); + }); + + it('omits the configured skills roster when not provided', () => { + const output = buildFindingsOutput([createReport()], createContext(), [], { + timestamp: '2026-01-01T00:00:00.000Z', + runId: '123', + }); + + expect(output.configuredSkills).toBeUndefined(); + }); +}); + +describe('buildConfiguredSkillsList', () => { + it('marks matched skills as triggered and unmatched skills as not', () => { + const result = buildConfiguredSkillsList({ + allTriggers: [{ name: 'matched-skill' }, { name: 'skipped-skill' }], + matchedTriggers: [{ name: 'matched-skill' }], + }); + + expect(result).toEqual([ + { name: 'matched-skill', triggered: true }, + { name: 'skipped-skill', triggered: false }, + ]); + }); + + it('deduplicates multiple trigger blocks for the same skill', () => { + const result = buildConfiguredSkillsList({ + allTriggers: [{ name: 'multi-trigger-skill' }, { name: 'multi-trigger-skill' }], + matchedTriggers: [{ name: 'multi-trigger-skill' }], + }); + + expect(result).toEqual([{ name: 'multi-trigger-skill', triggered: true }]); + }); + + it('returns an empty list when nothing is configured', () => { + expect(buildConfiguredSkillsList({ allTriggers: [], matchedTriggers: [] })).toEqual([]); + }); + + it('includes a skill whose only trigger is neither matched nor a PR-check skip', () => { + const result = buildConfiguredSkillsList({ + allTriggers: [{ name: 'nightly-sweep' }], + matchedTriggers: [], + }); + + expect(result).toEqual([{ name: 'nightly-sweep', triggered: false }]); + }); }); function createFinding(): Finding { diff --git a/packages/warden/src/action/reporting/output.ts b/packages/warden/src/action/reporting/output.ts index 2cce40cb0..a209e755e 100644 --- a/packages/warden/src/action/reporting/output.ts +++ b/packages/warden/src/action/reporting/output.ts @@ -100,6 +100,10 @@ export const FindingsOutputSchema = z.object({ })), triggerResults: z.array(TriggerRunResultSchema).optional(), findingObservations: z.array(FindingObservationSchema), + configuredSkills: z.array(z.object({ + name: z.string(), + triggered: z.boolean(), + })).optional(), }); export type FindingsOutput = z.infer; @@ -116,6 +120,27 @@ interface BuildFindingsOutputOptions { timestamp?: string; runId?: string; triggerResults?: ReplayTriggerResult[]; + configuredSkills?: { name: string; triggered: boolean }[]; +} + +export function buildConfiguredSkillsList({ + allTriggers, + matchedTriggers, +}: { + allTriggers: { name: string }[]; + matchedTriggers: { name: string }[]; +}): { name: string; triggered: boolean }[] { + const matchedNames = new Set(matchedTriggers.map((t) => t.name)); + const seen = new Set(); + const result: { name: string; triggered: boolean }[] = []; + + for (const trigger of allTriggers) { + if (seen.has(trigger.name)) continue; + seen.add(trigger.name); + result.push({ name: trigger.name, triggered: matchedNames.has(trigger.name) }); + } + + return result; } function serializeTriggerError(error: unknown): z.infer { @@ -222,6 +247,7 @@ export function buildFindingsOutput( triggerResults: options.triggerResults.map(serializeTriggerResult), }), findingObservations, + ...(options.configuredSkills && { configuredSkills: options.configuredSkills }), }; return FindingsOutputSchema.parse(output); From 2caa19eac402490cc89a3c42834ece5b455c69e3 Mon Sep 17 00:00:00 2001 From: Caleb Aston Date: Mon, 20 Jul 2026 15:40:26 -0600 Subject: [PATCH 2/5] feat(workflow): accept configuredSkills in writeFindingsOutput options --- packages/warden/src/action/workflow/base.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/warden/src/action/workflow/base.ts b/packages/warden/src/action/workflow/base.ts index 45a24c78f..d5b1ddb5f 100644 --- a/packages/warden/src/action/workflow/base.ts +++ b/packages/warden/src/action/workflow/base.ts @@ -380,11 +380,15 @@ export function writeFindingsOutput( reports: SkillReport[], context: EventContext, findingObservations: FindingObservation[] = [], - options: { triggerResults?: ReplayTriggerResult[] } = {} + options: { + triggerResults?: ReplayTriggerResult[]; + configuredSkills?: { name: string; triggered: boolean }[]; + } = {} ): string { const filePath = getFindingsOutputPath(context.repoPath); const output = buildFindingsOutput(reports, context, findingObservations, { triggerResults: options.triggerResults, + configuredSkills: options.configuredSkills, }); mkdirSync(dirname(filePath), { recursive: true }); From 264a53b6585d6bed30180d9c33b0e3ac1b117c3e Mon Sep 17 00:00:00 2001 From: Caleb Aston Date: Mon, 20 Jul 2026 15:48:35 -0600 Subject: [PATCH 3/5] feat(workflow): compute and publish configuredSkills roster in pr-workflow --- .../src/action/workflow/pr-workflow.test.ts | 30 ++++++++++- .../warden/src/action/workflow/pr-workflow.ts | 54 ++++++++++++++++--- 2 files changed, 75 insertions(+), 9 deletions(-) diff --git a/packages/warden/src/action/workflow/pr-workflow.test.ts b/packages/warden/src/action/workflow/pr-workflow.test.ts index 52a988ea6..92e5320d1 100644 --- a/packages/warden/src/action/workflow/pr-workflow.test.ts +++ b/packages/warden/src/action/workflow/pr-workflow.test.ts @@ -26,6 +26,7 @@ const RUNTIME_CLAUDE_FIXTURES_DIR = join(FIXTURES_DIR, 'runtime-claude'); const EMPTY_AUXILIARY_MODEL_FIXTURES_DIR = join(FIXTURES_DIR, 'empty-auxiliary-model'); const LAYERED_AUXILIARY_MODEL_FIXTURES_DIR = join(FIXTURES_DIR, 'layered-auxiliary-model'); const NO_MATCH_EMPTY_AUXILIARY_MODEL_FIXTURES_DIR = join(FIXTURES_DIR, 'no-match-empty-auxiliary-model'); +const SCHEDULE_ONLY_FIXTURES_DIR = join(FIXTURES_DIR, 'schedule'); const EVENT_PAYLOAD_PATH = join(FIXTURES_DIR, 'event-payloads/pull_request_opened.json'); const PR_HEAD_SHA = 'abc123def456'; const PREVIOUS_HEAD_SHA = 'previous123sha456'; @@ -381,6 +382,30 @@ describe('runPRWorkflow', () => { report, }), ], + configuredSkills: [{ name: 'test-skill', triggered: true }], + } + ); + }); + + it('analyze mode lists a schedule-only skill as configured but not triggered on a PR run', async () => { + await runPRWorkflow( + mockOctokit, + createDefaultInputs({ mode: 'analyze' }), + 'pull_request', + EVENT_PAYLOAD_PATH, + SCHEDULE_ONLY_FIXTURES_DIR + ); + + expect(mockRunSkillTask).not.toHaveBeenCalled(); + expect(mockWriteFindingsOutput).toHaveBeenCalledWith( + [], + expect.objectContaining({ + repository: expect.objectContaining({ fullName: 'test-owner/test-repo' }), + }), + [], + { + triggerResults: [], + configuredSkills: [{ name: 'test-skill', triggered: false }], } ); }); @@ -2410,7 +2435,10 @@ describe('runPRWorkflow', () => { skill: undefined, resolvedReason: 'fix_evaluation', }), - ] + ], + { + configuredSkills: [{ name: 'test-skill', triggered: false }], + } ); }); diff --git a/packages/warden/src/action/workflow/pr-workflow.ts b/packages/warden/src/action/workflow/pr-workflow.ts index 879668db2..9e1c1d81e 100644 --- a/packages/warden/src/action/workflow/pr-workflow.ts +++ b/packages/warden/src/action/workflow/pr-workflow.ts @@ -81,6 +81,7 @@ import { import { renderSkillReport } from '../../output/renderer.js'; import { FindingsOutputSchema, + buildConfiguredSkillsList, type FindingsOutput, type ReplayTriggerResult, } from '../reporting/output.js'; @@ -93,6 +94,7 @@ interface InitResult { context: EventContext; runnerConcurrency?: number; auxiliaryOptions: AuxiliaryWorkflowOptions; + resolvedTriggers: ResolvedTrigger[]; matchedTriggers: ResolvedTrigger[]; skippedTriggers: ResolvedTrigger[]; skipCoreCheck?: SkippedCoreCheck; @@ -345,7 +347,14 @@ async function initializeWorkflow( console.log('No triggers matched for this event'); } - return { context, runnerConcurrency, auxiliaryOptions, matchedTriggers, skippedTriggers }; + return { + context, + runnerConcurrency, + auxiliaryOptions, + resolvedTriggers, + matchedTriggers, + skippedTriggers, + }; } catch (error) { if ( error instanceof ConfigLoadError && @@ -358,6 +367,7 @@ async function initializeWorkflow( context, runnerConcurrency, auxiliaryOptions, + resolvedTriggers: [], matchedTriggers: [], skippedTriggers: [], skipCoreCheck: { @@ -980,7 +990,9 @@ async function finalizeWorkflow( failureReasons: string[], canResolveStale: boolean, gate: ReviewFeedbackGate, - triggerErrors: string[] + triggerErrors: string[], + matchedTriggers: ResolvedTrigger[], + resolvedTriggers: ResolvedTrigger[] ): Promise { await dismissPreviousReviewIfResolved( octokit, @@ -999,6 +1011,7 @@ async function finalizeWorkflow( try { const findingsPath = writeFindingsOutput(reports, context, findingObservations, { triggerResults: toReplayTriggerResults(results), + configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }), }); logAction(`Findings written to ${findingsPath}`); } catch (error) { @@ -1572,7 +1585,11 @@ async function finalizeReportWorkflow( canResolveStale: boolean, gate: ReviewFeedbackGate, triggerErrors: string[], - options: { failOnWriteError?: boolean } = {} + options: { + failOnWriteError?: boolean; + matchedTriggers?: ResolvedTrigger[]; + resolvedTriggers?: ResolvedTrigger[]; + } = {} ): Promise { await dismissPreviousReviewIfResolved( octokit, @@ -1590,6 +1607,10 @@ async function finalizeReportWorkflow( try { const findingsPath = writeFindingsOutput(reports, context, findingObservations, { triggerResults: toReplayTriggerResults(results), + configuredSkills: buildConfiguredSkillsList({ + allTriggers: options.resolvedTriggers ?? [], + matchedTriggers: options.matchedTriggers ?? [], + }), }); logAction(`Findings written to ${findingsPath}`); } catch (error) { @@ -1712,6 +1733,7 @@ async function runAnalyzeMode( const { context, runnerConcurrency, + resolvedTriggers, matchedTriggers, skipCoreCheck, } = initResult; @@ -1721,7 +1743,10 @@ async function runAnalyzeMode( setOutput('high-count', 0); setOutput('summary', skipCoreCheck?.title ?? 'No triggers matched'); try { - const findingsPath = writeFindingsOutput([], context, [], { triggerResults: [] }); + const findingsPath = writeFindingsOutput([], context, [], { + triggerResults: [], + configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }), + }); logAction(`Findings written to ${findingsPath}`); } catch (error) { setFailed(`Failed to write findings output: ${error}`); @@ -1747,6 +1772,7 @@ async function runAnalyzeMode( try { const findingsPath = writeFindingsOutput(reports, context, [], { triggerResults: toReplayTriggerResults(results), + configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }), }); logAction(`Findings written to ${findingsPath}`); } catch (error) { @@ -1771,6 +1797,7 @@ async function runReportMode( const { context, auxiliaryOptions, + resolvedTriggers, matchedTriggers, skippedTriggers, skipCoreCheck, @@ -1792,7 +1819,10 @@ async function runReportMode( const outputs = { findingsCount: 0, highCount: 0, summary: skipCoreCheck.title }; setWorkflowOutputs(outputs); try { - const findingsPath = writeFindingsOutput([], context, [], { triggerResults: [] }); + const findingsPath = writeFindingsOutput([], context, [], { + triggerResults: [], + configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }), + }); logAction(`Findings written to ${findingsPath}`); } catch (error) { warnAction(`Failed to write findings output: ${error}`); @@ -1827,6 +1857,7 @@ async function runReportMode( try { const findingsPath = writeFindingsOutput([], context, cleanupFindingObservations, { triggerResults: [], + configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }), }); logAction(`Findings written to ${findingsPath}`); } catch (error) { @@ -1899,7 +1930,7 @@ async function runReportMode( canResolveStale, gate, triggerErrors, - { failOnWriteError: true }, + { failOnWriteError: true, matchedTriggers, resolvedTriggers }, ); } catch (error) { if (error instanceof ActionFailedError) { @@ -1938,6 +1969,7 @@ export async function runPRWorkflow( context, runnerConcurrency, auxiliaryOptions, + resolvedTriggers, matchedTriggers, skippedTriggers, skipCoreCheck, @@ -1988,7 +2020,9 @@ export async function runPRWorkflow( setOutput('high-count', 0); setOutput('summary', skipCoreCheck.title); try { - writeFindingsOutput([], context); + writeFindingsOutput([], context, [], { + configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }), + }); } catch (error) { warnAction(`Failed to write findings output: ${error}`); } @@ -2008,7 +2042,9 @@ export async function runPRWorkflow( setOutput('high-count', 0); setOutput('summary', 'No triggers matched'); try { - writeFindingsOutput([], context, cleanupFindingObservations); + writeFindingsOutput([], context, cleanupFindingObservations, { + configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }), + }); } catch (error) { warnAction(`Failed to write findings output: ${error}`); } @@ -2088,6 +2124,8 @@ export async function runPRWorkflow( canResolveStale, gate, triggerErrors, + matchedTriggers, + resolvedTriggers, ); handleTriggerErrors(triggerErrors, matchedTriggers.length); From 28e16086db1c5afa6d406b98e7b86cead76cd5b2 Mon Sep 17 00:00:00 2001 From: Caleb Aston Date: Tue, 21 Jul 2026 08:57:41 -0600 Subject: [PATCH 4/5] docs(reporting): document buildConfiguredSkillsList dedup behavior --- packages/warden/src/action/reporting/output.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/warden/src/action/reporting/output.ts b/packages/warden/src/action/reporting/output.ts index a209e755e..3b2e48094 100644 --- a/packages/warden/src/action/reporting/output.ts +++ b/packages/warden/src/action/reporting/output.ts @@ -123,6 +123,7 @@ interface BuildFindingsOutputOptions { configuredSkills?: { name: string; triggered: boolean }[]; } +/** Build the configured-skills roster, deduping by name since a skill's multiple trigger blocks (e.g. PR + schedule) share one name. */ export function buildConfiguredSkillsList({ allTriggers, matchedTriggers, From 297f1de404e372c1de39b485d1cecb66b297d8c1 Mon Sep 17 00:00:00 2001 From: Caleb Aston Date: Tue, 21 Jul 2026 08:57:48 -0600 Subject: [PATCH 5/5] fix(workflow): require matchedTriggers/resolvedTriggers in finalizeReportWorkflow options --- packages/warden/src/action/workflow/pr-workflow.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/warden/src/action/workflow/pr-workflow.ts b/packages/warden/src/action/workflow/pr-workflow.ts index 9e1c1d81e..895d582ee 100644 --- a/packages/warden/src/action/workflow/pr-workflow.ts +++ b/packages/warden/src/action/workflow/pr-workflow.ts @@ -1587,9 +1587,9 @@ async function finalizeReportWorkflow( triggerErrors: string[], options: { failOnWriteError?: boolean; - matchedTriggers?: ResolvedTrigger[]; - resolvedTriggers?: ResolvedTrigger[]; - } = {} + matchedTriggers: ResolvedTrigger[]; + resolvedTriggers: ResolvedTrigger[]; + } ): Promise { await dismissPreviousReviewIfResolved( octokit, @@ -1608,8 +1608,8 @@ async function finalizeReportWorkflow( const findingsPath = writeFindingsOutput(reports, context, findingObservations, { triggerResults: toReplayTriggerResults(results), configuredSkills: buildConfiguredSkillsList({ - allTriggers: options.resolvedTriggers ?? [], - matchedTriggers: options.matchedTriggers ?? [], + allTriggers: options.resolvedTriggers, + matchedTriggers: options.matchedTriggers, }), }); logAction(`Findings written to ${findingsPath}`);