diff --git a/experiments/claude-code-sonnet-5-no-skills.ts b/experiments/claude-code-sonnet-5-no-skills.ts index 5452fe38..5a266f92 100644 --- a/experiments/claude-code-sonnet-5-no-skills.ts +++ b/experiments/claude-code-sonnet-5-no-skills.ts @@ -8,7 +8,7 @@ import { localStackRuntime } from '@supabase-evals/sandbox'; // Same as claude-code-sonnet-5 but with no skills, to measure skills' impact. export default defineExperiment({ - suite: ['no-skills', 'regression'], + suite: ['no-skills'], agent: claudeCodeAgent({ model: 'claude-sonnet-5', reasoningEffort: 'high', @@ -18,6 +18,4 @@ export default defineExperiment({ }), localStack: localStackRuntime(), skills: [], - // Evals that override `skills: []` already run under the baseline experiment. Skip them from running again here. - skipEval: (ev) => ev.metadata.skills?.length === 0, }); diff --git a/experiments/claude-code-sonnet-5.ts b/experiments/claude-code-sonnet-5.ts index 1572000e..e2b01a50 100644 --- a/experiments/claude-code-sonnet-5.ts +++ b/experiments/claude-code-sonnet-5.ts @@ -7,7 +7,7 @@ import { import { localStackRuntime } from '@supabase-evals/sandbox'; export default defineExperiment({ - suite: ['benchmark', 'regression'], + suite: ['benchmark'], agent: claudeCodeAgent({ model: 'claude-sonnet-5', reasoningEffort: 'high', diff --git a/experiments/codex-gpt-5.6-luna-no-skills.ts b/experiments/codex-gpt-5.6-luna-no-skills.ts index 40ccfbaf..34abfe99 100644 --- a/experiments/codex-gpt-5.6-luna-no-skills.ts +++ b/experiments/codex-gpt-5.6-luna-no-skills.ts @@ -7,7 +7,7 @@ import { import { localStackRuntime } from '@supabase-evals/sandbox'; export default defineExperiment({ - suite: ['no-skills'], + suite: ['no-skills', 'regression'], agent: codexAgent({ model: 'gpt-5.6-luna', reasoningEffort: 'medium', @@ -17,4 +17,6 @@ export default defineExperiment({ }), localStack: localStackRuntime(), skills: [], + // Evals that override `skills: []` already run under the baseline experiment. Skip them from running again here. + skipEval: (ev) => ev.metadata.skills?.length === 0, }); diff --git a/experiments/codex-gpt-5.6-luna.ts b/experiments/codex-gpt-5.6-luna.ts index 141b1459..d4e18052 100644 --- a/experiments/codex-gpt-5.6-luna.ts +++ b/experiments/codex-gpt-5.6-luna.ts @@ -7,7 +7,7 @@ import { import { localStackRuntime } from '@supabase-evals/sandbox'; export default defineExperiment({ - suite: ['benchmark'], + suite: ['benchmark', 'regression'], agent: codexAgent({ model: 'gpt-5.6-luna', reasoningEffort: 'medium',