From a49b3076f86e328c7c092099316997cd329c5cb9 Mon Sep 17 00:00:00 2001 From: WUKUNTAI Date: Thu, 13 Aug 2026 10:56:46 +0800 Subject: [PATCH 1/3] fix(libsy): warn when capable_first cannot reach the efficient tier Signed-off-by: WUKUNTAI --- crates/libsy/src/algorithms/stage.rs | 21 ++++- crates/libsy/src/algorithms/util/stage.rs | 100 ++++++++++++++++++++++ crates/libsy/src/lib.rs | 2 +- 3 files changed, 121 insertions(+), 2 deletions(-) diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs index 6512d763..3010b079 100644 --- a/crates/libsy/src/algorithms/stage.rs +++ b/crates/libsy/src/algorithms/stage.rs @@ -22,7 +22,8 @@ use super::llm_class::{LlmClassifierConfig, LlmTaskClassifier, TaskClassifierCon use super::util::prompts::{SystemPromptProcessor, TargetPrompts}; use super::util::stage::{ DecisionSource, HandoffNoteConfig, PickerMode, StageClassifier, StageTargets, - record_decision_source, record_routing_decision, + max_efficient_confidence, record_decision_source, record_routing_decision, + scorer_cannot_leave_default, }; use super::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSignalProcessor}; use crate::core::algorithm::{Algorithm, Driver}; @@ -158,6 +159,24 @@ fn build_route( ), }); } + // A threshold above the efficient ceiling silently disables the scorer for + // capable_first: every turn reads as "signals were weak" when in fact that + // branch cannot fire at all. Warn rather than reject, since the judge and the + // hard de-escalation shortcut still work and the setting stays a policy choice. + if scorer_cannot_leave_default(config.mode, config.confidence_threshold) { + tracing::warn!( + confidence_threshold = config.confidence_threshold, + efficient_ceiling = max_efficient_confidence(), + "stage_router picker \"capable_first\" with confidence_threshold {} cannot de-escalate: \ + production_intensity is the only signal on the efficient side, so the scorer tops out \ + at {:.4}. Every turn will fall through to the judge, or to the capable tier when no \ + judge is configured. Set confidence_threshold at or below {:.4} for the scorer to \ + reach the efficient tier.", + config.confidence_threshold, + max_efficient_confidence(), + max_efficient_confidence(), + ); + } // The tiers are a fixed pair; their targets are whatever the deployment calls // them, and the classifier scores onto those names. let targets = StageTargets::new(capable.clone(), efficient.clone()); diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rs index 3f16334f..1962e548 100644 --- a/crates/libsy/src/algorithms/util/stage.rs +++ b/crates/libsy/src/algorithms/util/stage.rs @@ -334,6 +334,28 @@ pub fn score_signal(signal: &ToolSignals) -> ScoreResult { } } +/// Highest confidence the scorer can reach toward the efficient tier. +/// +/// `production_intensity` is the only dimension on the efficient side and +/// [`ratio`] bounds it to `1.0`, so the efficient direction has exactly one +/// signal unit to work with — there is nothing for it to corroborate with. The +/// `tanh` squash then pulls that unit down from `SCORE_GAIN * SIGNAL_UNIT` to +/// roughly `0.4621`, which lands just under the commonly used `0.5` threshold. +pub fn max_efficient_confidence() -> f64 { + (SCORE_GAIN * SIGNAL_UNIT).tanh() +} + +/// True when `confidence_threshold` puts every scorer outcome out of reach of +/// `mode`'s non-default tier, so the scorer can never change a turn's tier. +/// +/// Only [`PickerMode::CapableFirst`] can hit this: leaving its default means +/// picking efficient, and that side caps at [`max_efficient_confidence`]. +/// Escalation stacks several dimensions, so `efficient_first` always has some +/// reachable band and is never reported here. +pub fn scorer_cannot_leave_default(mode: PickerMode, confidence_threshold: f64) -> bool { + matches!(mode, PickerMode::CapableFirst) && confidence_threshold > max_efficient_confidence() +} + /// Hard **escalate** — force the capable tier no matter what the scorer would /// say. Fires on a critical error or a compacted context. fn should_escalate(signal: &ToolSignals) -> bool { @@ -649,6 +671,84 @@ mod tests { ); } + #[test] + fn production_is_the_only_signal_pushing_toward_efficient() { + // The de-escalation ceiling is a property of the dimension set: only + // `production_intensity` is negative, and `ratio` bounds it to 1.0. A + // turn that is pure production therefore scores the strongest efficient + // confidence the scorer can ever produce. + let mut signal = signal_from(json!([{"role": "user", "content": "hi"}])); + signal.recent_write_count = 3; + signal.recent_edit_count = 1; + let scored = score_signal(&signal); + assert!(scored.score < 0.0, "expected an efficient lean: {scored:?}"); + assert!( + (scored.confidence - max_efficient_confidence()).abs() < 1e-12, + "pure production should reach the ceiling: {scored:?}" + ); + } + + #[test] + fn the_efficient_ceiling_sits_below_a_half() { + // tanh compresses one full signal unit (5.0 × 0.10) to ~0.4621, so the + // widely used 0.5 threshold sits above everything the efficient side can + // score. This is the constant the capable_first guard is built on. + assert!((max_efficient_confidence() - 0.462_117_157_260_009_7).abs() < 1e-12); + assert!(max_efficient_confidence() < 0.5); + } + + #[test] + fn capable_first_reports_an_unreachable_efficient_tier() { + // Above the ceiling the scorer cannot leave the capable default, so the + // combination is inert rather than merely selective. + assert!(scorer_cannot_leave_default(PickerMode::CapableFirst, 0.5)); + assert!(scorer_cannot_leave_default(PickerMode::CapableFirst, 0.47)); + assert!(!scorer_cannot_leave_default(PickerMode::CapableFirst, 0.45)); + // The gate is inclusive, so the ceiling itself still fires. + assert!(!scorer_cannot_leave_default( + PickerMode::CapableFirst, + max_efficient_confidence() + )); + } + + #[test] + fn efficient_first_is_never_reported_as_inert() { + // Escalation has more signals to stack, and this guard only speaks to the + // efficient direction. + assert!(!scorer_cannot_leave_default( + PickerMode::EfficientFirst, + 0.5 + )); + assert!(!scorer_cannot_leave_default( + PickerMode::EfficientFirst, + 1.0 + )); + } + + #[test] + fn capable_first_above_the_ceiling_never_picks_efficient() { + // End-to-end on `pick_tier`: the strongest possible efficient signal is + // still handed on rather than resolved. + let mut signal = signal_from(json!([{"role": "user", "content": "hi"}])); + signal.recent_write_count = 3; + signal.recent_edit_count = 1; + assert!(matches!( + pick_tier(&signal, PickerMode::CapableFirst, 0.5), + PickOutcome::ConsultClassifier { + default_tier: Tier::Capable, + .. + } + )); + assert!(matches!( + pick_tier(&signal, PickerMode::CapableFirst, 0.45), + PickOutcome::Resolved { + tier: Tier::Efficient, + source: DecisionSource::Dimensions, + .. + } + )); + } + #[test] fn the_picker_mode_names_the_tier_an_undecided_turn_falls_back_to() { assert_eq!(PickerMode::CapableFirst.default_tier(), Tier::Capable); diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs index 3b1ede9a..047298fc 100644 --- a/crates/libsy/src/lib.rs +++ b/crates/libsy/src/lib.rs @@ -34,7 +34,7 @@ pub use algorithms::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSignals}; pub use algorithms::util::stage::{ CodingAgentDimensions, DECISION_SOURCE_KEY, DecisionSource, HandoffNoteConfig, PickOutcome, PickerMode, ScoreResult, StageClassifier, StageTargets, Tier, dimensions_from_signal, - pick_tier, score_signal, + max_efficient_confidence, pick_tier, score_signal, scorer_cannot_leave_default, }; mod observability; From f86c6db5aab2d0585a2141f961f3d248885551ad Mon Sep 17 00:00:00 2001 From: WUKUNTAI Date: Thu, 13 Aug 2026 11:15:07 +0800 Subject: [PATCH 2/3] fix(libsy): report an unreachable scorer ceiling for both pickers Signed-off-by: WUKUNTAI --- crates/libsy/src/algorithms/stage.rs | 35 +++++--- crates/libsy/src/algorithms/util/stage.rs | 105 ++++++++++++++++++---- crates/libsy/src/lib.rs | 3 +- 3 files changed, 112 insertions(+), 31 deletions(-) diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs index 3010b079..f15800ec 100644 --- a/crates/libsy/src/algorithms/stage.rs +++ b/crates/libsy/src/algorithms/stage.rs @@ -22,8 +22,8 @@ use super::llm_class::{LlmClassifierConfig, LlmTaskClassifier, TaskClassifierCon use super::util::prompts::{SystemPromptProcessor, TargetPrompts}; use super::util::stage::{ DecisionSource, HandoffNoteConfig, PickerMode, StageClassifier, StageTargets, - max_efficient_confidence, record_decision_source, record_routing_decision, - scorer_cannot_leave_default, + max_capable_confidence, max_efficient_confidence, record_decision_source, + record_routing_decision, scorer_cannot_leave_default, }; use super::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSignalProcessor}; use crate::core::algorithm::{Algorithm, Driver}; @@ -159,22 +159,29 @@ fn build_route( ), }); } - // A threshold above the efficient ceiling silently disables the scorer for - // capable_first: every turn reads as "signals were weak" when in fact that - // branch cannot fire at all. Warn rather than reject, since the judge and the - // hard de-escalation shortcut still work and the setting stays a policy choice. + // A threshold above the scorer's ceiling in the direction that leaves the + // default tier silently disables the scorer: those turns read as "signals were + // weak" when in fact the branch cannot fire at all. Warn rather than reject — + // the hard overrides still fire, and the setting stays a policy choice. if scorer_cannot_leave_default(config.mode, config.confidence_threshold) { + let (picker, other_tier, ceiling) = match config.mode { + PickerMode::CapableFirst => ("capable_first", "efficient", max_efficient_confidence()), + PickerMode::EfficientFirst => ("efficient_first", "capable", max_capable_confidence()), + }; tracing::warn!( confidence_threshold = config.confidence_threshold, - efficient_ceiling = max_efficient_confidence(), - "stage_router picker \"capable_first\" with confidence_threshold {} cannot de-escalate: \ - production_intensity is the only signal on the efficient side, so the scorer tops out \ - at {:.4}. Every turn will fall through to the judge, or to the capable tier when no \ - judge is configured. Set confidence_threshold at or below {:.4} for the scorer to \ - reach the efficient tier.", + ceiling, + "stage_router picker \"{}\" with confidence_threshold {} can never select the {} tier: \ + the scorer tops out at {:.4} in that direction, so no signal clears the gate. Turns \ + the hard overrides do not claim fall through to the judge, or to the default tier when \ + no judge is configured. Set confidence_threshold at or below {:.4} for the scorer to \ + reach the {} tier.", + picker, config.confidence_threshold, - max_efficient_confidence(), - max_efficient_confidence(), + other_tier, + ceiling, + ceiling, + other_tier, ); } // The tiers are a fixed pair; their targets are whatever the deployment calls diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rs index 1962e548..b86fedd7 100644 --- a/crates/libsy/src/algorithms/util/stage.rs +++ b/crates/libsy/src/algorithms/util/stage.rs @@ -334,26 +334,48 @@ pub fn score_signal(signal: &ToolSignals) -> ScoreResult { } } +/// Confidence a turn scoring `units` maxed signals lands on, after the squash. +fn confidence_for_units(units: f64) -> f64 { + (SCORE_GAIN * SIGNAL_UNIT * units).tanh() +} + /// Highest confidence the scorer can reach toward the efficient tier. /// /// `production_intensity` is the only dimension on the efficient side and /// [`ratio`] bounds it to `1.0`, so the efficient direction has exactly one /// signal unit to work with — there is nothing for it to corroborate with. The -/// `tanh` squash then pulls that unit down from `SCORE_GAIN * SIGNAL_UNIT` to -/// roughly `0.4621`, which lands just under the commonly used `0.5` threshold. +/// `tanh` squash then pulls that unit down to roughly `0.4621`, which lands just +/// under the commonly used `0.5` threshold. pub fn max_efficient_confidence() -> f64 { - (SCORE_GAIN * SIGNAL_UNIT).tanh() + confidence_for_units(1.0) +} + +/// Highest confidence the scorer can reach toward the capable tier. +/// +/// Escalation stacks two units, not three: `severity` contributes one at its +/// [`HARD_SEVERITY`] cap (critical is intercepted by the override), and +/// `spinning` / `exploring` partition the not-producing case so only one of them +/// can fire. `production_intensity` is zero whenever either does, so nothing +/// subtracts. That caps escalation confidence at roughly `0.7616`. +pub fn max_capable_confidence() -> f64 { + confidence_for_units(2.0) } /// True when `confidence_threshold` puts every scorer outcome out of reach of /// `mode`'s non-default tier, so the scorer can never change a turn's tier. /// -/// Only [`PickerMode::CapableFirst`] can hit this: leaving its default means -/// picking efficient, and that side caps at [`max_efficient_confidence`]. -/// Escalation stacks several dimensions, so `efficient_first` always has some -/// reachable band and is never reported here. +/// Both pickers can hit this, because `tanh` bounds confidence well below `1.0` +/// in either direction — `capable_first` from +/// [`max_efficient_confidence`], `efficient_first` from +/// [`max_capable_confidence`]. Above its ceiling the scorer still runs and still +/// reports a score, but it can only ever confirm the default tier. pub fn scorer_cannot_leave_default(mode: PickerMode, confidence_threshold: f64) -> bool { - matches!(mode, PickerMode::CapableFirst) && confidence_threshold > max_efficient_confidence() + let ceiling = match mode { + // Leaving the capable default means picking efficient, and vice versa. + PickerMode::CapableFirst => max_efficient_confidence(), + PickerMode::EfficientFirst => max_capable_confidence(), + }; + confidence_threshold > ceiling } /// Hard **escalate** — force the capable tier no matter what the scorer would @@ -689,12 +711,36 @@ mod tests { } #[test] - fn the_efficient_ceiling_sits_below_a_half() { - // tanh compresses one full signal unit (5.0 × 0.10) to ~0.4621, so the - // widely used 0.5 threshold sits above everything the efficient side can - // score. This is the constant the capable_first guard is built on. + fn error_and_stall_signals_are_the_only_ones_pushing_toward_capable() { + // The escalation ceiling is two units, not three: severity at its hard cap + // plus whichever of spinning/exploring fires. A deep turn that errored and + // is reading without producing reaches exactly that. + let mut signal = signal_from(json!([{"role": "user", "content": "hi"}])); + signal.severity = HARD_SEVERITY as f32; + signal.turn_depth = STALL_MIN_TURN_DEPTH; + signal.recent_read_count = 2; + let dimensions = dimensions_from_signal(&signal); + assert_eq!(dimensions.exploring, 1.0); + assert_eq!(dimensions.spinning, 0.0); + let scored = score_signal(&signal); + assert!(scored.score > 0.0, "expected a capable lean: {scored:?}"); + // `severity` is an f32, so the widened 0.7 divides out a few ulps short of + // one whole unit. The ceiling itself is exact; the signal reaching it is not. + assert!( + (scored.confidence - max_capable_confidence()).abs() < 1e-7, + "an errored, exploring turn should reach the ceiling: {scored:?}" + ); + } + + #[test] + fn both_ceilings_sit_below_the_thresholds_operators_reach_for() { + // tanh compresses one signal unit to ~0.4621 and two to ~0.7616, so 0.5 + // and 1.0 respectively sit above everything each side can score. These are + // the constants the picker guard is built on. assert!((max_efficient_confidence() - 0.462_117_157_260_009_7).abs() < 1e-12); + assert!((max_capable_confidence() - 0.761_594_155_955_764_9).abs() < 1e-12); assert!(max_efficient_confidence() < 0.5); + assert!(max_capable_confidence() < 1.0); } #[test] @@ -712,16 +758,43 @@ mod tests { } #[test] - fn efficient_first_is_never_reported_as_inert() { - // Escalation has more signals to stack, and this guard only speaks to the - // efficient direction. + fn efficient_first_reports_an_unreachable_capable_tier() { + // Escalation has a second unit to stack, so its ceiling is higher — but + // `tanh` still keeps it short of 1.0, which the range check accepts. + assert!(scorer_cannot_leave_default(PickerMode::EfficientFirst, 1.0)); + assert!(scorer_cannot_leave_default(PickerMode::EfficientFirst, 0.8)); assert!(!scorer_cannot_leave_default( PickerMode::EfficientFirst, 0.5 )); assert!(!scorer_cannot_leave_default( PickerMode::EfficientFirst, - 1.0 + max_capable_confidence() + )); + } + + #[test] + fn efficient_first_at_its_ceiling_never_picks_capable() { + // End-to-end on `pick_tier`, mirroring the capable_first case: the + // strongest possible escalation signal is handed on rather than resolved. + let mut signal = signal_from(json!([{"role": "user", "content": "hi"}])); + signal.severity = HARD_SEVERITY as f32; + signal.turn_depth = STALL_MIN_TURN_DEPTH; + signal.recent_read_count = 2; + assert!(matches!( + pick_tier(&signal, PickerMode::EfficientFirst, 1.0), + PickOutcome::ConsultClassifier { + default_tier: Tier::Efficient, + .. + } + )); + assert!(matches!( + pick_tier(&signal, PickerMode::EfficientFirst, 0.75), + PickOutcome::Resolved { + tier: Tier::Capable, + source: DecisionSource::Dimensions, + .. + } )); } diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs index 047298fc..ee0ddeea 100644 --- a/crates/libsy/src/lib.rs +++ b/crates/libsy/src/lib.rs @@ -34,7 +34,8 @@ pub use algorithms::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSignals}; pub use algorithms::util::stage::{ CodingAgentDimensions, DECISION_SOURCE_KEY, DecisionSource, HandoffNoteConfig, PickOutcome, PickerMode, ScoreResult, StageClassifier, StageTargets, Tier, dimensions_from_signal, - max_efficient_confidence, pick_tier, score_signal, scorer_cannot_leave_default, + max_capable_confidence, max_efficient_confidence, pick_tier, score_signal, + scorer_cannot_leave_default, }; mod observability; From 1380c9a43ef7f13e50db629413b06aabf452bbb8 Mon Sep 17 00:00:00 2001 From: WUKUNTAI Date: Thu, 13 Aug 2026 11:28:14 +0800 Subject: [PATCH 3/3] fix(libsy): build the capable ceiling from the severity a signal can carry Signed-off-by: WUKUNTAI --- crates/libsy/src/algorithms/util/stage.rs | 57 +++++++++++++++++++++-- 1 file changed, 52 insertions(+), 5 deletions(-) diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rs index b86fedd7..7a26531d 100644 --- a/crates/libsy/src/algorithms/util/stage.rs +++ b/crates/libsy/src/algorithms/util/stage.rs @@ -350,6 +350,16 @@ pub fn max_efficient_confidence() -> f64 { confidence_for_units(1.0) } +/// Signal units `severity` contributes at its hard cap. +/// +/// A hair under one, not one: [`ToolSignals::severity`] is an `f32`, so the +/// scorer divides the widened `0.7` by an `f64` [`HARD_SEVERITY`] and lands a few +/// ulps short. The ceiling has to be built on the value a real signal produces — +/// on the exact unit, the strongest escalation falls below its own ceiling. +fn max_severity_units() -> f64 { + f64::from(HARD_SEVERITY as f32) / HARD_SEVERITY +} + /// Highest confidence the scorer can reach toward the capable tier. /// /// Escalation stacks two units, not three: `severity` contributes one at its @@ -358,7 +368,7 @@ pub fn max_efficient_confidence() -> f64 { /// can fire. `production_intensity` is zero whenever either does, so nothing /// subtracts. That caps escalation confidence at roughly `0.7616`. pub fn max_capable_confidence() -> f64 { - confidence_for_units(2.0) + confidence_for_units(max_severity_units() + 1.0) } /// True when `confidence_threshold` puts every scorer outcome out of reach of @@ -724,10 +734,8 @@ mod tests { assert_eq!(dimensions.spinning, 0.0); let scored = score_signal(&signal); assert!(scored.score > 0.0, "expected a capable lean: {scored:?}"); - // `severity` is an f32, so the widened 0.7 divides out a few ulps short of - // one whole unit. The ceiling itself is exact; the signal reaching it is not. assert!( - (scored.confidence - max_capable_confidence()).abs() < 1e-7, + (scored.confidence - max_capable_confidence()).abs() < 1e-12, "an errored, exploring turn should reach the ceiling: {scored:?}" ); } @@ -738,11 +746,50 @@ mod tests { // and 1.0 respectively sit above everything each side can score. These are // the constants the picker guard is built on. assert!((max_efficient_confidence() - 0.462_117_157_260_009_7).abs() < 1e-12); - assert!((max_capable_confidence() - 0.761_594_155_955_764_9).abs() < 1e-12); + assert!((max_capable_confidence() - 0.761_594_152_379_704_8).abs() < 1e-12); assert!(max_efficient_confidence() < 0.5); assert!(max_capable_confidence() < 1.0); } + #[test] + fn a_threshold_at_either_ceiling_still_resolves() { + // The guard stays silent at exactly the ceiling, so the strongest signal in + // each direction has to clear the inclusive gate there. This is the boundary + // an approximate ceiling would quietly break. + let mut producing = signal_from(json!([{"role": "user", "content": "hi"}])); + producing.recent_write_count = 3; + producing.recent_edit_count = 1; + assert!(matches!( + pick_tier( + &producing, + PickerMode::CapableFirst, + max_efficient_confidence() + ), + PickOutcome::Resolved { + tier: Tier::Efficient, + source: DecisionSource::Dimensions, + .. + } + )); + + let mut stalled = signal_from(json!([{"role": "user", "content": "hi"}])); + stalled.severity = HARD_SEVERITY as f32; + stalled.turn_depth = STALL_MIN_TURN_DEPTH; + stalled.recent_read_count = 2; + assert!(matches!( + pick_tier( + &stalled, + PickerMode::EfficientFirst, + max_capable_confidence() + ), + PickOutcome::Resolved { + tier: Tier::Capable, + source: DecisionSource::Dimensions, + .. + } + )); + } + #[test] fn capable_first_reports_an_unreachable_efficient_tier() { // Above the ceiling the scorer cannot leave the capable default, so the