feat(libsy): make the unmatched verdict threshold step configurable

crates/libsy/src/algorithms/llm_class.rs

@@ -73,10 +73,15 @@ impl TaskClassifierVerdict {
     }
 
     /// Returns the number of threshold steps assigned to this capability boundary.
-    fn boundary_steps(&self) -> Option<u8> {
+    ///
+    /// `unmatched_steps` comes from the route configuration: a request that matches no
+    /// capability rule has no evidence behind its solve probability, so operators can
+    /// require more confidence for it than for an uncertain verdict.
+    fn boundary_steps(&self, unmatched_steps: u8) -> Option<u8> {
         match self.capability_boundary.as_str() {
             "supported" => Some(0),
-            "uncertain" | "unmatched" => Some(1),
+            "uncertain" => Some(1),
+            "unmatched" => Some(unmatched_steps),
             "unsupported" => Some(2),
             _ => None,
         }
@@ -220,19 +225,22 @@ impl ClassifierInput for TaskInput {
 struct TaskClassifierPolicy {
     base_threshold: f64,
     threshold_step: f64,
+    unmatched_steps: u8,
 }
 
 impl TaskClassifierPolicy {
     fn new(config: &TaskClassifierConfig) -> Self {
         Self {
             base_threshold: config.base_threshold,
             threshold_step: config.threshold_step,
+            unmatched_steps: config.unmatched_steps,
         }
     }
 
     /// Returns the required solve probability for one validated verdict.
     fn threshold(&self, verdict: &TaskClassifierVerdict) -> Option<f64> {
-        Some(self.base_threshold + f64::from(verdict.boundary_steps()?) * self.threshold_step)
+        let steps = verdict.boundary_steps(self.unmatched_steps)?;
+        Some(self.base_threshold + f64::from(steps) * self.threshold_step)
     }
 }
 
@@ -304,9 +312,13 @@ pub struct TaskClassifierConfig {
     pub base_threshold: f64,
     /// Amount added per capability-boundary step.
     ///
-    /// Supported verdicts use `base_threshold`, uncertain and unmatched verdicts use one
-    /// step, and unsupported verdicts use two steps.
+    /// Supported verdicts use `base_threshold`, uncertain verdicts use one step, unmatched
+    /// verdicts use `unmatched_steps`, and unsupported verdicts use two steps.
     pub threshold_step: f64,
+    /// Threshold steps applied to an unmatched verdict (no capability rule applies).
+    ///
+    /// Defaults to 1, the same as an uncertain verdict. 2 treats it like unsupported.
+    pub unmatched_steps: u8,
     /// How often the classifier re-decides this session's target.
     pub classify_trigger: ClassifyTrigger,
     /// Uses the first user message as the SessionKey for sticky routing when session metadata is unavailable.
@@ -331,6 +343,8 @@ struct TaskClassifierConfigWire {
     base_threshold: f64,
     #[serde(default)]
     threshold_step: f64,
+    #[serde(default = "default_unmatched_steps")]
+    unmatched_steps: u8,
     #[serde(default)]
     classify_trigger: ClassifyTrigger,
     #[serde(default)]
@@ -359,6 +373,7 @@ impl<'de> Deserialize<'de> for TaskClassifierConfig {
         Ok(Self {
             base_threshold: wire.base_threshold,
             threshold_step: wire.threshold_step,
+            unmatched_steps: wire.unmatched_steps,
             classify_trigger: wire.classify_trigger,
             message_hash_fallback: wire.message_hash_fallback,
             recent_turn_window: wire.recent_turn_window,
@@ -368,6 +383,13 @@ impl<'de> Deserialize<'de> for TaskClassifierConfig {
     }
 }
 
+/// Unmatched verdicts share the uncertain step unless the route says otherwise.
+pub const DEFAULT_UNMATCHED_STEPS: u8 = 1;
+
+const fn default_unmatched_steps() -> u8 {
+    DEFAULT_UNMATCHED_STEPS
+}
+
 const fn default_judge_max_output_tokens() -> u64 {
     DEFAULT_JUDGE_MAX_OUTPUT_TOKENS
 }
@@ -377,6 +399,7 @@ impl Default for TaskClassifierConfig {
         Self {
             base_threshold: 0.0,
             threshold_step: 0.0,
+            unmatched_steps: DEFAULT_UNMATCHED_STEPS,
             classify_trigger: ClassifyTrigger::default(),
             message_hash_fallback: false,
             recent_turn_window: None,
@@ -405,6 +428,14 @@ impl TaskClassifierConfig {
                 ),
             });
         }
+        if self.unmatched_steps > 2 {
+            return Err(LibsyError::AlgorithmError {
+                message: format!(
+                    "unmatched_steps must be 0, 1, or 2, got {}",
+                    self.unmatched_steps
+                ),
+            });
+        }
         let unsupported_threshold = self.base_threshold + 2.0 * self.threshold_step;
         if unsupported_threshold > 1.0 && unsupported_threshold - 1.0 > f64::EPSILON {
             return Err(LibsyError::AlgorithmError {
@@ -1378,6 +1409,49 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn unmatched_steps_raise_only_the_unmatched_threshold() -> Result<()> {
+        let policy = TaskClassifierPolicy::new(&TaskClassifierConfig {
+            threshold_step: 0.1,
+            unmatched_steps: 2,
+            ..test_config(0.75)
+        });
+
+        // 0.85 clears one step but not two: unmatched now routes to the capable target.
+        assert_eq!(
+            selected(&policy, Some(&verdict(0.85, "unmatched", "none")))?,
+            "capable"
+        );
+        assert_eq!(
+            selected(&policy, Some(&verdict(0.95, "unmatched", "none")))?,
+            "efficient"
+        );
+        // Other boundaries keep their steps.
+        assert_eq!(
+            selected(&policy, Some(&verdict(0.85, "uncertain", "UNC-1")))?,
+            "efficient"
+        );
+        assert_eq!(
+            selected(&policy, Some(&verdict(0.75, "supported", "SUP-2")))?,
+            "efficient"
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn unmatched_steps_default_to_one_and_reject_values_above_two() {
+        let parsed: TaskClassifierConfig =
+            serde_json::from_value(serde_json::json!({ "base_threshold": 0.5 }))
+                .expect("config without unmatched_steps parses");
+        assert_eq!(parsed.unmatched_steps, 1);
+
+        let invalid = TaskClassifierConfig {
+            unmatched_steps: 3,
+            ..test_config(0.5)
+        };
+        assert!(invalid.validate().is_err());
+    }
+
     /// The text of each message a judge with `recent_turn_window` would be sent.
     /// The no-window case is covered by `capability_judge_builds_a_structured_request`.
     fn capability_judge(recent_turn_window: Option<usize>) -> Result<CapabilityJudge> {

crates/libsy/src/lib.rs

@@ -20,8 +20,8 @@ mod algorithms;
 pub use algorithms::advisor_gate::{AdvisorGate, AdvisorGateConfig, GateTrigger};
 pub use algorithms::composite::{CompositeRouter, CompositeRouterConfig};
 pub use algorithms::llm_class::{
-    CustomClassifierConfig, CustomClassifierPolicy, LlmClassifierConfig, LlmTaskClassifier,
-    TaskClassifierConfig,
+    CustomClassifierConfig, CustomClassifierPolicy, DEFAULT_UNMATCHED_STEPS, LlmClassifierConfig,
+    LlmTaskClassifier, TaskClassifierConfig,
 };
 pub use algorithms::noop::Noop;
 pub use algorithms::passthrough::Passthrough;

crates/switchyard-py/src/libsy_bindings.rs

@@ -14,10 +14,11 @@ use pyo3::prelude::*;
 use serde_json::Value;
 use switchyard_libsy::{
     Algorithm, CallModel, ClassifierContractConfig, ClassifierResponseFormat, ClassifyTrigger,
-    CustomClassifierConfig, CustomClassifierPolicy, DeescalationConfig, EscalationJudgeConfig,
-    HandoffNoteConfig, LibsyError as RustLibsyError, LlmClassifierConfig, LlmFallback,
-    LlmTaskClassifier, Noop, PickerMode, Random, RoutingOutcome, RuntimeModels, StageRouter,
-    StageRouterConfig, Step as RustStep, StepStream, TaskClassifierConfig, ToolSemantics,
+    CustomClassifierConfig, CustomClassifierPolicy, DEFAULT_UNMATCHED_STEPS, DeescalationConfig,
+    EscalationJudgeConfig, HandoffNoteConfig, LibsyError as RustLibsyError, LlmClassifierConfig,
+    LlmFallback, LlmTaskClassifier, Noop, PickerMode, Random, RoutingOutcome, RuntimeModels,
+    StageRouter, StageRouterConfig, Step as RustStep, StepStream, TaskClassifierConfig,
+    ToolSemantics,
 };
 use switchyard_protocol::{
     Category, LlmClientError, LlmResponse, LlmResponseStream, LlmResponseStreamEvent, Metadata,
@@ -330,6 +331,7 @@ impl PyTaskClassifierConfig {
             inner: TaskClassifierConfig {
                 base_threshold,
                 threshold_step,
+                unmatched_steps: DEFAULT_UNMATCHED_STEPS,
                 classify_trigger: classify_trigger(session_affinity),
                 message_hash_fallback,
                 recent_turn_window,

crates/switchyard-runner/src/algorithm.rs

@@ -12,10 +12,10 @@ use std::sync::Arc;
 use libsy::{
     AdvisorGate, AdvisorGateConfig, Algorithm, ClassifierContractConfig, ClassifierResponseFormat,
     ClassifyTrigger, CompositeRouter, CompositeRouterConfig, CustomClassifierConfig,
-    CustomClassifierPolicy, EscalationJudgeConfig, GateTrigger, HandoffNoteConfig,
-    LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough, PickerMode,
-    PlanExecute, PlanExecuteConfig, Random, StageRouter, StageRouterConfig, SubagentRouter,
-    SubagentRouterConfig, TaskClassifierConfig, ToolSemantics,
+    CustomClassifierPolicy, DEFAULT_UNMATCHED_STEPS, EscalationJudgeConfig, GateTrigger,
+    HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough,
+    PickerMode, PlanExecute, PlanExecuteConfig, Random, StageRouter, StageRouterConfig,
+    SubagentRouter, SubagentRouterConfig, TaskClassifierConfig, ToolSemantics,
 };
 use serde::Deserialize;
 use switchyard_protocol::{Category, ModelId};
@@ -103,6 +103,7 @@ struct CapabilityClassifierRouteConfig {
     weak_target: String,
     base_threshold: f64,
     threshold_step: f64,
+    unmatched_steps: u8,
     classify_trigger: ClassifyTrigger,
     message_hash_fallback: bool,
     recent_turn_window: Option<usize>,
@@ -237,6 +238,9 @@ pub struct LlmClassifierRouteConfig {
     /// Capability mode: how much to raise the threshold when the judge is
     /// uncertain. Added once for an uncertain verdict and twice for unsupported.
     pub threshold_step: Option<f64>,
+    /// Capability mode: threshold steps for an unmatched verdict (no capability rule
+    /// applies). 1 by default, the same as uncertain; 2 treats it like unsupported.
+    pub unmatched_steps: Option<u8>,
     /// How often the judge runs: every request, once per user turn, or once per session.
     pub classify_trigger: ClassifyTrigger,
     /// Reuses the session's target by hashing the first user message when no
@@ -524,6 +528,7 @@ impl StageClassifierConfig {
         TaskClassifierConfig {
             base_threshold: self.base_threshold,
             threshold_step: self.threshold_step,
+            unmatched_steps: DEFAULT_UNMATCHED_STEPS,
             classify_trigger: self.classify_trigger,
             message_hash_fallback: self.message_hash_fallback,
             recent_turn_window: self.recent_turn_window,
@@ -878,6 +883,7 @@ impl LlmClassifierRouteConfig {
             weak_target,
             base_threshold,
             threshold_step,
+            unmatched_steps,
             classify_trigger,
             message_hash_fallback,
             recent_turn_window,
@@ -933,6 +939,7 @@ impl LlmClassifierRouteConfig {
                             base_threshold,
                         )?,
                         threshold_step: threshold_step.unwrap_or_default(),
+                        unmatched_steps: unmatched_steps.unwrap_or(DEFAULT_UNMATCHED_STEPS),
                         classify_trigger: *classify_trigger,
                         message_hash_fallback: *message_hash_fallback,
                         recent_turn_window: *recent_turn_window,
@@ -956,11 +963,15 @@ impl LlmClassifierRouteConfig {
                         "llm_classifier route {route_name} mode escalation cannot use classify_trigger"
                     )));
                 }
-                if mode.is_some()
-                    && (base_threshold.is_some()
-                        || threshold_step.is_some()
-                        || *message_hash_fallback
-                        || recent_turn_window.is_some())
+                // `unmatched_steps` is new, so no existing escalation configuration carries it:
+                // reject it even when the mode is implied by `escalation`. The older capability
+                // keys stay tolerated in that implicit form for compatibility.
+                if unmatched_steps.is_some()
+                    || (mode.is_some()
+                        && (base_threshold.is_some()
+                            || threshold_step.is_some()
+                            || *message_hash_fallback
+                            || recent_turn_window.is_some()))
                 {
                     return Err(AlgorithmConfigError::new(format!(
                         "llm_classifier route {route_name} mode escalation cannot use capability routing settings"
@@ -992,6 +1003,7 @@ impl LlmClassifierRouteConfig {
                     || weak_target.is_some()
                     || base_threshold.is_some()
                     || threshold_step.is_some()
+                    || unmatched_steps.is_some()
                     || escalation.is_some()
                     || *response_format_type != ClassifierResponseFormat::JsonSchema
                 {
@@ -1231,6 +1243,7 @@ fn build_algorithm(
                     let classifier_config = TaskClassifierConfig {
                         base_threshold: config.base_threshold,
                         threshold_step: config.threshold_step,
+                        unmatched_steps: config.unmatched_steps,
                         classify_trigger: config.classify_trigger,
                         message_hash_fallback: config.message_hash_fallback,
                         recent_turn_window: config.recent_turn_window,

crates/switchyard-runner/src/config.rs

@@ -1183,6 +1183,34 @@ new = ["send_message"]
         Ok(())
     }
 
+    #[test]
+    fn unmatched_steps_is_a_capability_setting() -> RunnerResult<()> {
+        // Accepted on a capability route; the value range is checked by the algorithm.
+        runner_from_toml(&VALID_CONFIG.replace(
+            "base_threshold = 0.5",
+            "base_threshold = 0.5\nunmatched_steps = 2",
+        ))?;
+        assert!(
+            error_message(&VALID_CONFIG.replace(
+                "base_threshold = 0.5",
+                "base_threshold = 0.5\nunmatched_steps = 3",
+            ))
+            .contains("unmatched_steps must be 0, 1, or 2")
+        );
+
+        // Rejected on an escalation route, including the implicit form where the
+        // `escalation` table alone selects the mode: the key is new, so no existing
+        // configuration relies on it being ignored there.
+        assert!(
+            error_message(&VALID_CONFIG.replace(
+                "base_threshold = 0.5",
+                "base_threshold = 0.5\nunmatched_steps = 2\nescalation = { confirmations = 2 }",
+            ))
+            .contains("mode escalation cannot use capability routing settings")
+        );
+        Ok(())
+    }
+
     #[test]
     fn a_target_reasoning_effort_parses_and_is_rejected_where_unsupported() -> RunnerResult<()> {
         let strong = "[targets.strong]\nid = \"strong/model\"\nllm_client = \"responses\"";

crates/switchyard-server/README.md

@@ -129,7 +129,8 @@ routes to `weak_target` or `strong_target`. Beyond the three targets it accepts
 | Key | Default | Meaning |
 |---|---|---|
 | `base_threshold` | *required* | Lowest solve probability that routes a task to `weak_target`. Raise it to send less traffic to the weak model. |
-| `threshold_step` | `0.0` | Finite, non-negative amount added once for uncertain or unmatched verdicts and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. |
+| `threshold_step` | `0.0` | Finite, non-negative amount added once for uncertain verdicts, `unmatched_steps` times for unmatched verdicts, and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. |
+| `unmatched_steps` | `1` | Threshold steps for an unmatched verdict. `0`, `1`, or `2`; `2` treats it like unsupported. |
 | `classify_trigger` | `every_request` | When the judge runs. `every_request` judges every request including tool continuations, `user_turn` judges each new user message and holds that target across the tool calls between, `new_session` judges once and reuses that target for the session. |
 | `message_hash_fallback` | `false` | Extends affinity to clients that send no session header, keying on the first user message. Requires `classify_trigger = "new_session"` or `"user_turn"`. |
 

docs/reference/toml_schema.md

@@ -239,7 +239,8 @@ Capability mode classifies before serving. See
 | `strong_target` | Yes | — | Capable tier. |
 | `weak_target` | Yes | — | Efficient tier. |
 | `base_threshold` | Yes | — | Lowest solve probability that routes to the weak target. In `[0, 1]`. |
-| `threshold_step` | No | `0.0` | Finite, non-negative amount added once for uncertain or unmatched verdicts and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. |
+| `threshold_step` | No | `0.0` | Finite, non-negative amount added once for uncertain verdicts, `unmatched_steps` times for unmatched verdicts, and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. |
+| `unmatched_steps` | No | `1` | Threshold steps applied to an unmatched verdict (no capability rule applies). `0`, `1`, or `2`; `2` treats it like an unsupported verdict. |
 | `classify_trigger` | No | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and retains that target across intervening tool calls only when requests carry a session ID; without a session ID, it behaves like `every_request`. `new_session` judges once and reuses that target for the session. |
 | `message_hash_fallback` | No | `false` | Retains the target against a hash of the first user message when a request carries no session ID. Requires `classify_trigger = "new_session"` or `"user_turn"`. |
 | `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. |

docs/routing_algorithms/llm_classifier_routing.md

@@ -68,7 +68,8 @@ greater than or equal to the applicable threshold. Otherwise it routes to
 `strong_target`:
 
 - `supported` uses `base_threshold`.
-- `uncertain` and `unmatched` use `base_threshold + threshold_step`.
+- `uncertain` uses `base_threshold + threshold_step`.
+- `unmatched` uses `base_threshold + unmatched_steps * threshold_step` (`unmatched_steps` defaults to `1`).
 - `unsupported` uses `base_threshold + 2 * threshold_step`.
 
 An invalid, inconsistent, or unparseable verdict routes to
@@ -119,6 +120,7 @@ for the server merge behavior.
 |---|---|---|
 | `base_threshold` | required | Lowest `p_solve` that routes a supported task to `weak_target`. Must be between `0` and `1`. |
 | `threshold_step` | `0.0` | Amount added for each boundary step. Must be finite and non-negative, and `base_threshold + 2 * threshold_step` must not exceed `1`. |
+| `unmatched_steps` | `1` | Boundary steps applied to an `unmatched` verdict. `0`, `1`, or `2`; `2` requires the same confidence as an `unsupported` verdict. |
 | `recent_turn_window` | unset | When unset, the judge sees the opening user task and the latest user message when they differ. When set to `N`, it sees the opening user task and the last `N` conversation messages after that task. `0` keeps only the opening task. Client system and developer instructions are not shown to the judge. |
 | `classify_trigger` | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and holds that target across the tool calls between. `new_session` judges once and reuses that target for the session. |
 | `message_hash_fallback` | `false` | When session metadata is absent, keys affinity from the first user-message text. Requires `classify_trigger = "new_session"` or `"user_turn"`. |