From d2450b89304a8279d16c51cdbcbca39391bf658c Mon Sep 17 00:00:00 2001 From: Lingjie Wu Date: Wed, 5 Aug 2026 10:51:27 +0800 Subject: [PATCH] Tolerate ADP pad-dummy surplus instead of asserting Every disaggregated attention-DP generation cell of Qwen3.5-397B dies on py_executor.py _pad_attention_dp_dummy_request assert self.expected_num_active_requests >= len(self.active_requests) Several ranks lose their event loop at once and the survivors HangDetector-abort ~300 s later. Attention DP is required to reproduce: the method returns early when it is off, and 54 TEP cells never hit it while 6 of 7 ADP cells did. The router derives the expectation as min(max(ceil(multiplier * fair_share), max(per_rank_loads)), max_num_active_requests) (AttentionDpRouter._expected_num_active_requests). The per-rank-load floor normally keeps the value at or above this rank's own load, but the hard cap is applied last, so any rank holding max_num_active_requests + 1 requests breaks the relation. An attention-DP pad dummy does exactly that when it survives an iteration that was skipped fleet-wide: can_queue False skips both _forward_step and _update_request_states, _update_request_states_tp is the only place the dummy is removed, and the next gather_all_rank_states counts the survivor. The measured signature matches that derivation. Across 13 cells (dep8, dep16, dep32; max_batch_size 4 to 64), 1132 occurrences: len(active_requests) - expected_num_active_requests is exactly 1 every single time, and expected_num_active_requests always equals max_batch_size. Inside this method the expectation is consumed only by the idle-rank test, and a rank holding surplus requests needs no dummy, so tolerate the surplus and warn instead of asserting. The tolerated value is clamped into a local so downstream consumers still observe the router's number. Returning early would be wrong: _count_schedulable_active_requests excludes requests still in KV transfer, so a rank with many active requests can still have none schedulable and legitimately need a dummy. Also add qwen3_5_moe to the gate that scopes the existing ADP dummy fixes. Those branches all sit after the assert, so widening the gate alone does not stop the crash - measured, the run still died on the same line with the widened gate in place - but the model does need them. should_enable_dsv4_overlap_headroom is deliberately pinned to deepseek_v4 rather than reusing the widened gate, because it doubles max_num_sequences and changes the memory envelope. Validated on GB300 disaggregated serving (Qwen3.5-397B-A17B-NVFP4-V2, 9 ADP cells, 900 to 3600 s each): the tolerance fires 10 to 273 times per cell, 0 AssertionError, and 66352 decode iterations on a previously fatal cell. Signed-off-by: Lingjie Wu --- tensorrt_llm/_torch/pyexecutor/_util.py | 22 +++++++++--- tensorrt_llm/_torch/pyexecutor/py_executor.py | 31 ++++++++++++++-- .../_torch/executor/test_py_executor.py | 35 +++++++++++++++++++ .../_torch/executor/test_seq_slot_sizing.py | 7 ++++ 4 files changed, 89 insertions(+), 6 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/_util.py b/tensorrt_llm/_torch/pyexecutor/_util.py index be555fa55c42..ca2602d6fba1 100644 --- a/tensorrt_llm/_torch/pyexecutor/_util.py +++ b/tensorrt_llm/_torch/pyexecutor/_util.py @@ -2531,17 +2531,31 @@ def compute_max_num_sequences(mapping: Mapping, return max_batch_size * num_micro_batches +# Model types whose disaggregated attention-DP path has been measured against +# the ADP dummy fixes. The gate stays an explicit list rather than a capability +# check (``enable_attention_dp and kv_cache_transceiver is not None``) so that +# each entry is added only after its disagg ADP behavior has been exercised. +_ADP_DUMMY_FIX_MODEL_TYPES = ("deepseek_v4", "qwen3_5_moe") + + def should_enable_dsv4_adp_dummy_fixes(model_type: Optional[str], mapping: Mapping) -> bool: - """Gate DSv4 ADP dummy behavior while PP remains follow-up scope.""" - return model_type == "deepseek_v4" and not mapping.has_pp() + """Gate the ADP dummy fixes while PP remains follow-up scope.""" + return model_type in _ADP_DUMMY_FIX_MODEL_TYPES and not mapping.has_pp() def should_enable_dsv4_overlap_headroom( model_type: Optional[str], spec_config: Optional[SpeculativeConfig], mapping: Mapping, disable_overlap_scheduler: bool) -> bool: - """Gate extra sequence slots to the validated DSv4 MTP overlap path.""" - return (should_enable_dsv4_adp_dummy_fixes(model_type, mapping) + """Gate extra sequence slots to the validated DSv4 MTP overlap path. + + Deliberately NOT routed through ``should_enable_dsv4_adp_dummy_fixes``. + That gate now covers more model types, while this one doubles + ``max_num_sequences`` (see ``compute_max_num_sequences``) and therefore + changes the memory envelope; it must stay pinned to the one path it was + measured on. + """ + return (model_type == "deepseek_v4" and not mapping.has_pp() and spec_config is not None and spec_config.spec_dec_mode.is_mtp_eagle_one_model() and not disable_overlap_scheduler) diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 2fa5a5d0607b..6cf98b759f02 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -5870,13 +5870,40 @@ def _pad_attention_dp_dummy_request(self): if not self.enable_attention_dp: return - assert self.expected_num_active_requests >= len(self.active_requests) + expected_num_active_requests = self.expected_num_active_requests + if expected_num_active_requests < len(self.active_requests): + # Not fatal, and not a capacity violation. The router derives this + # value as + # min(max(ceil(multiplier * fair_share), max(per_rank_loads)), + # max_num_active_requests) + # (adp_router.AttentionDpRouter._expected_num_active_requests), so + # the per-rank-load floor normally keeps it at or above this rank's + # own load -- but the hard cap is applied last. Any rank that ends + # up holding max_num_active_requests + 1 requests therefore breaks + # the relation, e.g. when an attention-DP pad dummy survives an + # iteration that was skipped fleet-wide (can_queue False skips both + # _forward_step and _update_request_states, and + # _update_request_states_tp is the only place the dummy is removed) + # and the next gather_all_rank_states counts it. + # + # Inside this method the value is consumed only by the idle-rank + # test below, and a rank holding surplus requests needs no dummy, + # so tolerate the surplus. Asserting here took down the executor + # event loop on every affected rank at once, leaving the survivors + # to HangDetector-abort. + logger.warning( + f"active_requests ({len(self.active_requests)}) exceeds " + f"expected_num_active_requests " + f"({expected_num_active_requests}); tolerating (a busy rank " + f"needs no attention-DP dummy).") + expected_num_active_requests = len(self.active_requests) + num_active_request = self._count_schedulable_active_requests() if self._should_skip_dummy_for_benchmark_disagg(num_active_request): return - needs_dummy = (self.expected_num_active_requests > 0 + needs_dummy = (expected_num_active_requests > 0 and num_active_request == 0) if not needs_dummy: return diff --git a/tests/unittest/_torch/executor/test_py_executor.py b/tests/unittest/_torch/executor/test_py_executor.py index c9c05b39137f..bd9f4ea2f634 100644 --- a/tests/unittest/_torch/executor/test_py_executor.py +++ b/tests/unittest/_torch/executor/test_py_executor.py @@ -1489,6 +1489,41 @@ def test_pad_dummy_added_when_only_wait_scheduler_requests_disagg(): assert len(stub.active_requests) == 2 +def test_pad_dummy_tolerates_surplus_over_expected_on_busy_rank() -> None: + # expected_num_active_requests is capped at max_num_active_requests after + # the per-rank-load floor is applied, so a rank can legitimately end up + # holding more requests than the router expected -- e.g. when a pad dummy + # survives an iteration that was skipped fleet-wide and the next + # gather_all_rank_states counts it. This used to trip a bare assert and + # kill the executor event loop; a busy rank needs no dummy, so it must be + # tolerated instead. + stub = _StubADPExecutor() + stub.active_requests = [_make_adp_request(_STATE_GENERATION_IN_PROGRESS) for _ in range(3)] + stub.expected_num_active_requests = 2 + + _run_pad(stub) + + assert stub.add_dummy_calls == [] + assert len(stub.active_requests) == 3 + # Tolerating must not leak a mutated expectation to downstream consumers. + assert stub.expected_num_active_requests == 2 + + +def test_pad_dummy_still_added_when_surplus_requests_are_unschedulable() -> None: + # Tolerating the surplus must not short-circuit padding. A rank can hold + # more requests than expected AND have none of them schedulable (all parked + # at GENERATION_TO_COMPLETE), in which case it still schedules batch=0 and + # needs a dummy to stay in the forward-pass collectives. + stub = _StubADPExecutor() + stub.active_requests = [_make_adp_request(_STATE_GENERATION_TO_COMPLETE) for _ in range(3)] + stub.expected_num_active_requests = 2 + + _run_pad(stub) + + assert len(stub.add_dummy_calls) == 1 + assert stub.expected_num_active_requests == 2 + + def test_pad_dummy_allocation_failure_skips_padding(): # add_dummy_requests returns None when the rank has no free cache # resources for even a 1-token dummy (possible while non-schedulable diff --git a/tests/unittest/_torch/executor/test_seq_slot_sizing.py b/tests/unittest/_torch/executor/test_seq_slot_sizing.py index d42e6f4483c9..6db765ac4a79 100644 --- a/tests/unittest/_torch/executor/test_seq_slot_sizing.py +++ b/tests/unittest/_torch/executor/test_seq_slot_sizing.py @@ -48,6 +48,10 @@ ("deepseek_v4", True, False, 1, False, False), ("deepseek_v4", True, True, 2, False, False), ("deepseek_v4", True, True, 1, True, False), + # The widened ADP dummy gate must not leak into the headroom gate: + # doubling max_num_sequences changes the memory envelope and has only + # been validated on the DSv4 MTP overlap path. + ("qwen3_5_moe", True, True, 1, False, False), ], ) def test_dsv4_overlap_headroom_gate( @@ -71,6 +75,9 @@ def test_dsv4_overlap_headroom_gate( ("deepseek_v4", 1, True), ("deepseek_v3", 1, False), ("deepseek_v4", 2, False), + ("qwen3_5_moe", 1, True), + ("qwen3_5_moe", 2, False), + ("llama", 1, False), ], ) def test_dsv4_adp_dummy_fix_gate(model_type, pp_size, expected):