From b98f680291405910d629056eeac5cb1e30297057 Mon Sep 17 00:00:00 2001 From: WANDY666 <1060304770@qq.com> Date: Tue, 25 Aug 2026 14:46:20 +0000 Subject: [PATCH] fix(dp): serialize DP collectives after overlap forward The next infer thread may launch DP all-gathers before the previous overlap-stream forward finishes, causing NCCL to overlap with DeepEP/MTP kernels. Make the current stream wait before starting DP collectives while preserving overlap for request preparation. --- .../router/model_infer/mode_backend/dp_backend/impl.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lightllm/server/router/model_infer/mode_backend/dp_backend/impl.py b/lightllm/server/router/model_infer/mode_backend/dp_backend/impl.py index 9a81927bc..186229b06 100644 --- a/lightllm/server/router/model_infer/mode_backend/dp_backend/impl.py +++ b/lightllm/server/router/model_infer/mode_backend/dp_backend/impl.py @@ -130,6 +130,12 @@ def infer_loop(self): recover_paused=self.control_state_machine.try_recover_paused_reqs(), ) + if self.support_overlap: + # The previous infer thread releases forward before its GPU work finishes. + # Keep the next DP collective behind that work to avoid overlapping NCCL + # with the previous DeepEP/MTP kernels. + torch.cuda.current_stream().wait_stream(g_infer_context.get_overlap_stream()) + dp_prefill_req_nums, dp_decode_req_nums = self._dp_all_gather_prefill_and_decode_req_num( prefill_reqs=prefill_reqs, decode_reqs=decode_reqs )