fix(pd): add configurable decode capacity admission limit

lightllm/server/api_cli.py

@@ -68,6 +68,11 @@ def add_cli_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
             "dynamically; use <P>p<D>d for a fixed topology, for example 2p4d. Default: elastic."
         ),
     )
+    parser.add_argument(
+        "--disable_pd_master_decode_capacity_limit",
+        action="store_true",
+        help="Disable PD master admission control based on the total capacity of registered decode nodes.",
+    )
     parser.add_argument(
         "--pd_trans_mode",
         type=str,

lightllm/server/core/objs/start_args_type.py

@@ -22,6 +22,7 @@ class StartArgs:
     pd_master_ip: str = field(default="0.0.0.0")
     pd_master_port: int = field(default=1212)
     pd_master_mode: str = field(default="elastic")
+    disable_pd_master_decode_capacity_limit: bool = field(default=False)
     pd_trans_mode: str = field(default="nccl", metadata={"choices": ["nccl", "nixl"]})
     config_server_host: str = field(default=None)
     config_server_port: int = field(default=None)

lightllm/server/httpserver_for_pd_master/manager.py

@@ -129,6 +129,11 @@ async def generate(
         multimodal_params: MultimodalParams,
         request: Request,
     ):
+        if not self.args.disable_pd_master_decode_capacity_limit:
+            decode_capacity = sum(node.start_args["running_max_req_size"] for node in self.pd_manager.decode_nodes)
+            if self.running_request_count >= decode_capacity:
+                raise ServerBusyError()
+
         was_idle = self.running_request_count == 0
         self.running_request_count += 1
         if was_idle: