A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 03:00 UTC
[Feature] Add `--cpu-distributed-timeout-seconds` CLI Option for CPU Process Group Timeout (#42968)
Signed-off-by: fangyuchu <fangyuchu@qq.com> Signed-off-by: zWaNg3 <389750525@qq.com> Co-authored-by: zWaNg3 <389750525@qq.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
fangyuchu committed
May 22, 2026 at 06:42 UTC
e26e1f09280b6c54e1bc1d1fbc0118f7e309cb10
4 files changed
+33
-1
vllm/config/parallel.py
+4
@@ -292,6 +292,10 @@ class ParallelConfig:
292
timeout parameter. If None, PyTorch's default timeout is used (600s for NCCL).
293
Increase this for multi-node setups where model downloads may be slow."""
294
295
+ cpu_distributed_timeout_seconds: int | None = None
296
+ """Timeout (in seconds) for cpu communication groups. If None, PyTorch's
297
+ default timeout is used (1800s for gloo)."""
298
+
299
world_size: int = Field(init=False)
300
"""world_size is TPxPP, it affects the number of workers we create."""
301
vllm/distributed/parallel_state.py
+7
-1
@@ -335,6 +335,10 @@ class GroupCoordinator:
335
self_device_group = None
336
self_cpu_group = None
337
338
+ from vllm.distributed.utils import get_cpu_distributed_timeout_or_none
339
+
340
+ timeout = get_cpu_distributed_timeout_or_none()
341
+
342
for ranks in group_ranks:
343
device_group = torch.distributed.new_group(
344
ranks, backend=torch_distributed_backend
@@ -342,7 +346,9 @@ class GroupCoordinator:
346
# a group with `gloo` backend, to allow direct coordination between
347
# processes through the CPU.
348
with suppress_stdout():
345
- cpu_group = torch.distributed.new_group(ranks, backend="gloo")
349
+ cpu_group = torch.distributed.new_group(
350
+ ranks, backend="gloo", timeout=timeout
351
+ )
352
if self.rank in ranks:
353
self.ranks = ranks
354
self.world_size = len(ranks)
vllm/distributed/utils.py
+14
@@ -491,6 +491,16 @@ def get_cached_tcp_store_client(host: str, port: int) -> TCPStore:
491
return TCPStore(host, port, is_master=False, wait_for_workers=False)
492
493
494
+def get_cpu_distributed_timeout_or_none() -> timedelta | None:
495
+ from vllm.config import get_current_vllm_config_or_none
496
+
497
+ vllm_config = get_current_vllm_config_or_none()
498
+ if vllm_config is None:
499
+ return None
500
+ timeout_seconds = vllm_config.parallel_config.cpu_distributed_timeout_seconds
501
+ return timedelta(seconds=timeout_seconds) if timeout_seconds is not None else None
502
+
503
+
504
def init_gloo_process_group(
505
prefix_store: PrefixStore,
506
group_rank: int,
@@ -570,6 +580,10 @@ def stateless_init_torch_distributed_process_group(
580
init_method = get_tcp_uri(host, port)
581
backend = Backend(backend) # it is basically string
582
timeout = _get_default_timeout(backend)
583
+ if backend == "gloo":
584
+ gloo_timeout = get_cpu_distributed_timeout_or_none()
585
+ if gloo_timeout is not None:
586
+ timeout = gloo_timeout
587
588
if listen_socket is not None:
589
store = create_tcp_store(
vllm/engine/arg_utils.py
+8
@@ -456,6 +456,9 @@ class EngineArgs:
456
nnodes: int = ParallelConfig.nnodes
457
node_rank: int = ParallelConfig.node_rank
458
distributed_timeout_seconds: int | None = ParallelConfig.distributed_timeout_seconds
459
+ cpu_distributed_timeout_seconds: int | None = (
460
+ ParallelConfig.cpu_distributed_timeout_seconds
461
+ )
462
numa_bind: bool = ParallelConfig.numa_bind
463
numa_bind_nodes: list[int] | None = ParallelConfig.numa_bind_nodes
464
numa_bind_cpus: list[str] | None = ParallelConfig.numa_bind_cpus
@@ -951,6 +954,10 @@ class EngineArgs:
954
"--distributed-timeout-seconds",
955
**parallel_kwargs["distributed_timeout_seconds"],
956
)
957
+ parallel_group.add_argument(
958
+ "--cpu-distributed-timeout-seconds",
959
+ **parallel_kwargs["cpu_distributed_timeout_seconds"],
960
+ )
961
parallel_group.add_argument("--numa-bind", **parallel_kwargs["numa_bind"])
962
parallel_group.add_argument(
963
"--numa-bind-nodes", **parallel_kwargs["numa_bind_nodes"]
@@ -1950,6 +1957,7 @@ class EngineArgs:
1957
nnodes=self.nnodes,
1958
node_rank=self.node_rank,
1959
distributed_timeout_seconds=self.distributed_timeout_seconds,
1960
+ cpu_distributed_timeout_seconds=self.cpu_distributed_timeout_seconds,
1961
data_parallel_master_ip=data_parallel_address,
1962
data_parallel_rpc_port=data_parallel_rpc_port,
1963
data_parallel_backend=self.data_parallel_backend,