A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 04:00 UTC
[Feature] Add --distributed-timeout-seconds CLI option (#36047)
Signed-off-by: Shiyan Deng <dsy842974287@meta.com> Co-authored-by: Lu Fang <30275821+houseroad@users.noreply.github.com>
Shiyan Deng committed
Mar 5, 2026 at 20:57 UTC
03a49bb8f0c8ad3472a61ec163167898fda02917
3 files changed
+25
-2
vllm/config/parallel.py
+7
-1
@@ -234,9 +234,15 @@ class ParallelConfig:
234
"""distributed node rank for multi-node distributed
235
inference when distributed_executor_backend is mp."""
236
nnodes: int = 1
237
- """num of nodes for multi-node distributed
237
+ """num of nodes for multi-node distributed
238
inference when distributed_executor_backend is mp."""
239
240
+ distributed_timeout_seconds: int | None = None
241
+ """Timeout in seconds for distributed operations (e.g., init_process_group).
242
+ If set, this value is passed to torch.distributed.init_process_group as the
243
+ timeout parameter. If None, PyTorch's default timeout is used (600s for NCCL).
244
+ Increase this for multi-node setups where model downloads may be slow."""
245
+
246
world_size: int = Field(init=False)
247
"""world_size is TPxPP, it affects the number of workers we create."""
248
vllm/engine/arg_utils.py
+6
@@ -403,6 +403,7 @@ class EngineArgs:
403
master_port: int = ParallelConfig.master_port
404
nnodes: int = ParallelConfig.nnodes
405
node_rank: int = ParallelConfig.node_rank
406
+ distributed_timeout_seconds: int | None = ParallelConfig.distributed_timeout_seconds
407
tensor_parallel_size: int = ParallelConfig.tensor_parallel_size
408
prefill_context_parallel_size: int = ParallelConfig.prefill_context_parallel_size
409
decode_context_parallel_size: int = ParallelConfig.decode_context_parallel_size
@@ -814,6 +815,10 @@ class EngineArgs:
815
parallel_group.add_argument("--master-port", **parallel_kwargs["master_port"])
816
parallel_group.add_argument("--nnodes", "-n", **parallel_kwargs["nnodes"])
817
parallel_group.add_argument("--node-rank", "-r", **parallel_kwargs["node_rank"])
818
+ parallel_group.add_argument(
819
+ "--distributed-timeout-seconds",
820
+ **parallel_kwargs["distributed_timeout_seconds"],
821
+ )
822
parallel_group.add_argument(
823
"--tensor-parallel-size", "-tp", **parallel_kwargs["tensor_parallel_size"]
824
)
@@ -1701,6 +1706,7 @@ class EngineArgs:
1706
master_port=self.master_port,
1707
nnodes=self.nnodes,
1708
node_rank=self.node_rank,
1709
+ distributed_timeout_seconds=self.distributed_timeout_seconds,
1710
data_parallel_master_ip=data_parallel_address,
1711
data_parallel_rpc_port=data_parallel_rpc_port,
1712
data_parallel_backend=self.data_parallel_backend,
vllm/v1/worker/gpu_worker.py
+12
-1
@@ -6,6 +6,7 @@ import gc
6
import os
7
from collections.abc import Callable
8
from contextlib import AbstractContextManager, nullcontext
9
+from datetime import timedelta
10
from types import NoneType
11
from typing import TYPE_CHECKING, Any
12
@@ -942,8 +943,18 @@ def init_worker_distributed_environment(
943
set_custom_all_reduce(not parallel_config.disable_custom_all_reduce)
944
945
init_method = distributed_init_method or "env://"
946
+
947
+ timeout = None
948
+ if parallel_config.distributed_timeout_seconds is not None:
949
+ timeout = timedelta(seconds=parallel_config.distributed_timeout_seconds)
950
+
951
init_distributed_environment(
946
- parallel_config.world_size, rank, init_method, local_rank, backend
952
+ parallel_config.world_size,
953
+ rank,
954
+ init_method,
955
+ local_rank,
956
+ backend,
957
+ timeout,
958
)
959
960
ensure_model_parallel_initialized(