fix(distributed): propagate distributed_timeout_seconds to NCCL device groups (#45159)
Signed-off-by: jialoop-git <joane8913456@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
jialoop-git committed
Jul 7, 2026 at 10:52 UTC
a4f019fa25daf422d36804fac1eac1ebebf21d9f
2 files changed
+36
-3
vllm/distributed/parallel_state.py
+22
-3
@@ -269,11 +269,17 @@ def _create_subgroups_split_group(
269
must enter with the same ``split_ranks`` definition. Each rank receives
270
the subgroup it belongs to.
271
"""
272
+ from vllm.distributed.utils import (
273
+ get_cpu_distributed_timeout_or_none,
274
+ get_distributed_timeout_or_none,
275
+ )
276
+
277
device_backend_str = _device_backend_str(torch_distributed_backend)
278
self_device_group = torch.distributed.split_group(
279
split_ranks=group_ranks,
280
group_desc=f"{group_name}:device",
281
backend=device_backend_str,
282
+ timeout=get_distributed_timeout_or_none(),
283
)
284
# CPU subgroup: split_group requires the requested backend filter to
285
# include the parent's default device type (= the device the parent PG
@@ -284,6 +290,7 @@ def _create_subgroups_split_group(
290
split_ranks=group_ranks,
291
group_desc=f"{group_name}:cpu",
292
backend=f"cpu:gloo,{device_backend_str}",
293
+ timeout=get_cpu_distributed_timeout_or_none(),
294
)
295
return self_device_group, self_cpu_group
296
@@ -417,13 +424,19 @@ class GroupCoordinator:
424
self.rank_in_group = ranks.index(self.rank)
425
break
426
else:
420
- from vllm.distributed.utils import get_cpu_distributed_timeout_or_none
427
+ from vllm.distributed.utils import (
428
+ get_cpu_distributed_timeout_or_none,
429
+ get_distributed_timeout_or_none,
430
+ )
431
432
timeout = get_cpu_distributed_timeout_or_none()
433
+ device_timeout = get_distributed_timeout_or_none()
434
435
for ranks in group_ranks:
436
device_group = torch.distributed.new_group(
426
- ranks, backend=torch_distributed_backend
437
+ ranks,
438
+ backend=torch_distributed_backend,
439
+ timeout=device_timeout,
440
)
441
# a group with `gloo` backend, to allow direct coordination between
442
# processes through the CPU.
@@ -504,10 +517,16 @@ class GroupCoordinator:
517
This is a collective call: every world rank must invoke it. Used where we
518
want to issue ops that can run concurrently with ops on `device_group`.
519
"""
520
+ from vllm.distributed.utils import get_distributed_timeout_or_none
521
+
522
+ device_timeout = get_distributed_timeout_or_none()
523
sibling: ProcessGroup | None = None
524
for ranks in self.group_ranks:
525
pg = torch.distributed.new_group(
510
- ranks, backend=self.torch_distributed_backend, group_desc=group_desc
526
+ ranks,
527
+ backend=self.torch_distributed_backend,
528
+ group_desc=group_desc,
529
+ timeout=device_timeout,
530
)
531
if self.rank in ranks:
532
sibling = pg
vllm/distributed/utils.py
+14
@@ -533,6 +533,16 @@ def get_cpu_distributed_timeout_or_none() -> timedelta | None:
533
return timedelta(seconds=timeout_seconds) if timeout_seconds is not None else None
534
535
536
+def get_distributed_timeout_or_none() -> timedelta | None:
537
+ from vllm.config import get_current_vllm_config_or_none
538
+
539
+ vllm_config = get_current_vllm_config_or_none()
540
+ if vllm_config is None:
541
+ return None
542
+ timeout_seconds = vllm_config.parallel_config.distributed_timeout_seconds
543
+ return timedelta(seconds=timeout_seconds) if timeout_seconds is not None else None
544
+
545
+
546
def init_gloo_process_group(
547
prefix_store: PrefixStore,
548
group_rank: int,
@@ -616,6 +626,10 @@ def stateless_init_torch_distributed_process_group(
626
gloo_timeout = get_cpu_distributed_timeout_or_none()
627
if gloo_timeout is not None:
628
timeout = gloo_timeout
629
+ else:
630
+ device_timeout = get_distributed_timeout_or_none()
631
+ if device_timeout is not None:
632
+ timeout = device_timeout
633
634
if listen_socket is not None:
635
store = create_tcp_store(