A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 04:00 UTC
[Bugfix] Fix inner_dp_world initialization order for multi-node TP (#35892)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nhill@redhat.com>
Yongye Zhu committed
Mar 6, 2026 at 01:04 UTC
86e1060b17d9042ab8f7b7baba26b1d6cbc36c2b
3 files changed
+11
-3
.buildkite/test_areas/distributed.yaml
+3
@@ -67,6 +67,7 @@ steps:
67
- tests/v1/distributed
68
- tests/v1/engine/test_engine_core_client.py
69
- tests/distributed/test_symm_mem_allreduce.py
70
+ - tests/distributed/test_multiproc_executor.py
71
commands:
72
# https://github.com/NVIDIA/nccl/issues/1838
73
- export NCCL_CUMEM_HOST_ENABLE=0
@@ -95,6 +96,8 @@ steps:
96
- pytest -v -s distributed/test_pynccl.py
97
- pytest -v -s distributed/test_events.py
98
- pytest -v -s distributed/test_symm_mem_allreduce.py
99
+ # test multi-node TP with multiproc executor (simulated on single node)
100
+ - pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node
101
# TODO: create a dedicated test section for multi-GPU example tests
102
# when we have multiple distributed example tests
103
# OLD rlhf examples
tests/distributed/test_multiproc_executor.py
+4
-2
@@ -9,11 +9,11 @@ focusing on executor initialization, RPC calls, and distributed execution.
9
10
import multiprocessing
11
import os
12
+import socket
13
14
from tests.utils import multi_gpu_test
15
from vllm.config import VllmConfig
16
from vllm.engine.arg_utils import EngineArgs
16
-from vllm.utils import get_open_port
17
from vllm.v1.core.sched.output import SchedulerOutput
18
from vllm.v1.executor.multiproc_executor import MultiprocExecutor
19
@@ -333,7 +333,9 @@ def test_multiproc_executor_multi_node():
333
- Node 1 (rank 1): Uses GPUs 2,3 (CUDA_VISIBLE_DEVICES=2,3) with TP=2
334
Total world_size = 4, nnodes = 2
335
"""
336
- port = get_open_port()
336
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
337
+ s.bind(("", 0))
338
+ port = s.getsockname()[1]
339
# symm_mem does not work for simulating multi instance in single node
340
os.environ["VLLM_ALLREDUCE_USE_SYMM_MEM"] = "0"
341
vllm/v1/executor/multiproc_executor.py
+4
-1
@@ -608,7 +608,6 @@ class WorkerProc:
608
)
609
610
# Load model
611
- self._init_message_queues(input_shm_handle, vllm_config)
611
is_eep_new_worker = envs.VLLM_ELASTIC_EP_SCALE_UP_LAUNCH
612
if not is_eep_new_worker:
613
self.worker.init_device()
@@ -618,6 +617,10 @@ class WorkerProc:
617
)
618
self.worker.load_model()
619
620
+ # Initialize message queues after init_device() since multi-node setups
621
+ # (nnodes_within_dp > 1) require distributed groups to be initialized
622
+ self._init_message_queues(input_shm_handle, vllm_config)
623
+
624
# Enable environment variable cache (e.g. assume no more
625
# environment variable overrides after this point)
626
enable_envs_cache()