A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 03:45 UTC
[CI] Fix "2 Node Tests (4 GPUs in total)" (#31090)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Lucas Wilkinson committed
Dec 21, 2025 at 21:32 UTC
7e065eba596e1fd273c765358f5cb34468bffb17
4 files changed
+51
-27
.buildkite/test-amd.yaml
+2
-2
@@ -1254,13 +1254,13 @@ steps:
1254
- # the following commands are for the first node, with ip 192.168.10.10 (ray environment already set up)
1255
- VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed'
1256
- NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed'
1257
- - python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --nnodes=2 --node-rank=0 --master-addr=192.168.10.10 --master-port=12345 --enforce-eager --trust-remote-code
1257
+ - python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=0 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code
1258
- VLLM_MULTI_NODE=1 pytest -v -s distributed/test_multi_node_assignment.py
1259
- VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py
1260
- # the following commands are for the second node, with ip 192.168.10.11 (ray environment already set up)
1261
- VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed'
1262
- NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed'
1263
- - python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --nnodes=2 --node-rank=1 --master-addr=192.168.10.10 --master-port=12345 --enforce-eager --trust-remote-code
1263
+ - python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=1 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code
1264
1265
- label: Distributed Tests (2 GPUs) # 68min
1266
timeout_in_minutes: 90
.buildkite/test-pipeline.yaml
+2
-2
@@ -1109,13 +1109,13 @@ steps:
1109
- # the following commands are for the first node, with ip 192.168.10.10 (ray environment already set up)
1110
- VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed'
1111
- NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed'
1112
- - python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --nnodes=2 --node-rank=0 --master-addr=192.168.10.10 --master-port=12345 --enforce-eager --trust-remote-code
1112
+ - python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=0 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code
1113
- VLLM_MULTI_NODE=1 pytest -v -s distributed/test_multi_node_assignment.py
1114
- VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py
1115
- # the following commands are for the second node, with ip 192.168.10.11 (ray environment already set up)
1116
- VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed'
1117
- NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed'
1118
- - python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --nnodes=2 --node-rank=1 --master-addr=192.168.10.10 --master-port=12345 --enforce-eager --trust-remote-code
1118
+ - python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=1 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code
1119
1120
- label: Distributed Tests (2 GPUs) # 68min
1121
timeout_in_minutes: 90
.buildkite/test_areas/distributed.yaml
+1
-1
@@ -171,7 +171,7 @@ steps:
171
- tests/distributed/
172
- tests/examples/offline_inference/data_parallel.py
173
commands:
174
- - ./.buildkite/scripts/run-multi-node-test.sh /vllm-workspace/tests 2 2 public.ecr.aws/q9t5s3a7/vllm-ci-postmerge-repo:0bec63fa317e1fbd62e19b0fc31c43c81bf89077 "VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed' && NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed' && python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --nnodes=2 --node-rank=0 --master-addr=192.168.10.10 --master-port=12345 --enforce-eager --trust-remote-code && VLLM_MULTI_NODE=1 pytest -v -s distributed/test_multi_node_assignment.py && VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py" "VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed' && NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed' && python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --nnodes=2 --node-rank=1 --master-addr=192.168.10.10 --master-port=12345 --enforce-eager --trust-remote-code"
174
+ - ./.buildkite/scripts/run-multi-node-test.sh /vllm-workspace/tests 2 2 public.ecr.aws/q9t5s3a7/vllm-ci-postmerge-repo:0bec63fa317e1fbd62e19b0fc31c43c81bf89077 "VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed' && NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed' && python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=0 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code && VLLM_MULTI_NODE=1 pytest -v -s distributed/test_multi_node_assignment.py && VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py" "VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed' && NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed' && python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=1 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code"
175
176
- label: Distributed NixlConnector PD accuracy (4 GPUs)
177
timeout_in_minutes: 30
examples/offline_inference/data_parallel.py
+46
-22
@@ -14,19 +14,19 @@ Multi-node:
14
--model="ibm-research/PowerMoE-3b" \
15
-dp=2 \
16
-tp=2 \
17
- --nnodes=2 \
18
- --node-rank=0 \
19
- --master-addr=10.99.48.128 \
20
- --master-port=13345
17
+ --dp-num-nodes=2 \
18
+ --dp-node-rank=0 \
19
+ --dp-master-addr=10.99.48.128 \
20
+ --dp-master-port=13345
21
Node 1:
22
python examples/offline_inference/data_parallel.py \
23
--model="ibm-research/PowerMoE-3b" \
24
-dp=2 \
25
-tp=2 \
26
- --nnodes=2 \
27
- --node-rank=1 \
28
- --master-addr=10.99.48.128 \
29
- --master-port=13345
26
+ --dp-num-nodes=2 \
27
+ --dp-node-rank=1 \
28
+ --dp-master-addr=10.99.48.128 \
29
+ --dp-master-port=13345
30
"""
31
32
import os
@@ -48,7 +48,31 @@ def create_parser():
48
enable_expert_parallel=True,
49
)
50
51
- # Add timeout (not in EngineArgs)
51
+ # Add DP-specific args (separate from engine args to avoid conflicts)
52
+ parser.add_argument(
53
+ "--dp-num-nodes",
54
+ type=int,
55
+ default=1,
56
+ help="Total number of nodes for data parallel.",
57
+ )
58
+ parser.add_argument(
59
+ "--dp-node-rank",
60
+ type=int,
61
+ default=0,
62
+ help="Rank of the current node for data parallel.",
63
+ )
64
+ parser.add_argument(
65
+ "--dp-master-addr",
66
+ type=str,
67
+ default="",
68
+ help="Master node IP address for DP coordination.",
69
+ )
70
+ parser.add_argument(
71
+ "--dp-master-port",
72
+ type=int,
73
+ default=0,
74
+ help="Master node port for DP coordination.",
75
+ )
76
parser.add_argument(
77
"--timeout",
78
type=int,
@@ -132,26 +156,26 @@ if __name__ == "__main__":
156
parser = create_parser()
157
args = vars(parser.parse_args())
158
135
- # Extract DP-specific args
159
+ # Extract DP-specific args (pop to remove from engine_args)
160
dp_size = args.pop("data_parallel_size")
137
- nnodes = args.get("nnodes", 1)
138
- node_rank = args.get("node_rank", 0)
139
- master_addr = args.get("master_addr", "")
140
- master_port = args.get("master_port", 0)
161
+ dp_num_nodes = args.pop("dp_num_nodes")
162
+ dp_node_rank = args.pop("dp_node_rank")
163
+ dp_master_addr = args.pop("dp_master_addr")
164
+ dp_master_port = args.pop("dp_master_port")
165
timeout = args.pop("timeout")
166
167
# Remaining args are engine args
168
engine_args = args
169
146
- if nnodes == 1:
170
+ if dp_num_nodes == 1:
171
dp_master_ip = "127.0.0.1"
148
- dp_master_port = get_open_port()
172
+ dp_master_port_val = get_open_port()
173
else:
150
- dp_master_ip = master_addr
151
- dp_master_port = master_port
174
+ dp_master_ip = dp_master_addr
175
+ dp_master_port_val = dp_master_port
176
153
- assert dp_size % nnodes == 0, "dp_size should be divisible by nnodes"
154
- dp_per_node = dp_size // nnodes
177
+ assert dp_size % dp_num_nodes == 0, "dp_size should be divisible by dp_num_nodes"
178
+ dp_per_node = dp_size // dp_num_nodes
179
180
from multiprocessing import Process
181
@@ -162,7 +186,7 @@ if __name__ == "__main__":
186
187
procs = []
188
for local_dp_rank, global_dp_rank in enumerate(
165
- range(node_rank * dp_per_node, (node_rank + 1) * dp_per_node)
189
+ range(dp_node_rank * dp_per_node, (dp_node_rank + 1) * dp_per_node)
190
):
191
proc = Process(
192
target=main,
@@ -171,7 +195,7 @@ if __name__ == "__main__":
195
local_dp_rank,
196
global_dp_rank,
197
dp_master_ip,
174
- dp_master_port,
198
+ dp_master_port_val,
199
engine_args,
200
),
201
)