A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 12:30 UTC
Enabling "2 node" distributed tests in the AMD CI pipeline. (#32719)
Signed-off-by: DCCS-4560 <alivanov@chi-mi325x-pod1-112.ord.vultr.cpe.ice.amd.com> Co-authored-by: DCCS-4560 <alivanov@chi-mi325x-pod1-112.ord.vultr.cpe.ice.amd.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
Alexei-V-Ivanov-AMD committed
Jan 27, 2026 at 13:13 UTC
3c3c547ce0b35fb9d43808e8609f5e86fc34cca1
2 files changed
+46
-6
.buildkite/scripts/hardware_ci/run-amd-test.sh
+40
@@ -44,6 +44,17 @@ cleanup_docker() {
44
fi
45
}
46
47
+cleanup_network() {
48
+ for node in $(seq 0 $((NUM_NODES-1))); do
49
+ if docker pr -a -q -f name="node${node}" | grep -q .; then
50
+ docker stop "node${node}"
51
+ fi
52
+ done
53
+ if docker network ls | grep docker-net; then
54
+ docker network rm docker-net
55
+ fi
56
+}
57
+
58
# Call the cleanup docker function
59
cleanup_docker
60
@@ -224,6 +235,35 @@ if [[ $commands == *"--shard-id="* ]]; then
235
echo "All shards reported no tests collected. Failing the build."
236
exit 1
237
fi
238
+
239
+elif [[ $commands == *"VLLM_TEST_GROUP_NAME=mi325_4-2-node-tests-4-gpus-in-total"* ]]; then
240
+
241
+ export DCKR_VER=$(docker --version | sed 's/Docker version \(.*\), build .*/\1/')
242
+
243
+ if [[ "$commands" =~ ^(.*)"["(.*)"] && ["(.*)"]"$ ]]; then
244
+ prefix=$( echo "${BASH_REMATCH[1]}" | sed 's/;//g')
245
+ echo "PREFIX: ${prefix}"
246
+ export composite_command="(command rocm-smi || true)"
247
+ myIFS=$IFS
248
+ IFS=','
249
+ read -ra node0 <<< ${BASH_REMATCH[2]}
250
+ read -ra node1 <<< ${BASH_REMATCH[3]}
251
+ IFS=$myIFS
252
+ for i in "${!node0[@]}";do
253
+ command_node_0=$(echo ${node0[i]} | sed 's/\"//g')
254
+ command_node_1=$(echo ${node1[i]} | sed 's/\"//g')
255
+
256
+ export commands="./.buildkite/scripts/run-multi-node-test.sh /vllm-workspace/tests 2 2 ${image_name} '${command_node_0}' '${command_node_1}'"
257
+ echo "COMMANDS: ${commands}"
258
+ composite_command=$(echo "${composite_command} && ${commands}")
259
+ done
260
+ /bin/bash -c "${composite_command}"
261
+ cleanup_network
262
+ else
263
+ echo "Failed to parse node commands! Exiting."
264
+ cleanup_network
265
+ exit 111
266
+ fi
267
else
268
echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
269
docker run \
.buildkite/test-amd.yaml
+6
-6
@@ -1278,7 +1278,7 @@ steps:
1278
1279
- label: 2 Node Tests (4 GPUs in total) # 16min
1280
timeout_in_minutes: 30
1281
- mirror_hardwares: [amdexperimental]
1281
+ mirror_hardwares: [amdexperimental, amdmultinode]
1282
agent_pool: mi325_4
1283
# grade: Blocking
1284
working_dir: "/vllm-workspace/tests"
@@ -1292,15 +1292,15 @@ steps:
1292
- tests/distributed/
1293
- tests/examples/offline_inference/data_parallel.py
1294
commands:
1295
- - # the following commands are for the first node, with ip 192.168.10.10 (ray environment already set up)
1296
- - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed'
1297
- - NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed'
1295
+ - # the following commands are for the first node, with ip 192.168.10.10 (ray environment already set up) | grep 'Same node test passed' | grep 'Node count test passed'
1296
+ - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
1297
+ - NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py
1298
- python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=0 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code
1299
- VLLM_MULTI_NODE=1 pytest -v -s distributed/test_multi_node_assignment.py
1300
- VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py
1301
- # the following commands are for the second node, with ip 192.168.10.11 (ray environment already set up)
1302
- - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed'
1303
- - NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed'
1302
+ - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
1303
+ - NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py
1304
- python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=1 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code
1305
1306
- label: Distributed Tests (2 GPUs) # 68min