A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 22:45 UTC
[Hardware][AMD][CI] Refactor AMD tests to properly use BuildKite parallelism (#32745)
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
Matt committed
Feb 4, 2026 at 00:51 UTC
08e094997ecb28b4c5ad4dd28fd6fbb48adba279
2 files changed
+14
-61
.buildkite/scripts/hardware_ci/run-amd-test.sh
+5
-52
@@ -87,7 +87,7 @@ mkdir -p "${HF_CACHE}"
87
HF_MOUNT="/root/.cache/huggingface"
88
89
commands=$@
90
-echo "Commands:$commands"
90
+echo "Raw commands: $commands"
91
92
commands=${commands//"pytest -v -s basic_correctness/test_basic_correctness.py"/"pytest -v -s basic_correctness/test_basic_correctness.py"}
93
@@ -169,6 +169,9 @@ if [[ $commands == *" entrypoints/llm "* ]]; then
169
--ignore=entrypoints/llm/test_prompt_validation.py "}
170
fi
171
172
+commands=$(echo "$commands" | sed 's/ \\ / /g')
173
+echo "Final commands: $commands"
174
+
175
# --ignore=entrypoints/openai/test_encoder_decoder.py \
176
# --ignore=entrypoints/openai/test_embedding.py \
177
# --ignore=entrypoints/openai/test_oot_registration.py
@@ -176,7 +179,6 @@ fi
179
# --ignore=entrypoints/openai/test_models.py <= Fails on MI250 but passes on MI300 as of 2025-03-13
180
181
179
-PARALLEL_JOB_COUNT=8
182
MYPYTHONPATH=".."
183
184
# Test that we're launching on the machine that has
@@ -187,56 +189,7 @@ if [[ -z "$render_gid" ]]; then
189
exit 1
190
fi
191
190
-# check if the command contains shard flag, we will run all shards in parallel because the host have 8 GPUs.
191
-if [[ $commands == *"--shard-id="* ]]; then
192
- # assign job count as the number of shards used
193
- commands=$(echo "$commands" | sed -E "s/--num-shards[[:blank:]]*=[[:blank:]]*[0-9]*/--num-shards=${PARALLEL_JOB_COUNT} /g" | sed 's/ \\ / /g')
194
- for GPU in $(seq 0 $(($PARALLEL_JOB_COUNT-1))); do
195
- # assign shard-id for each shard
196
- commands_gpu=$(echo "$commands" | sed -E "s/--shard-id[[:blank:]]*=[[:blank:]]*[0-9]*/--shard-id=${GPU} /g" | sed 's/ \\ / /g')
197
- echo "Shard ${GPU} commands:$commands_gpu"
198
- echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
199
- docker run \
200
- --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
201
- --network=host \
202
- --shm-size=16gb \
203
- --group-add "$render_gid" \
204
- --rm \
205
- -e HIP_VISIBLE_DEVICES="${GPU}" \
206
- -e HF_TOKEN \
207
- -e AWS_ACCESS_KEY_ID \
208
- -e AWS_SECRET_ACCESS_KEY \
209
- -v "${HF_CACHE}:${HF_MOUNT}" \
210
- -e "HF_HOME=${HF_MOUNT}" \
211
- -e "PYTHONPATH=${MYPYTHONPATH}" \
212
- --name "${container_name}_${GPU}" \
213
- "${image_name}" \
214
- /bin/bash -c "${commands_gpu}" \
215
- |& while read -r line; do echo ">>Shard $GPU: $line"; done &
216
- PIDS+=($!)
217
- done
218
- #wait for all processes to finish and collect exit codes
219
- for pid in "${PIDS[@]}"; do
220
- wait "${pid}"
221
- STATUS+=($?)
222
- done
223
- at_least_one_shard_with_tests=0
224
- for st in "${STATUS[@]}"; do
225
- if [[ ${st} -ne 0 ]] && [[ ${st} -ne 5 ]]; then
226
- echo "One of the processes failed with $st"
227
- exit "${st}"
228
- elif [[ ${st} -eq 5 ]]; then
229
- echo "Shard exited with status 5 (no tests collected) - treating as success"
230
- else # This means st is 0
231
- at_least_one_shard_with_tests=1
232
- fi
233
- done
234
- if [[ ${#STATUS[@]} -gt 0 && ${at_least_one_shard_with_tests} -eq 0 ]]; then
235
- echo "All shards reported no tests collected. Failing the build."
236
- exit 1
237
- fi
238
-
239
-elif [[ $commands == *"VLLM_TEST_GROUP_NAME=mi325_4-2-node-tests-4-gpus-in-total"* ]]; then
192
+if [[ $commands == *"VLLM_TEST_GROUP_NAME=mi325_4-2-node-tests-4-gpus-in-total"* ]]; then
193
194
export DCKR_VER=$(docker --version | sed 's/Docker version \(.*\), build .*/\1/')
195
.buildkite/test-amd.yaml
+9
-9
@@ -542,7 +542,7 @@ steps:
542
- label: LoRA Test %N # 20min each
543
timeout_in_minutes: 30
544
mirror_hardwares: [amdexperimental]
545
- agent_pool: mi325_8
545
+ agent_pool: mi325_1
546
# grade: Blocking
547
source_file_dependencies:
548
- vllm/lora
@@ -636,7 +636,7 @@ steps:
636
- label: Kernels Attention Test %N # 23min
637
timeout_in_minutes: 35
638
mirror_hardwares: [amdexperimental, amdproduction]
639
- agent_pool: mi325_8
639
+ agent_pool: mi325_1
640
# grade: Blocking
641
source_file_dependencies:
642
- csrc/attention/
@@ -651,7 +651,7 @@ steps:
651
- label: Kernels Quantization Test %N # 64min
652
timeout_in_minutes: 90
653
mirror_hardwares: [amdexperimental]
654
- agent_pool: mi325_8
654
+ agent_pool: mi325_1
655
# grade: Blocking
656
source_file_dependencies:
657
- csrc/quantization/
@@ -664,7 +664,7 @@ steps:
664
- label: Kernels MoE Test %N # 40min
665
timeout_in_minutes: 60
666
mirror_hardwares: [amdexperimental, amdproduction]
667
- agent_pool: mi325_8
667
+ agent_pool: mi325_1
668
# grade: Blocking
669
source_file_dependencies:
670
- csrc/quantization/cutlass_w8a8/moe/
@@ -742,7 +742,7 @@ steps:
742
- label: Benchmarks # 11min
743
timeout_in_minutes: 20
744
mirror_hardwares: [amdexperimental, amdproduction]
745
- agent_pool: mi325_8
745
+ agent_pool: mi325_1
746
# grade: Blocking
747
working_dir: "/vllm-workspace/.buildkite"
748
source_file_dependencies:
@@ -753,7 +753,7 @@ steps:
753
- label: Benchmarks CLI Test # 7min
754
timeout_in_minutes: 20
755
mirror_hardwares: [amdexperimental, amdproduction]
756
- agent_pool: mi325_8
756
+ agent_pool: mi325_1
757
# grade: Blocking
758
source_file_dependencies:
759
- vllm/
@@ -827,7 +827,7 @@ steps:
827
- label: Basic Models Tests (Extra Initialization) %N
828
timeout_in_minutes: 45
829
mirror_hardwares: [amdexperimental, amdproduction]
830
- agent_pool: mi325_8
830
+ agent_pool: mi325_1
831
# grade: Blocking
832
torch_nightly: true
833
source_file_dependencies:
@@ -888,7 +888,7 @@ steps:
888
- label: Language Models Tests (Extra Standard) %N
889
timeout_in_minutes: 45
890
mirror_hardwares: [amdexperimental]
891
- agent_pool: mi325_8
891
+ agent_pool: mi325_1
892
# grade: Blocking
893
torch_nightly: true
894
source_file_dependencies:
@@ -909,7 +909,7 @@ steps:
909
- label: Language Models Tests (Hybrid) %N
910
timeout_in_minutes: 75
911
mirror_hardwares: [amdexperimental]
912
- agent_pool: mi325_8
912
+ agent_pool: mi325_1
913
# grade: Blocking
914
torch_nightly: true
915
source_file_dependencies: