@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 967c5c3bc3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

[ROCm][CI] Stage C mirrors (#42793)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>

Andreas Karatzas committed Jun 8, 2026 at 01:00 UTC 967c5c3bc38891f4465d3f4e99917ed837bb3833
11 files changed +213 -38
.buildkite/scripts/hardware_ci/run-amd-test.sh
+28 -4
@@ -28,8 +28,10 @@
28 ###############################################################################
29 set -o pipefail
30
31 -# Export Python path
32 -export PYTHONPATH=".."
31 +# Export Python path for commands that run directly on the host. Containerized
32 +# tests set this to /vllm-workspace below so spawned Python processes do not
33 +# depend on their current working directory.
34 +export PYTHONPATH="${PYTHONPATH:-..}"
35
36 ###############################################################################
37 # Helper Functions
@@ -377,6 +379,14 @@ HF_CACHE="$(realpath ~)/huggingface"
379 mkdir -p "${HF_CACHE}"
380 HF_MOUNT="/root/.cache/huggingface"
381
382 +# Hugging Face Hub defaults to 10s request/download timeouts, while the ROCm
383 +# CI image currently raises downloads to 60s. AMD model-test jobs routinely
384 +# start from a cold or partially-populated shared cache, and the 60s read cap
385 +# has still timed out before pytest reached the vLLM behavior under test.
386 +# Keep the CI default explicit and overridable from the Buildkite environment.
387 +: "${HF_HUB_DOWNLOAD_TIMEOUT:=300}"
388 +: "${HF_HUB_ETAG_TIMEOUT:=60}"
389 +
390 # ---- Command source selection ----
391 # Prefer VLLM_TEST_COMMANDS (preserves all inner quoting intact).
392 # Fall back to $* for backward compatibility, but warn that inner
@@ -416,7 +426,14 @@ fi
426
427 echo "Final commands: $commands"
428
419 -MYPYTHONPATH=".."
429 +MYPYTHONPATH="/vllm-workspace"
430 +
431 +container_job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-0}}"
432 +container_job_id="${container_job_id//[^A-Za-z0-9_.-]/_}"
433 +container_job_id_short="${container_job_id:0:8}"
434 +CONTAINER_TMPDIR="/tmp/vllm-${container_job_id_short}"
435 +CONTAINER_CACHE_ROOT="/tmp/vllm-buildkite-${container_job_id}/cache"
436 +CONTAINER_PREFLIGHT="mkdir -p \"\$TMPDIR\" \"\$TORCHINDUCTOR_CACHE_DIR\" \"\$TRITON_CACHE_DIR\" \"\$VLLM_CACHE_ROOT\" \"\$XDG_CACHE_HOME\" && python -c \"import encodings, importlib.metadata as im, importlib.util as iu; [im.version(d) for d in ('transformers', 'torch', 'ray', 'sympy', 'markupsafe', 'vllm')]; missing=[m for m in ('torch.utils.model_zoo', 'transformers.models.nomic_bert', 'ray.dag', 'sympy.physics', 'markupsafe._speedups') if iu.find_spec(m) is None]; assert not missing, missing\""
437
438 # Verify GPU access
439 render_gid=$(getent group render | cut -d: -f3)
@@ -493,6 +510,8 @@ else
510 --group-add "$render_gid" \
511 --rm \
512 -e HF_TOKEN \
513 + -e "HF_HUB_DOWNLOAD_TIMEOUT=${HF_HUB_DOWNLOAD_TIMEOUT}" \
514 + -e "HF_HUB_ETAG_TIMEOUT=${HF_HUB_ETAG_TIMEOUT}" \
515 -e AWS_ACCESS_KEY_ID \
516 -e AWS_SECRET_ACCESS_KEY \
517 -e BUILDKITE_PARALLEL_JOB \
@@ -500,10 +519,15 @@ else
519 -v "${HF_CACHE}:${HF_MOUNT}" \
520 -e "HF_HOME=${HF_MOUNT}" \
521 -e "PYTHONPATH=${MYPYTHONPATH}" \
522 + -e "TMPDIR=${CONTAINER_TMPDIR}/tmp" \
523 + -e "TORCHINDUCTOR_CACHE_DIR=${CONTAINER_CACHE_ROOT}/torchinductor" \
524 + -e "TRITON_CACHE_DIR=${CONTAINER_CACHE_ROOT}/triton" \
525 + -e "VLLM_CACHE_ROOT=${CONTAINER_CACHE_ROOT}/vllm" \
526 + -e "XDG_CACHE_HOME=${CONTAINER_CACHE_ROOT}/xdg" \
527 -e "PYTORCH_ROCM_ARCH=" \
528 --name "${container_name}" \
529 "${image_name}" \
506 - /bin/bash -c "${commands}"
530 + /bin/bash -c "${CONTAINER_PREFLIGHT} && ${commands}"
531
532 exit_code=$?
533 handle_pytest_exit "$exit_code"
.buildkite/test-amd.yaml
+67 -19
@@ -315,24 +315,6 @@ steps:
315 - pytest -v -s distributed/test_pp_cudagraph.py
316 - pytest -v -s distributed/test_pipeline_parallel.py
317
318 -#---------------------------------------------------------- mi250 · engine -----------------------------------------------------------#
319 -
320 -- label: Engine # TBD
321 - timeout_in_minutes: 180
322 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
323 - agent_pool: mi250_1
324 - optional: true
325 - working_dir: "/vllm-workspace/tests"
326 - source_file_dependencies:
327 - - vllm/
328 - - tests/engine
329 - - tests/test_sequence
330 - - tests/test_config
331 - - tests/test_logger
332 - - tests/test_vllm_port
333 - commands:
334 - - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py
335 -
318 #----------------------------------------------------------- mi250 · evals -----------------------------------------------------------#
319
320 - label: Multi-Modal Accuracy Eval (Small Models) # TBD
@@ -1351,6 +1333,7 @@ steps:
1333 timeout_in_minutes: 180
1334 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1335 agent_pool: mi300_1
1336 + optional: true
1337 working_dir: "/vllm-workspace/tests"
1338 source_file_dependencies:
1339 - csrc/
@@ -1409,6 +1392,7 @@ steps:
1392 timeout_in_minutes: 180
1393 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1394 agent_pool: mi300_1
1395 + optional: true
1396 working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
1397 source_file_dependencies:
1398 - csrc/
@@ -2183,10 +2167,72 @@ steps:
2167 commands:
2168 - pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness"
2169
2170 +- label: Speculators Correctness # TBD
2171 + timeout_in_minutes: 180
2172 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2173 + agent_pool: mi300_1
2174 + optional: true
2175 + working_dir: "/vllm-workspace/tests"
2176 + source_file_dependencies:
2177 + - vllm/config/speculative.py
2178 + - vllm/engine/arg_utils.py
2179 + - vllm/transformers_utils/config.py
2180 + - vllm/transformers_utils/configs/speculators/
2181 + - vllm/v1/spec_decode/
2182 + - vllm/v1/worker/gpu/spec_decode/
2183 + - vllm/v1/worker/gpu_model_runner.py
2184 + - vllm/v1/sample/
2185 + - vllm/v1/attention/backends/
2186 + - vllm/v1/attention/selector.py
2187 + - vllm/model_executor/model_loader/
2188 + - vllm/model_executor/layers/
2189 + - vllm/model_executor/models/llama_eagle3.py
2190 + - vllm/model_executor/models/qwen3.py
2191 + - vllm/model_executor/models/qwen3_dflash.py
2192 + - vllm/model_executor/models/registry.py
2193 + - vllm/_aiter_ops.py
2194 + - tests/evals/gsm8k/
2195 + - tests/v1/spec_decode/test_speculators_correctness.py
2196 + - vllm/platforms/rocm.py
2197 + commands:
2198 + - export VLLM_ALLOW_INSECURE_SERIALIZATION=1
2199 + - pytest -v -s v1/spec_decode/test_speculators_correctness.py -m slow_test
2200 +
2201 +- label: Extract Hidden States Integration # TBD
2202 + timeout_in_minutes: 180
2203 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2204 + agent_pool: mi300_1
2205 + working_dir: "/vllm-workspace/tests"
2206 + source_file_dependencies:
2207 + - vllm/config/speculative.py
2208 + - vllm/distributed/kv_transfer/kv_connector/
2209 + - vllm/model_executor/layers/attention/
2210 + - vllm/model_executor/layers/mamba/
2211 + - vllm/model_executor/model_loader/
2212 + - vllm/model_executor/models/extract_hidden_states.py
2213 + - vllm/model_executor/models/llama.py
2214 + - vllm/model_executor/models/qwen3_5.py
2215 + - vllm/model_executor/models/qwen3_next.py
2216 + - vllm/model_executor/models/registry.py
2217 + - vllm/transformers_utils/configs/extract_hidden_states.py
2218 + - vllm/transformers_utils/configs/qwen3_5.py
2219 + - vllm/v1/attention/backends/
2220 + - vllm/v1/attention/selector.py
2221 + - vllm/v1/kv_cache_interface.py
2222 + - vllm/v1/spec_decode/extract_hidden_states.py
2223 + - vllm/v1/worker/gpu_model_runner.py
2224 + - vllm/_aiter_ops.py
2225 + - tests/v1/kv_connector/extract_hidden_states_integration/
2226 + - vllm/platforms/rocm.py
2227 + commands:
2228 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
2229 + - pytest -v -s v1/kv_connector/extract_hidden_states_integration
2230 +
2231 - label: V1 attention (H100-MI300) # TBD
2232 timeout_in_minutes: 180
2233 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2234 agent_pool: mi300_1
2235 + optional: true
2236 working_dir: "/vllm-workspace/tests"
2237 source_file_dependencies:
2238 - vllm/config/attention.py
@@ -2545,6 +2591,7 @@ steps:
2591 timeout_in_minutes: 180
2592 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
2593 agent_pool: mi325_1
2594 + optional: true
2595 working_dir: "/vllm-workspace/tests"
2596 source_file_dependencies:
2597 - vllm/
@@ -2554,7 +2601,7 @@ steps:
2601 - tests/test_logger
2602 - tests/test_vllm_port
2603 commands:
2557 - - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py
2604 + - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py test_jit_monitor.py
2605
2606 #----------------------------------------------------------- mi325 · evals -----------------------------------------------------------#
2607
@@ -2636,6 +2683,7 @@ steps:
2683 timeout_in_minutes: 180
2684 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
2685 agent_pool: mi325_1
2686 + optional: true
2687 working_dir: "/vllm-workspace/tests"
2688 source_file_dependencies:
2689 - vllm/
.buildkite/test_areas/attention.yaml
+16 -2
@@ -2,8 +2,8 @@ group: Attention
2 depends_on:
3 - image-build
4 steps:
5 -- label: V1 attention (H100)
6 - key: v1-attention-h100
5 +- label: V1 attention (H100-MI300)
6 + key: v1-attention-h100-mi300
7 timeout_in_minutes: 30
8 device: h100
9 source_file_dependencies:
@@ -13,6 +13,20 @@ steps:
13 - tests/v1/attention
14 commands:
15 - pytest -v -s v1/attention
16 + mirror:
17 + amd:
18 + device: mi300_1
19 + timeout_in_minutes: 70
20 + depends_on:
21 + - image-build-amd
22 + source_file_dependencies:
23 + - vllm/config/attention.py
24 + - vllm/model_executor/layers/attention
25 + - vllm/v1/attention
26 + - tests/v1/attention
27 + - vllm/_aiter_ops.py
28 + - vllm/envs.py
29 + - vllm/platforms/rocm.py
30
31 - label: V1 attention (B200)
32 key: v1-attention-b200
.buildkite/test_areas/engine.yaml
+7 -1
@@ -26,6 +26,12 @@ steps:
26 - tests/test_jit_monitor.py
27 commands:
28 - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py test_jit_monitor.py
29 + mirror:
30 + amd:
31 + device: mi300_1
32 + timeout_in_minutes: 60
33 + depends_on:
34 + - image-build-amd
35
36 - label: Engine (1 GPU)
37 key: engine-1-gpu
@@ -38,7 +44,7 @@ steps:
44 - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py
45 mirror:
46 amd:
41 - device: mi325_1
47 + device: mi300_1
48 timeout_in_minutes: 40
49 depends_on:
50 - image-build-amd
.buildkite/test_areas/entrypoints.yaml
+21 -8
@@ -28,7 +28,6 @@ steps:
28 mirror:
29 amd:
30 device: mi325_1
31 - soft_fail: true
31 depends_on:
32 - image-build-amd
33
@@ -45,8 +44,7 @@ steps:
44 - pytest -v -s entrypoints/openai/chat_completion --ignore=entrypoints/openai/chat_completion/test_oot_registration.py
45 mirror:
46 amd:
48 - device: mi325_1
49 - soft_fail: true
47 + device: mi300_1
48 timeout_in_minutes: 80
49 depends_on:
50 - image-build-amd
@@ -68,8 +66,7 @@ steps:
66 - pytest -v -s tool_use
67 mirror:
68 amd:
71 - device: mi325_1
72 - soft_fail: true
69 + device: mi300_1
70 timeout_in_minutes: 60
71 depends_on:
72 - image-build-amd
@@ -89,7 +86,6 @@ steps:
86 mirror:
87 amd:
88 device: mi325_1
92 - soft_fail: true
89 timeout_in_minutes: 60
90 depends_on:
91 - image-build-amd
@@ -108,8 +104,7 @@ steps:
104 - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
105 mirror:
106 amd:
111 - device: mi325_1
112 - soft_fail: true
107 + device: mi300_1
108 depends_on:
109 - image-build-amd
110
@@ -155,3 +150,21 @@ steps:
150 - vllm/entrypoints/openai/
151 commands: # LMEval
152 - pytest -s entrypoints/openai/correctness/
153 + mirror:
154 + amd:
155 + device: mi300_1
156 + depends_on:
157 + - image-build-amd
158 + source_file_dependencies:
159 + - csrc/
160 + - vllm/entrypoints/openai/
161 + - vllm/model_executor/models/whisper.py
162 + - vllm/model_executor/layers/
163 + - vllm/v1/attention/backends/
164 + - vllm/v1/attention/selector.py
165 + - vllm/_aiter_ops.py
166 + - vllm/platforms/rocm.py
167 + - vllm/model_executor/model_loader/
168 + commands:
169 + - bash ../tools/install_torchcodec_rocm.sh || exit 1
170 + - pytest -s entrypoints/openai/correctness/
.buildkite/test_areas/kernels.yaml
+1 -1
@@ -86,7 +86,7 @@ steps:
86 parallelism: 2
87 mirror:
88 amd:
89 - device: mi325_1
89 + device: mi300_1
90 source_file_dependencies:
91 - csrc/quantization/
92 - vllm/model_executor/layers/quantization
.buildkite/test_areas/lm_eval.yaml
+15
@@ -12,6 +12,21 @@ steps:
12 autorun_on_main: true
13 commands:
14 - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt
15 + mirror:
16 + amd:
17 + device: mi300_1
18 + timeout_in_minutes: 55
19 + depends_on:
20 + - image-build-amd
21 + source_file_dependencies:
22 + - csrc/
23 + - vllm/model_executor/layers/quantization
24 + - vllm/model_executor/models/
25 + - vllm/model_executor/model_loader/
26 + - vllm/v1/attention/backends/
27 + - vllm/v1/attention/selector.py
28 + - vllm/_aiter_ops.py
29 + - vllm/platforms/rocm.py
30
31 # - label: LM Eval Large Models (4 GPUs)(A100)
32 # device: a100
.buildkite/test_areas/models_language.yaml
+1
@@ -51,6 +51,7 @@ steps:
51 torch_nightly: {}
52 amd:
53 device: mi325_1
54 + timeout_in_minutes: 90
55 depends_on:
56 - image-build-amd
57 commands:
.buildkite/test_areas/models_multimodal.yaml
+12 -3
@@ -15,7 +15,7 @@ steps:
15 - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
16 mirror:
17 amd:
18 - device: mi325_1
18 + device: mi300_1
19 depends_on:
20 - image-build-amd
21
@@ -33,7 +33,7 @@ steps:
33 - pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model
34 mirror:
35 amd:
36 - device: mi325_1
36 + device: mi300_1
37 depends_on:
38 - image-build-amd
39
@@ -50,7 +50,7 @@ steps:
50 - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
51 mirror:
52 amd:
53 - device: mi325_1
53 + device: mi300_1
54 depends_on:
55 - image-build-amd
56
@@ -153,3 +153,12 @@ steps:
153 - tests/models/multimodal/pooling
154 commands:
155 - pytest -v -s models/multimodal/pooling -m 'not core_model'
156 + mirror:
157 + amd:
158 + device: mi300_1
159 + timeout_in_minutes: 60
160 + depends_on:
161 + - image-build-amd
162 + source_file_dependencies:
163 + - vllm/
164 + - tests/models/multimodal/pooling
.buildkite/test_areas/spec_decode.yaml
+43
@@ -37,6 +37,21 @@ steps:
37 - tests/v1/e2e/spec_decode/
38 commands:
39 - pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness"
40 + mirror:
41 + amd:
42 + device: mi300_1
43 + timeout_in_minutes: 65
44 + depends_on:
45 + - image-build-amd
46 + source_file_dependencies:
47 + - vllm/v1/spec_decode/
48 + - vllm/v1/worker/gpu/spec_decode/
49 + - vllm/model_executor/model_loader/
50 + - vllm/v1/sample/
51 + - vllm/model_executor/layers/
52 + - vllm/transformers_utils/configs/speculators/
53 + - tests/v1/e2e/spec_decode/
54 + - vllm/platforms/rocm.py
55
56 - label: Spec Decode Speculators + MTP Nightly B200
57 key: spec-decode-speculators-mtp-nightly-b200
@@ -61,6 +76,20 @@ steps:
76 - tests/v1/e2e/spec_decode/
77 commands:
78 - pytest -v -s v1/e2e/spec_decode -k "ngram or suffix"
79 + mirror:
80 + amd:
81 + device: mi300_1
82 + timeout_in_minutes: 65
83 + depends_on:
84 + - image-build-amd
85 + source_file_dependencies:
86 + - vllm/v1/spec_decode/
87 + - vllm/v1/worker/gpu/spec_decode/
88 + - vllm/model_executor/model_loader/
89 + - vllm/v1/sample/
90 + - vllm/model_executor/layers/
91 + - tests/v1/e2e/spec_decode/
92 + - vllm/platforms/rocm.py
93
94 - label: Spec Decode Draft Model
95 key: spec-decode-draft-model
@@ -72,6 +101,20 @@ steps:
101 - tests/v1/e2e/spec_decode/
102 commands:
103 - pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference"
104 + mirror:
105 + amd:
106 + device: mi300_1
107 + timeout_in_minutes: 50
108 + depends_on:
109 + - image-build-amd
110 + source_file_dependencies:
111 + - vllm/v1/spec_decode/
112 + - vllm/v1/worker/gpu/spec_decode/
113 + - vllm/model_executor/model_loader/
114 + - vllm/v1/sample/
115 + - vllm/model_executor/layers/
116 + - tests/v1/e2e/spec_decode/
117 + - vllm/platforms/rocm.py
118
119 - label: Spec Decode Draft Model Nightly B200
120 key: spec-decode-draft-model-nightly-b200
docker/Dockerfile.rocm
+2
@@ -230,6 +230,7 @@ COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/requirements /requirements
230 COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks
231 COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tests /tests
232 COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/examples /examples
233 +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh
234 COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm /docker/
235 COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/.buildkite /.buildkite
236 COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/pyproject.toml /pyproject.toml
@@ -513,6 +514,7 @@ COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/requirements /requir
514 COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks
515 COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tests /tests
516 COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/examples /examples
517 +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh
518 COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm /docker/
519 COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/.buildkite /.buildkite
520 COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/pyproject.toml /pyproject.toml