@tduncan / Nvidia-Optimization-Stack-VLLM / commits / e581e14002

[ROCm][CI] Stage G gating (#50922)

Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>

Andreas Karatzas committed Sep 22, 2026 at 21:04 UTC e581e14002a0f3688c161fecc0ce36edc6866b69
7 files changed +250
.buildkite/test_areas/benchmarks.yaml
+15
@@ -34,3 +34,18 @@ steps:
34 - vllm/v1/attention/
35 commands:
36 - python3 benchmarks/attention_benchmarks/benchmark.py --backends flash flashinfer --batch-specs "8q1s1k"
37 + mirror:
38 + amd:
39 + label: ":amd: (MI355) Attention Benchmark Smoke"
40 + dind: false
41 + device: mi355_2
42 + timeout_in_minutes: 25
43 + depends_on:
44 + - image-build-amd
45 + source_file_dependencies:
46 + - benchmarks/attention_benchmarks/
47 + - vllm/v1/attention/
48 + - vllm/_aiter_ops.py
49 + - vllm/platforms/rocm.py
50 + commands:
51 + - python3 benchmarks/attention_benchmarks/benchmark.py --backends ROCM_ATTN ROCM_AITER_FA ROCM_AITER_UNIFIED_ATTN --batch-specs "8q1s1k"
.buildkite/test_areas/disaggregated.yaml
+19
@@ -205,6 +205,25 @@ steps:
205 commands:
206 - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
207 - bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh
208 + mirror:
209 + amd:
210 + label: ":amd: (MI355) MultiConnector (Nixl+Offloading) PD accuracy"
211 + dind: false
212 + device: mi355_2
213 + timeout_in_minutes: 40
214 + depends_on:
215 + - image-build-amd
216 + source_file_dependencies:
217 + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
218 + - vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
219 + - vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
220 + - vllm/distributed/kv_transfer/kv_connector/v1/offloading/
221 + - tests/v1/kv_connector/nixl_integration/
222 + - requirements/kv_connectors_rocm.txt
223 + - vllm/platforms/rocm.py
224 + commands:
225 + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
226 + - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh
227
228 - label: ":nvidia: (A100) NixlConnector PD + Spec Decode acceptance"
229 key: nixlconnector-pd-spec-decode-acceptance-2-gpus
.buildkite/test_areas/distributed.yaml
+52
@@ -219,6 +219,41 @@ steps:
219 # default does not fit an L4.
220 - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
221 - RDT_MODEL=hmellor/tiny-random-DeepseekV2ForCausalLM python3 examples/rl/rlhf_sharded_rdt_small_ep.py
222 + mirror:
223 + amd:
224 + label: ":amd: (MI355) Distributed Torchrun + Examples"
225 + dind: false
226 + device: mi355_4
227 + timeout_in_minutes: 45
228 + depends_on:
229 + - image-build-amd
230 + source_file_dependencies:
231 + - vllm/distributed/
232 + - "!vllm/distributed/kv_transfer/"
233 + - tests/distributed/test_torchrun_example.py
234 + - tests/distributed/test_torchrun_example_moe.py
235 + - examples/rl/
236 + - examples/features/data_parallel/data_parallel_offline.py
237 + - .buildkite/scripts/install-kv-connectors.sh
238 + - requirements/kv_connectors_rocm.txt
239 + - requirements/test/rocm.in
240 + - requirements/test/rocm.txt
241 + - vllm/platforms/rocm.py
242 + commands:
243 + # Sharded RDT needs Ray >= 2.56; keep the ROCm image's dependencies.
244 + - uv pip install --system --no-deps 'ray==2.56.1'
245 + - export NCCL_CUMEM_HOST_ENABLE=0
246 + - torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py
247 + - PP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py
248 + - TP_SIZE=4 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
249 + - PP_SIZE=2 TP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
250 + - DP_SIZE=4 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
251 + - TP_SIZE=2 DP_SIZE=2 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
252 + - python3 examples/features/data_parallel/data_parallel_offline.py --enforce-eager
253 + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_nccl.py
254 + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_ipc.py
255 + - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
256 + - RDT_MODEL=hmellor/tiny-random-DeepseekV2ForCausalLM python3 examples/rl/rlhf_sharded_rdt_small_ep.py
257
258 - label: ":nvidia: (L4) Distributed DP Extended"
259 key: distributed-dp-tests-4-gpus
@@ -291,6 +326,23 @@ steps:
326 - export NCCL_CUMEM_HOST_ENABLE=0
327 # test with torchrun tp=2 and dp=4 with ep
328 - torchrun --nproc-per-node=8 ../examples/features/torchrun/torchrun_dp_example_offline.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
329 + mirror:
330 + amd:
331 + label: ":amd: (MI355) Distributed DP + EP"
332 + dind: false
333 + device: mi355_8
334 + timeout_in_minutes: 50
335 + depends_on:
336 + - image-build-amd
337 + source_file_dependencies:
338 + - examples/features/torchrun/torchrun_dp_example_offline.py
339 + - vllm/config/parallel.py
340 + - vllm/distributed/
341 + - "!vllm/distributed/kv_transfer/"
342 + - vllm/v1/engine/llm_engine.py
343 + - vllm/v1/executor/uniproc_executor.py
344 + - vllm/v1/worker/gpu_worker.py
345 + - vllm/platforms/rocm.py
346
347 - label: ":nvidia: (A100) Distributed"
348 key: distributed-tests-4xa100
.buildkite/test_areas/e2e_integration.yaml
+79
@@ -11,6 +11,28 @@ steps:
11 working_dir: "/vllm-workspace"
12 commands:
13 - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010
14 + mirror:
15 + amd:
16 + label: ":amd: (MI355) DeepSeek V2-Lite Sync EPLB Accuracy"
17 + dind: false
18 + device: mi355_4
19 + timeout_in_minutes: 40
20 + env:
21 + VLLM_ENGINE_READY_TIMEOUT_S: "1800"
22 + depends_on:
23 + - image-build-amd
24 + source_file_dependencies:
25 + - vllm/model_executor/models/
26 + - vllm/model_executor/model_loader/
27 + - vllm/distributed/eplb
28 + - vllm/model_executor/layers/fused_moe/
29 + - vllm/model_executor/layers/quantization/
30 + - vllm/v1/attention/backends/
31 + - vllm/v1/attention/backends/mla/
32 + - vllm/v1/attention/selector.py
33 + - .buildkite/scripts/scheduled_integration_test/
34 + - vllm/_aiter_ops.py
35 + - vllm/platforms/rocm.py
36
37 - label: ":nvidia: (H100) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy"
38 key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-4xh100
@@ -21,6 +43,25 @@ steps:
43 working_dir: "/vllm-workspace"
44 commands:
45 - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020
46 + mirror:
47 + amd:
48 + label: ":amd: (MI355) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy TP2"
49 + dind: false
50 + device: mi355_4
51 + timeout_in_minutes: 30
52 + depends_on:
53 + - image-build-amd
54 + source_file_dependencies:
55 + - vllm/model_executor/models/
56 + - vllm/model_executor/model_loader/
57 + - vllm/model_executor/layers/quantization/
58 + - vllm/distributed/eplb
59 + - vllm/model_executor/layers/fused_moe/
60 + - vllm/v1/attention/backends/
61 + - vllm/v1/attention/selector.py
62 + - .buildkite/scripts/scheduled_integration_test/
63 + - vllm/_aiter_ops.py
64 + - vllm/platforms/rocm.py
65
66 - label: ":nvidia: (B200) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy"
67 key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-2xb200
@@ -31,6 +72,25 @@ steps:
72 working_dir: "/vllm-workspace"
73 commands:
74 - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1
75 + mirror:
76 + amd:
77 + label: ":amd: (MI355) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy TP1"
78 + dind: false
79 + device: mi355_2
80 + timeout_in_minutes: 25
81 + depends_on:
82 + - image-build-amd
83 + source_file_dependencies:
84 + - vllm/model_executor/models/
85 + - vllm/model_executor/model_loader/
86 + - vllm/model_executor/layers/quantization/
87 + - vllm/model_executor/layers/fused_moe/
88 + - vllm/distributed/eplb
89 + - vllm/v1/attention/backends/
90 + - vllm/v1/attention/selector.py
91 + - .buildkite/scripts/scheduled_integration_test/
92 + - vllm/_aiter_ops.py
93 + - vllm/platforms/rocm.py
94
95 - label: ":nvidia: (H100) Qwen3-30B-A3B-FP8 DP4 Async EPLB Accuracy"
96 key: qwen3-30b-a3b-fp8-dp4-async-eplb-accuracy
@@ -41,6 +101,25 @@ steps:
101 working_dir: "/vllm-workspace"
102 commands:
103 - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_dp4_async_eplb.sh 0.8 200 8050
104 + mirror:
105 + amd:
106 + label: ":amd: (MI355) Qwen3-30B-A3B-FP8 DP4 Async EPLB Accuracy"
107 + dind: false
108 + device: mi355_4
109 + timeout_in_minutes: 30
110 + depends_on:
111 + - image-build-amd
112 + source_file_dependencies:
113 + - vllm/model_executor/models/
114 + - vllm/model_executor/model_loader/
115 + - vllm/model_executor/layers/quantization/
116 + - vllm/distributed/eplb
117 + - vllm/model_executor/layers/fused_moe/
118 + - vllm/v1/attention/backends/
119 + - vllm/v1/attention/selector.py
120 + - .buildkite/scripts/scheduled_integration_test/
121 + - vllm/_aiter_ops.py
122 + - vllm/platforms/rocm.py
123
124 - label: ":nvidia: (H100) DeepSeek V2-Lite Prefetch Offload Accuracy"
125 key: deepseek-v2-lite-prefetch-offload-accuracy-h100
.buildkite/test_areas/expert_parallelism.yaml
+13
@@ -40,6 +40,19 @@ steps:
40 commands:
41 - pytest -v -s distributed/test_eplb_execute.py
42 - pytest -v -s distributed/test_eplb_spec_decode.py
43 + mirror:
44 + amd:
45 + label: ":amd: (MI355) EPLB Execution"
46 + dind: false
47 + device: mi355_4
48 + timeout_in_minutes: 35
49 + depends_on:
50 + - image-build-amd
51 + source_file_dependencies:
52 + - vllm/distributed/eplb
53 + - tests/distributed/test_eplb_execute.py
54 + - tests/distributed/test_eplb_spec_decode.py
55 + - vllm/platforms/rocm.py
56
57 - label: ":nvidia: (H100) Elastic EP Scaling"
58 key: elastic-ep-scaling-test
.buildkite/test_areas/lm_eval.yaml
+48
@@ -114,6 +114,27 @@ steps:
114 commands:
115 - export VLLM_USE_DEEP_GEMM=0 # We found Triton is faster than DeepGEMM for H100
116 - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4
117 + mirror:
118 + amd:
119 + label: ":amd: (MI355) LM Eval Large Models FP8"
120 + dind: false
121 + device: mi355_4
122 + timeout_in_minutes: 45
123 + depends_on:
124 + - image-build-amd
125 + source_file_dependencies:
126 + - csrc/
127 + - vllm/model_executor/layers/quantization
128 + - vllm/model_executor/models/
129 + - vllm/model_executor/model_loader/
130 + - vllm/v1/attention/backends/
131 + - vllm/v1/attention/selector.py
132 + - .buildkite/lm-eval-harness/
133 + - vllm/_aiter_ops.py
134 + - vllm/platforms/rocm.py
135 + commands:
136 + - export VLLM_USE_DEEP_GEMM=0
137 + - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-tp4.txt --tp-size=4
138
139 - label: ":nvidia: (B200) LM Eval Small Models"
140 key: lm-eval-small-models-1xb200
@@ -342,6 +363,33 @@ steps:
363 - vllm/third_party/flash_linear_attention/ops/
364 commands:
365 - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-blackwell.txt
366 + mirror:
367 + amd:
368 + label: ":amd: (MI355) LM Eval Qwen3.5 Models"
369 + dind: false
370 + device: mi355_2
371 + timeout_in_minutes: 40
372 + depends_on:
373 + - image-build-amd
374 + source_file_dependencies:
375 + - vllm/model_executor/models/qwen3_5.py
376 + - vllm/model_executor/models/qwen3_5_mtp.py
377 + - vllm/transformers_utils/configs/qwen3_5.py
378 + - vllm/transformers_utils/configs/qwen3_5_moe.py
379 + - vllm/model_executor/models/qwen2.py
380 + - vllm/model_executor/models/qwen3.py
381 + - vllm/model_executor/models/qwen3_next.py
382 + - vllm/model_executor/models/qwen3_next_mtp.py
383 + - vllm/third_party/flash_linear_attention/ops/
384 + - tests/evals/gsm8k/configs/models-qwen35-mi355.txt
385 + - tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-DEP2.yaml
386 + - tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-MXFP4-AITER-TP2.yaml
387 + - tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-MXFP4-EMU-TP2.yaml
388 + - tests/evals/gsm8k/test_gsm8k_correctness.py
389 + - vllm/_aiter_ops.py
390 + - vllm/platforms/rocm.py
391 + commands:
392 + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-mi355.txt
393
394 - label: ":nvidia: (H200) LM Eval DeepSeek R1 TP"
395 key: lm-eval-deepseek-r1-tp-8xh200
.buildkite/test_areas/rust_frontend.yaml
+24
@@ -204,3 +204,27 @@ steps:
204 - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py -k "not 4 and not server_info"
205 - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py -k "not 4 and not server_info"
206 - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py -k "not 4 and not server_info"
207 + mirror:
208 + amd:
209 + label: ":amd: (MI355) Rust Frontend Distributed"
210 + dind: false
211 + device: mi355_4
212 + timeout_in_minutes: 45
213 + env:
214 + VLLM_ENGINE_READY_TIMEOUT_S: "1800"
215 + depends_on:
216 + - image-build-amd
217 + source_file_dependencies:
218 + - rust/
219 + - vllm/distributed/
220 + - "!vllm/distributed/kv_transfer/"
221 + - vllm/engine/
222 + - vllm/v1/executor/
223 + - vllm/v1/engine/
224 + - vllm/v1/worker/
225 + - tests/utils.py
226 + - tests/v1/distributed/test_dense_dp_world_size.py
227 + - tests/v1/distributed/test_external_lb_dp.py
228 + - tests/v1/distributed/test_hybrid_lb_dp.py
229 + - tests/v1/distributed/test_internal_lb_dp.py
230 + - vllm/platforms/rocm.py