@tduncan / Nvidia-Optimization-Stack-VLLM / commits / d406f09eb8

[ROCm][CI] Stage F gating (#55252)

Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>

Andreas Karatzas committed Sep 12, 2026 at 02:40 UTC d406f09eb8d05eaf81d768bf74212bc6d24cd11d
9 files changed +236
.buildkite/test_areas/disaggregated.yaml
+35
@@ -172,6 +172,23 @@ steps:
172 commands:
173 - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
174 - bash v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh
175 + mirror:
176 + amd:
177 + label: ":amd: (MI355) Hybrid SSM NixlConnector PD prefix cache"
178 + dind: false
179 + device: mi355_2
180 + timeout_in_minutes: 25
181 + depends_on:
182 + - image-build-amd
183 + source_file_dependencies:
184 + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
185 + - vllm/v1/core/sched/
186 + - vllm/v1/core/kv_cache_coordinator.py
187 + - tests/v1/kv_connector/nixl_integration/
188 + - vllm/platforms/rocm.py
189 + commands:
190 + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
191 + - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh
192
193 - label: ":nvidia: (L4) MultiConnector (Nixl+Offloading) PD accuracy"
194 key: multiconnector-nixl-offloading-pd-accuracy-2-gpus
@@ -234,6 +251,24 @@ steps:
251 commands:
252 - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
253 - bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh
254 + mirror:
255 + amd:
256 + label: ":amd: (MI355) MultiConnector (Nixl+Offloading) PD edge cases"
257 + dind: false
258 + device: mi355_2
259 + timeout_in_minutes: 25
260 + depends_on:
261 + - image-build-amd
262 + source_file_dependencies:
263 + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
264 + - vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
265 + - vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
266 + - vllm/distributed/kv_transfer/kv_connector/v1/offloading/
267 + - tests/v1/kv_connector/nixl_integration/
268 + - vllm/platforms/rocm.py
269 + commands:
270 + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
271 + - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh
272
273 # P TP 4 - D DPEP 4 test case for DSv4-Flash
274 - label: ":nvidia: (H200) DSv4-Flash Disaggregated DP EP"
.buildkite/test_areas/distributed.yaml
+39
@@ -105,6 +105,26 @@ steps:
105 - export NCCL_CUMEM_HOST_ENABLE=0
106 - pytest -v -s entrypoints/llm/test_collective_rpc.py
107 - pytest -v -s ./compile/test_wrapper.py
108 + mirror:
109 + amd:
110 + label: ":amd: (MI355) Distributed Compile + RPC"
111 + dind: false
112 + device: mi355_2
113 + timeout_in_minutes: 65
114 + depends_on:
115 + - image-build-amd
116 + source_file_dependencies:
117 + - vllm/compilation/
118 + - vllm/distributed/
119 + - "!vllm/distributed/kv_transfer/"
120 + - vllm/engine/
121 + - vllm/executor/
122 + - vllm/worker/worker_base.py
123 + - vllm/v1/engine/
124 + - vllm/v1/worker/
125 + - tests/compile/test_wrapper.py
126 + - tests/entrypoints/llm/test_collective_rpc.py
127 + - vllm/platforms/rocm.py
128
129 - label: ":nvidia: (L4) Distributed Torchrun + Shutdown"
130 key: distributed-torchrun-shutdown-tests-2-gpus
@@ -335,6 +355,25 @@ steps:
355 - pytest -v -s distributed/test_pp_cudagraph.py
356 - pytest -v -s distributed/test_pipeline_parallel.py
357 - pytest -v -s v1/distributed/test_pp_dp_v2.py
358 + mirror:
359 + amd:
360 + label: ":amd: (MI250) Pipeline + Context Parallelism"
361 + device: mi250_4
362 + timeout_in_minutes: 90
363 + depends_on:
364 + - image-build-amd
365 + source_file_dependencies:
366 + - vllm/distributed/
367 + - "!vllm/distributed/kv_transfer/"
368 + - vllm/engine/
369 + - vllm/executor/
370 + - vllm/model_executor/models/
371 + - vllm/model_executor/layers/
372 + - vllm/v1/attention/backends/
373 + - vllm/v1/attention/selector.py
374 + - tests/distributed/
375 + - vllm/_aiter_ops.py
376 + - vllm/platforms/rocm.py
377
378 - label: ":nvidia: (L4) RayExecutorV2"
379 key: rayexecutorv2-4-gpus
.buildkite/test_areas/lm_eval.yaml
+42
@@ -79,6 +79,26 @@ steps:
79 autorun_on_main: true
80 commands:
81 - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small-tp.txt
82 + mirror:
83 + amd:
84 + label: ":amd: (MI355) LM Eval Small Models Distributed"
85 + dind: false
86 + device: mi355_2
87 + timeout_in_minutes: 45
88 + depends_on:
89 + - image-build-amd
90 + source_file_dependencies:
91 + - csrc/
92 + - vllm/model_executor/layers/quantization
93 + - vllm/model_executor/models/
94 + - vllm/model_executor/model_loader/
95 + - vllm/v1/attention/backends/
96 + - vllm/v1/attention/selector.py
97 + - tests/evals/gsm8k/configs/models-mi3xx-fp8-and-mixed.txt
98 + - vllm/_aiter_ops.py
99 + - vllm/platforms/rocm.py
100 + commands:
101 + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt
102
103 - label: ":nvidia: (B200) LM Eval PCP"
104 key: lm-eval-pcp-4xb200
@@ -375,6 +395,28 @@ steps:
395 commands:
396 - uv pip install --system 'gpt-oss[eval]==0.0.5'
397 - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-b200.txt
398 + mirror:
399 + amd:
400 + label: ":amd: (MI355) GPQA Eval (GPT-OSS)"
401 + dind: false
402 + device: mi355_2
403 + timeout_in_minutes: 40
404 + depends_on:
405 + - image-build-amd
406 + source_file_dependencies:
407 + - csrc/
408 + - vllm/model_executor/layers/quantization
409 + - vllm/model_executor/models/
410 + - vllm/model_executor/model_loader/
411 + - vllm/v1/attention/backends/
412 + - vllm/v1/attention/selector.py
413 + - vllm/model_executor/layers/fused_moe/
414 + - tests/evals/gpt_oss/
415 + - vllm/_aiter_ops.py
416 + - vllm/platforms/rocm.py
417 + commands:
418 + - uv pip install --system 'gpt-oss[eval]==0.0.5'
419 + - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt
420
421 - label: ":nvidia: (B200) Qwen3.8-Flash-Next-FP8 Accuracy Eval"
422 key: accuracy-eval-qwen3-8-flash-next-fp8-b200
.buildkite/test_areas/misc.yaml
+47
@@ -251,6 +251,35 @@ steps:
251 commands:
252 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
253 - pytest -v -s -m 'distributed' tests/v1/kv_connector/extract_hidden_states_integration
254 + mirror:
255 + amd:
256 + label: ":amd: (MI355) Extract Hidden States Integration"
257 + dind: false
258 + device: mi355_2
259 + timeout_in_minutes: 20
260 + depends_on:
261 + - image-build-amd
262 + source_file_dependencies:
263 + - vllm/config/speculative.py
264 + - vllm/distributed/kv_transfer/kv_connector/
265 + - vllm/model_executor/layers/attention/
266 + - vllm/model_executor/layers/mamba/
267 + - vllm/model_executor/model_loader/
268 + - vllm/model_executor/models/extract_hidden_states.py
269 + - vllm/model_executor/models/llama.py
270 + - vllm/model_executor/models/qwen3_5.py
271 + - vllm/model_executor/models/qwen3_next.py
272 + - vllm/model_executor/models/registry.py
273 + - vllm/transformers_utils/configs/extract_hidden_states.py
274 + - vllm/transformers_utils/configs/qwen3_5.py
275 + - vllm/v1/attention/backends/
276 + - vllm/v1/attention/selector.py
277 + - vllm/v1/kv_cache_interface.py
278 + - vllm/v1/spec_decode/extract_hidden_states.py
279 + - vllm/v1/worker/gpu_model_runner.py
280 + - vllm/_aiter_ops.py
281 + - tests/v1/kv_connector/extract_hidden_states_integration/
282 + - vllm/platforms/rocm.py
283
284 - label: ":nvidia: (H200 MIG 18GB) Regression"
285 key: regression
@@ -537,6 +566,24 @@ steps:
566 - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
567 - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA
568 - VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k FLASH_ATTN
569 + mirror:
570 + amd:
571 + label: ":amd: (MI250) Batch Invariance"
572 + device: mi250_1
573 + timeout_in_minutes: 30
574 + depends_on:
575 + - image-build-amd
576 + source_file_dependencies:
577 + - vllm/v1/attention
578 + - vllm/model_executor/layers
579 + - tests/v1/determinism/
580 + - vllm/_aiter_ops.py
581 + - vllm/platforms/rocm.py
582 + commands:
583 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
584 + - pip install pytest-timeout pytest-forked
585 + - pytest -v -s v1/determinism/test_batch_invariance.py
586 + - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
587
588 - label: ":nvidia: (B200) Batch Invariance"
589 key: batch-invariance-b200
.buildkite/test_areas/models_basic.yaml
+9
@@ -156,3 +156,12 @@ steps:
156 device: cpu-small
157 commands:
158 - pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/test_qwen3_5_mtp_config.py models/transformers/fusers/
159 + mirror:
160 + amd:
161 + label: ":amd: (MI250) Basic Models Other"
162 + device: mi250_1
163 + no_gpu: true
164 + working_dir: "/vllm-workspace/tests"
165 + timeout_in_minutes: 25
166 + depends_on:
167 + - image-build-amd
.buildkite/test_areas/models_multimodal.yaml
+9
@@ -98,6 +98,15 @@ steps:
98 commands:
99 - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
100 parallelism: 4
101 + mirror:
102 + amd:
103 + label: ":amd: (MI250) Multimodal Processor Shard %N"
104 + device: mi250_1
105 + no_gpu: true
106 + working_dir: "/vllm-workspace/tests"
107 + timeout_in_minutes: 80
108 + depends_on:
109 + - image-build-amd
110
111 - label: ":nvidia: (H200 MIG 18GB) Multimodal Processor"
112 key: multi-modal-processor
.buildkite/test_areas/pytorch.yaml
+8
@@ -160,6 +160,14 @@ steps:
160 # Use `find` to launch multiple instances of pytest so that
161 # they do not suffer from https://github.com/vllm-project/vllm/issues/28965
162 - "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_cudagraph.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'"
163 + mirror:
164 + amd:
165 + label: ":amd: (MI355) PyTorch Fullgraph"
166 + dind: false
167 + device: mi355_1
168 + timeout_in_minutes: 90
169 + depends_on:
170 + - image-build-amd
171
172 # Hopper-only DeepSeek-V2-Lite cases in this file require two 29.3-GiB model
173 # instances and cannot fit a 35GB MIG slice. L4 retains the original coverage:
.buildkite/test_areas/rust_frontend.yaml
+29
@@ -131,6 +131,20 @@ steps:
131 - export VLLM_USE_RUST_FRONTEND=1
132 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
133 - pytest -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
134 + mirror:
135 + amd:
136 + label: ":amd: (MI355) Rust Frontend Core Correctness"
137 + dind: false
138 + device: mi355_1
139 + timeout_in_minutes: 20
140 + depends_on:
141 + - image-build-amd
142 + source_file_dependencies:
143 + - rust/
144 + - vllm/entrypoints/openai/
145 + - tests/utils.py
146 + - tests/entrypoints/openai/correctness/test_lmeval.py
147 + - vllm/platforms/rocm.py
148
149 - label: ":nvidia: (H200 MIG 35GB) Rust Frontend Tool Use"
150 key: rust-frontend-tool-use
@@ -147,6 +161,21 @@ steps:
161 - export VLLM_USE_RUST_FRONTEND=1
162 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
163 - pytest -v -s tool_use --ignore=tool_use/mistral --models llama3.2 -k "not test_response_format_with_tool_choice_required and not test_parallel_tool_calls_false and not test_tool_call_and_choice"
164 + mirror:
165 + amd:
166 + label: ":amd: (MI355) Rust Frontend Tool Use"
167 + dind: false
168 + device: mi355_1
169 + timeout_in_minutes: 25
170 + depends_on:
171 + - image-build-amd
172 + source_file_dependencies:
173 + - rust/
174 + - vllm/entrypoints/openai/
175 + - vllm/tool_parsers/
176 + - tests/utils.py
177 + - tests/tool_use/
178 + - vllm/platforms/rocm.py
179
180 - label: ":nvidia: (L4) Rust Frontend Distributed"
181 key: rust-frontend-distributed
.buildkite/test_areas/rust_frontend_cargo.yaml
+18
@@ -14,6 +14,15 @@ steps:
14 - .buildkite/scripts/run-rust-frontend-cargo-ci.sh
15 commands:
16 - .buildkite/scripts/run-rust-frontend-cargo-ci.sh style-clippy
17 + mirror:
18 + amd:
19 + label: ":amd: (MI250) Rust Frontend Cargo Style + Clippy"
20 + device: mi250_1
21 + no_gpu: true
22 + working_dir: "/vllm-workspace"
23 + timeout_in_minutes: 35
24 + depends_on:
25 + - image-build-amd
26
27 - label: ":computer: (CPU) Rust Frontend Cargo"
28 key: rust-frontend-cargo-tests
@@ -28,3 +37,12 @@ steps:
37 - .buildkite/scripts/run-rust-frontend-cargo-ci.sh
38 commands:
39 - .buildkite/scripts/run-rust-frontend-cargo-ci.sh test
40 + mirror:
41 + amd:
42 + label: ":amd: (MI250) Rust Frontend Cargo"
43 + device: mi250_1
44 + no_gpu: true
45 + working_dir: "/vllm-workspace"
46 + timeout_in_minutes: 35
47 + depends_on:
48 + - image-build-amd