[ROCm][CI] Stage E gating (#50920)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
Andreas Karatzas committed
Aug 28, 2026 at 23:52 UTC
cacc429f62c3738c9c95093e9bd410e96103221a
10 files changed
+234
.buildkite/test_areas/compile.yaml
+43
@@ -83,6 +83,27 @@ steps:
83
- pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "inductor_partition and not +rms_norm and not +quant_fp8"
84
# Qwen/Deepseek requires +quant_fp8 as -quant_fp8 rms+quant fusion is not supported
85
- pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "inductor_partition and not +rms_norm and +quant_fp8 and (qwen3 or deepseek)"
86
+ mirror:
87
+ amd:
88
+ label: ":amd: (MI355) Fusion E2E Quick"
89
+ dind: false
90
+ device: mi355_1
91
+ working_dir: "/vllm-workspace/"
92
+ timeout_in_minutes: 25
93
+ depends_on:
94
+ - image-build-amd
95
+ source_file_dependencies:
96
+ - csrc/quantization/
97
+ - vllm/model_executor/
98
+ - vllm/v1/attention/
99
+ - vllm/compilation/
100
+ - tests/compile/fusions_e2e/
101
+ - vllm/_aiter_ops.py
102
+ - vllm/platforms/rocm.py
103
+ commands:
104
+ - rocm-smi
105
+ - "pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k 'inductor_partition and not +rms_norm and not +quant_fp8'"
106
+ - "pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k 'inductor_partition and +rms_norm and +quant_fp8 and qwen3'"
107
108
- label: ":nvidia: (H100) Fusion E2E Config Sweep"
109
key: fusion-e2e-config-sweep-h100
@@ -103,6 +124,28 @@ steps:
124
- nvidia-smi
125
# Run just llama3 (fp8) for all config combinations
126
- pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "llama-3"
127
+ mirror:
128
+ amd:
129
+ label: ":amd: (MI355) Fusion E2E Config Sweep"
130
+ dind: false
131
+ device: mi355_1
132
+ working_dir: "/vllm-workspace/"
133
+ timeout_in_minutes: 30
134
+ depends_on:
135
+ - image-build-amd
136
+ source_file_dependencies:
137
+ - csrc/quantization/
138
+ - vllm/compilation/
139
+ - vllm/model_executor/layers/layernorm.py
140
+ - vllm/model_executor/layers/activation.py
141
+ - vllm/model_executor/layers/attention/attention.py
142
+ - vllm/model_executor/layers/quantization/input_quant_fp8.py
143
+ - tests/compile/fusions_e2e/
144
+ - vllm/_aiter_ops.py
145
+ - vllm/platforms/rocm.py
146
+ commands:
147
+ - rocm-smi
148
+ - pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "llama-3"
149
150
- label: ":nvidia: (B200) Fusion E2E Config Sweep"
151
key: fusion-e2e-config-sweep-b200
.buildkite/test_areas/cuda.yaml
+25
@@ -17,6 +17,18 @@ steps:
17
- pytest -v -s cuda/test_cuda_context.py
18
- pytest -v -s cuda/test_platform_no_cuda_init.py
19
- pytest -v -s cuda/test_cuda_compatibility_path.py
20
+ mirror:
21
+ amd:
22
+ label: ":amd: (MI355) CUDA Platform"
23
+ dind: false
24
+ device: mi355_1
25
+ working_dir: "/vllm-workspace/tests"
26
+ timeout_in_minutes: 20
27
+ depends_on:
28
+ - image-build-amd
29
+ source_file_dependencies:
30
+ - vllm/
31
+ - tests/cuda
32
33
- label: ":nvidia: (H200) CUDAGraph"
34
device: h200_35gb
@@ -34,3 +46,16 @@ steps:
46
- pytest -v -s v1/cudagraph/test_cudagraph_mode.py
47
- pytest -v -s v1/cudagraph/test_breakable_cudagraph.py
48
- pytest -v -s v1/cudagraph/test_encoder_cudagraph.py
49
+ mirror:
50
+ amd:
51
+ label: ":amd: (MI250) CUDAGraph"
52
+ device: mi250_1
53
+ timeout_in_minutes: 45
54
+ depends_on:
55
+ - image-build-amd
56
+ source_file_dependencies:
57
+ - tests/v1/cudagraph
58
+ - vllm/v1/cudagraph_dispatcher.py
59
+ - vllm/config/compilation.py
60
+ - vllm/compilation
61
+ - vllm/platforms/rocm.py
.buildkite/test_areas/docker.yaml
+16
@@ -14,3 +14,19 @@ steps:
14
- tests/tools/test_docker_build_metadata_args.py
15
commands:
16
- pytest -v -s tools/test_docker_build_metadata_args.py
17
+ mirror:
18
+ amd:
19
+ label: ":amd: (MI250) Docker Build Metadata"
20
+ device: mi250_1
21
+ no_gpu: true
22
+ timeout_in_minutes: 35
23
+ depends_on:
24
+ - image-build-amd
25
+ source_file_dependencies:
26
+ - .buildkite/scripts/docker-build-metadata-args.sh
27
+ - .buildkite/scripts/ci-bake-rocm.sh
28
+ - docker/Dockerfile.rocm
29
+ - docker/Dockerfile.rocm_base
30
+ - docker/ci-rocm.hcl
31
+ - docker/docker-bake-rocm.hcl
32
+ - tests/tools/test_docker_build_metadata_args.py
.buildkite/test_areas/e2e_integration.yaml
+20
@@ -51,3 +51,23 @@ steps:
51
working_dir: "/vllm-workspace"
52
commands:
53
- bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_prefetch_offload.sh 0.25 200 8030
54
+ mirror:
55
+ amd:
56
+ label: ":amd: (MI355) DeepSeek V2-Lite Prefetch Offload Accuracy"
57
+ dind: false
58
+ device: mi355_1
59
+ working_dir: "/vllm-workspace"
60
+ timeout_in_minutes: 20
61
+ depends_on:
62
+ - image-build-amd
63
+ source_file_dependencies:
64
+ - vllm/model_executor/models/
65
+ - vllm/model_executor/model_loader/
66
+ - vllm/model_executor/layers/fused_moe/
67
+ - vllm/model_executor/layers/quantization/
68
+ - vllm/v1/attention/backends/
69
+ - vllm/v1/attention/backends/mla/
70
+ - vllm/v1/attention/selector.py
71
+ - .buildkite/scripts/scheduled_integration_test/
72
+ - vllm/_aiter_ops.py
73
+ - vllm/platforms/rocm.py
.buildkite/test_areas/kernels.yaml
+14
@@ -235,6 +235,13 @@ steps:
235
- vllm/model_executor/layers/mamba/ops
236
commands:
237
- pytest -v -s kernels/mamba
238
+ mirror:
239
+ amd:
240
+ label: ":amd: (MI250) Mamba Kernels"
241
+ device: mi250_1
242
+ timeout_in_minutes: 55
243
+ depends_on:
244
+ - image-build-amd
245
246
- label: ":nvidia: (H100) DeepGEMM Kernels"
247
key: kernels-deepgemm-test-h100
@@ -393,6 +400,13 @@ steps:
400
- pip install helion==1.4.0
401
- pytest -v -s kernels/helion/ --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
402
parallelism: 2
403
+ mirror:
404
+ amd:
405
+ label: ":amd: (MI250) Helion Kernels Shard %N"
406
+ device: mi250_1
407
+ timeout_in_minutes: 45
408
+ depends_on:
409
+ - image-build-amd
410
411
412
- label: ":nvidia: (H100) FP8 MoE Kernels"
.buildkite/test_areas/lm_eval.yaml
+16
@@ -395,3 +395,19 @@ steps:
395
- tests/evals/mrcr/
396
commands:
397
- pytest -s -v evals/mrcr/test_mrcr_correctness.py --config-list-file=evals/mrcr/configs/models-small.txt
398
+ mirror:
399
+ amd:
400
+ label: ":amd: (MI355) MRCR Eval Small Models"
401
+ dind: false
402
+ device: mi355_1
403
+ timeout_in_minutes: 40
404
+ depends_on:
405
+ - image-build-amd
406
+ source_file_dependencies:
407
+ - vllm/model_executor/models/
408
+ - vllm/model_executor/model_loader/
409
+ - vllm/v1/attention/backends/
410
+ - vllm/v1/attention/selector.py
411
+ - vllm/_aiter_ops.py
412
+ - vllm/platforms/rocm.py
413
+ - tests/evals/mrcr/
.buildkite/test_areas/model_runner_v2.yaml
+16
@@ -149,3 +149,19 @@ steps:
149
- >-
150
pytest -v -s
151
'v1/e2e/spec_decode/mtp/test_mtp.py::test_mtp_correctness[qwen3_5-hybrid]'
152
+ mirror:
153
+ amd:
154
+ label: ":amd: (MI355) Model Runner V2 Spec Decode"
155
+ dind: false
156
+ device: mi355_1
157
+ timeout_in_minutes: 75
158
+ depends_on:
159
+ - image-build-amd
160
+ source_file_dependencies:
161
+ - vllm/platforms/rocm.py
162
+ - vllm/v1/worker/gpu/
163
+ - vllm/v1/worker/gpu_worker.py
164
+ - tests/v1/spec_decode/test_max_len.py
165
+ - tests/v1/spec_decode/test_rejection_sampler_utils.py
166
+ - tests/v1/spec_decode/test_synthetic_rejection_sampler_utils.py
167
+ - tests/v1/e2e/spec_decode/
.buildkite/test_areas/models_basic.yaml
+8
@@ -14,6 +14,14 @@ steps:
14
commands:
15
# Run a subset of model initialization tests
16
- pytest -v -s models/test_initialization.py::test_can_initialize_small_subset
17
+ mirror:
18
+ amd:
19
+ label: ":amd: (MI355) Basic Models (Initialization)"
20
+ dind: false
21
+ device: mi355_1
22
+ timeout_in_minutes: 25
23
+ depends_on:
24
+ - image-build-amd
25
26
- label: ":nvidia: (H200) Basic Models (Extra Initialization) Shard %N"
27
device: h200_35gb
.buildkite/test_areas/models_language.yaml
+33
@@ -144,6 +144,32 @@ steps:
144
- tests/models/language/generation_ppl_test
145
commands:
146
- pytest -v -s models/language/generation_ppl_test
147
+ mirror:
148
+ amd:
149
+ label: ":amd: (MI250) Language Models (PPL)"
150
+ device: mi250_1
151
+ timeout_in_minutes: 65
152
+ depends_on:
153
+ - image-build-amd
154
+ source_file_dependencies:
155
+ - vllm/model_executor/models/qwen3_5.py
156
+ - vllm/model_executor/models/qwen3_5_mtp.py
157
+ - vllm/transformers_utils/configs/qwen3_5.py
158
+ - vllm/transformers_utils/configs/qwen3_5_moe.py
159
+ - vllm/model_executor/models/qwen2.py
160
+ - vllm/model_executor/models/qwen3.py
161
+ - vllm/model_executor/models/qwen3_next.py
162
+ - vllm/model_executor/models/qwen3_next_mtp.py
163
+ - vllm/model_executor/layers/fla/ops/
164
+ - vllm/_aiter_ops.py
165
+ - vllm/v1/attention/backends/triton_attn.py
166
+ - vllm/v1/attention/backends/rocm_attn.py
167
+ - vllm/v1/attention/backends/rocm_aiter_unified_attn.py
168
+ - vllm/v1/attention/backends/rocm_aiter_fa.py
169
+ - vllm/v1/attention/backends/flex_attention.py
170
+ - vllm/v1/attention/ops/
171
+ - vllm/platforms/rocm.py
172
+ - tests/models/language/generation_ppl_test
173
174
- label: ":nvidia: (H200) Language Models (Extended Pooling) Shard %N"
175
device: h200_35gb
@@ -180,3 +206,10 @@ steps:
206
- tests/models/language/pooling_mteb_test
207
commands:
208
- pytest -v -s models/language/pooling_mteb_test
209
+ mirror:
210
+ amd:
211
+ label: ":amd: (MI250) Language Models (MTEB)"
212
+ device: mi250_1
213
+ timeout_in_minutes: 60
214
+ depends_on:
215
+ - image-build-amd
.buildkite/test_areas/rust_frontend.yaml
+43
@@ -36,6 +36,29 @@ steps:
36
- pytest -v -s entrypoints/openai/test_return_token_ids.py -k "not test_comparison"
37
- pytest -v -s entrypoints/serve/instrumentator/test_uds.py
38
- pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server"
39
+ mirror:
40
+ amd:
41
+ label: ":amd: (MI355) Rust Frontend OpenAI Coverage"
42
+ dind: false
43
+ device: mi355_1
44
+ timeout_in_minutes: 35
45
+ depends_on:
46
+ - image-build-amd
47
+ source_file_dependencies:
48
+ - rust/
49
+ - vllm/benchmarks/
50
+ - vllm/entrypoints/openai/
51
+ - vllm/entrypoints/serve/
52
+ - vllm/v1/sample/
53
+ - tests/utils.py
54
+ - tests/benchmarks/test_serve_cli.py
55
+ - tests/entrypoints/openai/chat_completion/test_chat_completion.py
56
+ - tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py
57
+ - tests/entrypoints/openai/completion/test_shutdown.py
58
+ - tests/entrypoints/openai/test_return_token_ids.py
59
+ - tests/entrypoints/openai/test_uds.py
60
+ - tests/v1/sample/test_logprobs_e2e.py
61
+ - vllm/platforms/rocm.py
62
63
- label: ":nvidia: (H200) Rust Frontend Serve/Admin Coverage"
64
timeout_in_minutes: 25
@@ -65,6 +88,26 @@ steps:
88
# - pytest -v -s entrypoints/serve/dev/test_sleep.py
89
# /tokenizer_info is not implemented in the Rust frontend (the CLI flag is accepted as a no-op).
90
- pytest -v -s entrypoints/serve/tokenize/test_tokenization.py -k "not tokenizer_info"
91
+ mirror:
92
+ amd:
93
+ label: ":amd: (MI355) Rust Frontend Serve/Admin Coverage"
94
+ dind: false
95
+ device: mi355_1
96
+ timeout_in_minutes: 30
97
+ depends_on:
98
+ - image-build-amd
99
+ source_file_dependencies:
100
+ - rust/
101
+ - vllm/entrypoints/openai/
102
+ - vllm/entrypoints/serve/
103
+ - vllm/v1/engine/
104
+ - tests/utils.py
105
+ - tests/entrypoints/serve/dev/rpc/test_collective_rpc.py
106
+ - tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py
107
+ - tests/entrypoints/serve/instrumentator/test_basic.py
108
+ - tests/entrypoints/serve/instrumentator/test_metrics.py
109
+ - tests/entrypoints/serve/tokenize/test_tokenization.py
110
+ - vllm/platforms/rocm.py
111
112
- label: ":nvidia: (H200) Rust Frontend Core Correctness"
113
timeout_in_minutes: 20