[CI][ROCm] Add ten AMD parity groups (#57876)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
Andreas Karatzas committed
Sep 21, 2026 at 06:08 UTC
82daf9f5756e1868be0aa751afaec4726beca12a
16 files changed
+525
-3
.buildkite/test-amd.yaml
+239
@@ -1361,6 +1361,23 @@ steps:
1361
- bash ../tools/install_torchcodec_rocm.sh || exit 1
1362
- pytest -s entrypoints/openai/correctness/
1363
1364
+- label: ":amd: (MI300) Scale-out EC Connector E2E"
1365
+ timeout_in_minutes: 60
1366
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1367
+ dind: false
1368
+ agent_pool: mi300_2
1369
+ num_gpus: 2
1370
+ optional: true
1371
+ working_dir: "/vllm-workspace/tests"
1372
+ source_file_dependencies:
1373
+ - vllm/distributed/ec_transfer/
1374
+ - vllm/entrypoints/scale_out/
1375
+ - tests/entrypoints/scale_out/ec_integration/
1376
+ - vllm/platforms/rocm.py
1377
+ commands:
1378
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
1379
+ - bash entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh
1380
+
1381
#----------------------------------------------------------- mi300 · evals -----------------------------------------------------------#
1382
1383
- label: ":amd: (MI300) DeepSeek V2-Lite Prefetch Offload Accuracy" # TBD
@@ -3190,6 +3207,58 @@ steps:
3207
v1/e2e/spec_decode/draft_model/test_draft_model.py::test_draft_model_tensor_parallelism
3208
v1/e2e/spec_decode/draft_model/test_draft_model.py::test_draft_model_engine_args_tensor_parallelism
3209
3210
+- label: ":amd: (MI300) Mooncake EC TCP E2E"
3211
+ timeout_in_minutes: 60
3212
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
3213
+ dind: false
3214
+ agent_pool: mi300_2
3215
+ num_gpus: 2
3216
+ optional: true
3217
+ working_dir: "/vllm-workspace"
3218
+ source_file_dependencies:
3219
+ - vllm/distributed/ec_transfer/
3220
+ - vllm/config/ec_transfer.py
3221
+ - vllm/config/multimodal.py
3222
+ - vllm/config/vllm.py
3223
+ - vllm/multimodal/
3224
+ - vllm/v1/core/encoder_cache_manager.py
3225
+ - vllm/v1/core/sched/
3226
+ - vllm/v1/engine/
3227
+ - vllm/v1/worker/ec_connector_model_runner_mixin.py
3228
+ - vllm/v1/worker/gpu_model_runner.py
3229
+ - vllm/v1/worker/gpu_worker.py
3230
+ - vllm/v1/worker/gpu/ec_connector.py
3231
+ - vllm/v1/worker/gpu/model_runner.py
3232
+ - vllm/v1/worker/mm_encoder_model_runner.py
3233
+ - vllm/v1/worker/encoder_cudagraph.py
3234
+ - vllm/v1/worker/encoder_cudagraph_defs.py
3235
+ - vllm/v1/attention/ops/vit_attn_wrappers.py
3236
+ - vllm/v1/worker/gpu/model_states/
3237
+ - vllm/v1/worker/gpu/mm/
3238
+ - tests/v1/cudagraph/test_encoder_cudagraph.py
3239
+ - tests/models/multimodal/generation/test_vit_cudagraph.py
3240
+ - examples/disaggregated/disaggregated_encoder/
3241
+ - tests/v1/ec_connector/
3242
+ - requirements/kv_connectors.txt
3243
+ - vllm/platforms/rocm.py
3244
+ - requirements/rocm.txt
3245
+ commands:
3246
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
3247
+ - export PYTHON_BIN='python'
3248
+ - export PYTHONUNBUFFERED='1'
3249
+ - export VLLM_HOST_IP='127.0.0.1'
3250
+ - export VLLM_USE_V2_MODEL_RUNNER='1'
3251
+ - export E_CUDAGRAPH_MM_ENCODER='1'
3252
+ - export E_MM_ENCODER_ATTN_BACKEND='TRITON_ATTN'
3253
+ - export MOONCAKE_EC_PROTOCOL='tcp'
3254
+ - export USE_MM_PROMPTS='1'
3255
+ - export SKIP_BASELINE='0'
3256
+ - export CONCURRENCY='3'
3257
+ - export REPEAT='2'
3258
+ - export LOG_PATH='/tmp/mooncake-ec-e2e'
3259
+ - export BASELINE_FILE='/tmp/mooncake-ec-e2e/baseline.json'
3260
+ - bash tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh
3261
+
3262
- label: ":amd: (MI300) CrossLayer KV layout Distributed NixlConnector PD accuracy" # TBD
3263
timeout_in_minutes: 60
3264
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
@@ -3385,6 +3454,27 @@ steps:
3454
3455
#-------------------------------------------------------- mi355 · distributed --------------------------------------------------------#
3456
3457
+- label: ":amd: (MI355) Sharded RDT Weight Transfer"
3458
+ timeout_in_minutes: 45
3459
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3460
+ dind: false
3461
+ agent_pool: mi355_1
3462
+ num_gpus: 1
3463
+ optional: true
3464
+ working_dir: "/vllm-workspace/tests"
3465
+ source_file_dependencies:
3466
+ - vllm/distributed/weight_transfer/
3467
+ - tests/distributed/test_sharded_rdt_plan.py
3468
+ - tests/distributed/test_sharded_rdt_producer.py
3469
+ - tests/distributed/test_sharded_rdt_trainer.py
3470
+ - vllm/platforms/rocm.py
3471
+ commands:
3472
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
3473
+ - uv pip install --system --no-deps 'ray==2.56.1'
3474
+ - pytest -v -s distributed/test_sharded_rdt_plan.py
3475
+ - pytest -v -s distributed/test_sharded_rdt_producer.py
3476
+ - pytest -v -s distributed/test_sharded_rdt_trainer.py
3477
+
3478
- label: ":amd: (MI355) Distributed DP Basic" # TBD
3479
timeout_in_minutes: 45
3480
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
@@ -3464,6 +3554,35 @@ steps:
3554
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
3555
- DP_EP=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
3556
3557
+- label: ":amd: (MI355) Kimi-Linear-48B-A3B Disaggregated DP EP"
3558
+ timeout_in_minutes: 105
3559
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3560
+ dind: false
3561
+ agent_pool: mi355_4
3562
+ num_gpus: 4
3563
+ optional: true
3564
+ working_dir: "/vllm-workspace/tests"
3565
+ source_file_dependencies:
3566
+ - vllm/platforms/rocm.py
3567
+ - vllm/models/kimi_k3/
3568
+ - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
3569
+ - tests/v1/kv_connector/nixl_integration/
3570
+ - requirements/kv_connectors_rocm.txt
3571
+ commands:
3572
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
3573
+ - export ENABLE_HMA_FLAG='1'
3574
+ - export DP_EP='1'
3575
+ - export GPU_MEMORY_UTILIZATION='0.9'
3576
+ - export VLLM_SSM_CONV_STATE_LAYOUT='DS'
3577
+ - export PREFILLER_TP_SIZE='2'
3578
+ - export DECODER_TP_SIZE='2'
3579
+ - export MODEL_NAMES='moonshotai/Kimi-Linear-48B-A3B-Instruct'
3580
+ - export PREFILL_BLOCK_SIZE='2048'
3581
+ - export DECODE_BLOCK_SIZE='2048'
3582
+ - export VLLM_SERVE_EXTRA_ARGS='--trust-remote-code,--mamba-cache-mode align,--max-model-len 32768'
3583
+ - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
3584
+ - bash v1/kv_connector/nixl_integration/run_accuracy_test.sh
3585
+
3586
- label: ":amd: (MI355) Distributed Features" # TBD
3587
timeout_in_minutes: 90
3588
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
@@ -3661,6 +3780,51 @@ steps:
3780
- uv pip install --system 'gpt-oss[eval]==0.0.5'
3781
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt
3782
3783
+- label: ":amd: (MI355) LM Eval Watermarking"
3784
+ timeout_in_minutes: 45
3785
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3786
+ dind: false
3787
+ agent_pool: mi355_1
3788
+ num_gpus: 1
3789
+ optional: true
3790
+ working_dir: "/vllm-workspace/tests"
3791
+ source_file_dependencies:
3792
+ - .buildkite/test_areas/lm_eval.yaml
3793
+ - tests/evals/gsm8k/
3794
+ - vllm/config/watermarking.py
3795
+ - vllm/sampling_params.py
3796
+ - vllm/v1/watermarking/
3797
+ - vllm/v1/worker/gpu/
3798
+ - vllm/platforms/rocm.py
3799
+ commands:
3800
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
3801
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-watermark.txt
3802
+
3803
+- label: ":amd: (MI355) LM Eval Watermark Feature Combination"
3804
+ timeout_in_minutes: 60
3805
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3806
+ dind: false
3807
+ agent_pool: mi355_2
3808
+ num_gpus: 2
3809
+ optional: true
3810
+ working_dir: "/vllm-workspace/tests"
3811
+ source_file_dependencies:
3812
+ - .buildkite/test_areas/lm_eval.yaml
3813
+ - tests/evals/gsm8k/
3814
+ - vllm/config/speculative.py
3815
+ - vllm/config/vllm.py
3816
+ - vllm/config/watermarking.py
3817
+ - vllm/sampling_params.py
3818
+ - vllm/model_executor/models/qwen3_dspark.py
3819
+ - vllm/model_executor/models/qwen3_dflash.py
3820
+ - vllm/model_executor/warmup/
3821
+ - vllm/v1/watermarking/
3822
+ - vllm/v1/worker/gpu/
3823
+ - vllm/platforms/rocm.py
3824
+ commands:
3825
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
3826
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-watermark-dspark-tp2.txt
3827
+
3828
- label: ":amd: (MI355) LM Eval Qwen3-5 Models" # TBD
3829
timeout_in_minutes: 40
3830
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
@@ -3684,6 +3848,27 @@ steps:
3848
commands:
3849
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-mi355.txt
3850
3851
+- label: ":amd: (MI355) Qwen3-8-Flash-Next-FP8 Accuracy Eval"
3852
+ timeout_in_minutes: 105
3853
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3854
+ dind: false
3855
+ agent_pool: mi355_4
3856
+ num_gpus: 4
3857
+ optional: true
3858
+ working_dir: "/vllm-workspace/tests"
3859
+ source_file_dependencies:
3860
+ - csrc/libtorch_stable/gdn/
3861
+ - tests/evals/qwen4_exp/
3862
+ - vllm/model_executor/layers/mamba/
3863
+ - vllm/models/qwen4_exp/
3864
+ - vllm/transformers_utils/configs/qwen4_exp.py
3865
+ - vllm/v1/attention/backends/short_conv_attn.py
3866
+ - vllm/platforms/rocm.py
3867
+ commands:
3868
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
3869
+ - uv pip install --system 'evalscope==1.10.0'
3870
+ - pytest -s -v evals/qwen4_exp/test_accuracy.py --config-list-file=configs/models-rocm.txt
3871
+
3872
- label: ":amd: (MI355) LM Eval Small Models Distributed"
3873
timeout_in_minutes: 55
3874
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
@@ -3705,6 +3890,25 @@ steps:
3890
commands:
3891
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt
3892
3893
+- label: ":amd: (MI355) MoE Refactor Integration Shard %N"
3894
+ timeout_in_minutes: 105
3895
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3896
+ dind: false
3897
+ agent_pool: mi355_2
3898
+ num_gpus: 2
3899
+ parallelism: 4
3900
+ optional: true
3901
+ working_dir: "/vllm-workspace/tests"
3902
+ source_file_dependencies:
3903
+ - vllm/platforms/rocm.py
3904
+ - tests/evals/gsm8k/
3905
+ - vllm/model_executor/layers/fused_moe/
3906
+ - vllm/model_executor/layers/quantization/
3907
+ - vllm/_aiter_ops.py
3908
+ commands:
3909
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
3910
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor/config-rocm-shard-$$BUILDKITE_PARALLEL_JOB.txt
3911
+
3912
- label: ":amd: (MI355) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy" # TBD
3913
timeout_in_minutes: 25
3914
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
@@ -3925,6 +4129,22 @@ steps:
4129
- pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py
4130
- pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_op_registration.py
4131
4132
+- label: ":amd: (MI355) DeepSeek V4 Fused Kernels"
4133
+ timeout_in_minutes: 60
4134
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4135
+ dind: false
4136
+ agent_pool: mi355_1
4137
+ num_gpus: 1
4138
+ optional: true
4139
+ working_dir: "/vllm-workspace/tests"
4140
+ source_file_dependencies:
4141
+ - csrc/libtorch_stable/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu
4142
+ - vllm/models/deepseek_v4/common/ops/
4143
+ - tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
4144
+ - vllm/platforms/rocm.py
4145
+ commands:
4146
+ - pytest -v -s kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
4147
+
4148
- label: ":amd: (MI355) Attention Kernels Shard %N" # TBD
4149
timeout_in_minutes: 90
4150
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
@@ -4418,6 +4638,25 @@ steps:
4638
- pytest -v -s v1/e2e/general/test_kv_sharing_fast_prefill.py
4639
- pytest -v -s v1/e2e/general/test_mamba_prefix_cache.py::test_mamba_prefix_cache_mrv2 v1/e2e/general/test_mamba_prefix_cache.py::test_mamba_prefix_cache_mrv2_async
4640
4641
+- label: ":amd: (MI355) Extract Hidden States Integration Single GPU"
4642
+ timeout_in_minutes: 45
4643
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4644
+ dind: false
4645
+ agent_pool: mi355_1
4646
+ num_gpus: 1
4647
+ optional: true
4648
+ working_dir: "/vllm-workspace"
4649
+ source_file_dependencies:
4650
+ - vllm/v1/spec_decode/extract_hidden_states.py
4651
+ - vllm/model_executor/models/extract_hidden_states.py
4652
+ - vllm/transformers_utils/configs/extract_hidden_states.py
4653
+ - vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py
4654
+ - tests/v1/kv_connector/extract_hidden_states_integration
4655
+ - vllm/platforms/rocm.py
4656
+ commands:
4657
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
4658
+ - pytest -v -s -m 'not distributed' tests/v1/kv_connector/extract_hidden_states_integration
4659
+
4660
- label: ":amd: (MI355) V1 Attention Shard %N" # TBD
4661
timeout_in_minutes: 125
4662
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
.buildkite/test_areas/disaggregated.yaml
+21
@@ -318,3 +318,24 @@ steps:
318
commands:
319
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
320
- bash v1/kv_connector/nixl_integration/run_accuracy_test.sh
321
+ mirror:
322
+ amd:
323
+ label: ":amd: (MI355) Kimi-Linear-48B-A3B Disaggregated DP EP"
324
+ dind: false
325
+ device: mi355_4
326
+ num_devices: 4
327
+ timeout_in_minutes: 105
328
+ working_dir: /vllm-workspace/tests
329
+ depends_on:
330
+ - image-build-amd
331
+ source_file_dependencies:
332
+ - vllm/platforms/rocm.py
333
+ - vllm/models/kimi_k3/
334
+ - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
335
+ - tests/v1/kv_connector/nixl_integration/
336
+ - requirements/kv_connectors_rocm.txt
337
+ env:
338
+ VLLM_WORKER_MULTIPROC_METHOD: "spawn"
339
+ commands:
340
+ - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
341
+ - bash v1/kv_connector/nixl_integration/run_accuracy_test.sh
.buildkite/test_areas/disaggregated_ec.yaml
+19
@@ -14,3 +14,22 @@ steps:
14
- tests/entrypoints/scale_out/ec_integration/
15
commands:
16
- bash entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh
17
+ mirror:
18
+ amd:
19
+ label: ":amd: (MI300) Scale-out EC Connector E2E"
20
+ dind: false
21
+ device: mi300_2
22
+ num_devices: 2
23
+ timeout_in_minutes: 60
24
+ working_dir: /vllm-workspace/tests
25
+ depends_on:
26
+ - image-build-amd
27
+ source_file_dependencies:
28
+ - vllm/distributed/ec_transfer/
29
+ - vllm/entrypoints/scale_out/
30
+ - tests/entrypoints/scale_out/ec_integration/
31
+ - vllm/platforms/rocm.py
32
+ env:
33
+ VLLM_WORKER_MULTIPROC_METHOD: "spawn"
34
+ commands:
35
+ - bash entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh
.buildkite/test_areas/disaggregated_mooncake.yaml
+42
@@ -83,3 +83,45 @@ steps:
83
PY
84
)"
85
- bash tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh
86
+ mirror:
87
+ amd:
88
+ label: ":amd: (MI300) Mooncake EC TCP E2E"
89
+ dind: false
90
+ device: mi300_2
91
+ num_devices: 2
92
+ timeout_in_minutes: 60
93
+ working_dir: /vllm-workspace
94
+ depends_on:
95
+ - image-build-amd
96
+ source_file_dependencies:
97
+ - vllm/distributed/ec_transfer/
98
+ - vllm/config/ec_transfer.py
99
+ - vllm/config/multimodal.py
100
+ - vllm/config/vllm.py
101
+ - vllm/multimodal/
102
+ - vllm/v1/core/encoder_cache_manager.py
103
+ - vllm/v1/core/sched/
104
+ - vllm/v1/engine/
105
+ - vllm/v1/worker/ec_connector_model_runner_mixin.py
106
+ - vllm/v1/worker/gpu_model_runner.py
107
+ - vllm/v1/worker/gpu_worker.py
108
+ - vllm/v1/worker/gpu/ec_connector.py
109
+ - vllm/v1/worker/gpu/model_runner.py
110
+ - vllm/v1/worker/mm_encoder_model_runner.py
111
+ - vllm/v1/worker/encoder_cudagraph.py
112
+ - vllm/v1/worker/encoder_cudagraph_defs.py
113
+ - vllm/v1/attention/ops/vit_attn_wrappers.py
114
+ - vllm/v1/worker/gpu/model_states/
115
+ - vllm/v1/worker/gpu/mm/
116
+ - tests/v1/cudagraph/test_encoder_cudagraph.py
117
+ - tests/models/multimodal/generation/test_vit_cudagraph.py
118
+ - examples/disaggregated/disaggregated_encoder/
119
+ - tests/v1/ec_connector/
120
+ - requirements/kv_connectors.txt
121
+ - vllm/platforms/rocm.py
122
+ - requirements/rocm.txt
123
+ env:
124
+ VLLM_WORKER_MULTIPROC_METHOD: "spawn"
125
+ PYTHON_BIN: "python"
126
+ commands:
127
+ - bash tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh
.buildkite/test_areas/distributed.yaml
+23
@@ -33,6 +33,29 @@ steps:
33
- pytest -v -s distributed/test_sharded_rdt_plan.py
34
- pytest -v -s distributed/test_sharded_rdt_producer.py
35
- pytest -v -s distributed/test_sharded_rdt_trainer.py
36
+ mirror:
37
+ amd:
38
+ label: ":amd: (MI355) Sharded RDT Weight Transfer"
39
+ dind: false
40
+ device: mi355_1
41
+ num_devices: 1
42
+ timeout_in_minutes: 45
43
+ working_dir: /vllm-workspace/tests
44
+ depends_on:
45
+ - image-build-amd
46
+ source_file_dependencies:
47
+ - vllm/distributed/weight_transfer/
48
+ - tests/distributed/test_sharded_rdt_plan.py
49
+ - tests/distributed/test_sharded_rdt_producer.py
50
+ - tests/distributed/test_sharded_rdt_trainer.py
51
+ - vllm/platforms/rocm.py
52
+ env:
53
+ VLLM_WORKER_MULTIPROC_METHOD: "spawn"
54
+ commands:
55
+ - uv pip install --system --no-deps 'ray==2.56.1'
56
+ - pytest -v -s distributed/test_sharded_rdt_plan.py
57
+ - pytest -v -s distributed/test_sharded_rdt_producer.py
58
+ - pytest -v -s distributed/test_sharded_rdt_trainer.py
59
60
- label: ":nvidia: (L4) Distributed DP Basic"
61
key: distributed-dp-tests-2-gpus
.buildkite/test_areas/kernels.yaml
+17
@@ -111,6 +111,23 @@ steps:
111
commands:
112
- pytest -v -s kernels/test_fused_deepseek_v4_*.py
113
- pytest -v -s kernels/test_top_k_per_row.py
114
+ mirror:
115
+ amd:
116
+ label: ":amd: (MI355) DeepSeek V4 Fused Kernels"
117
+ dind: false
118
+ device: mi355_1
119
+ num_devices: 1
120
+ timeout_in_minutes: 60
121
+ working_dir: /vllm-workspace/tests
122
+ depends_on:
123
+ - image-build-amd
124
+ source_file_dependencies:
125
+ - csrc/libtorch_stable/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu
126
+ - vllm/models/deepseek_v4/common/ops/
127
+ - tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
128
+ - vllm/platforms/rocm.py
129
+ commands:
130
+ - pytest -v -s kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
131
132
- label: ":nvidia: (B200) DeepSeek V4 Kernels"
133
key: deepseek-v4-kernel-test-b200
.buildkite/test_areas/lm_eval.yaml
+93
-1
@@ -66,6 +66,28 @@ steps:
66
autorun_on_main: true
67
commands:
68
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-watermark.txt
69
+ mirror:
70
+ amd:
71
+ label: ":amd: (MI355) LM Eval Watermarking"
72
+ dind: false
73
+ device: mi355_1
74
+ num_devices: 1
75
+ timeout_in_minutes: 45
76
+ working_dir: /vllm-workspace/tests
77
+ depends_on:
78
+ - image-build-amd
79
+ source_file_dependencies:
80
+ - .buildkite/test_areas/lm_eval.yaml
81
+ - tests/evals/gsm8k/
82
+ - vllm/config/watermarking.py
83
+ - vllm/sampling_params.py
84
+ - vllm/v1/watermarking/
85
+ - vllm/v1/worker/gpu/
86
+ - vllm/platforms/rocm.py
87
+ env:
88
+ VLLM_WORKER_MULTIPROC_METHOD: "spawn"
89
+ commands:
90
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-watermark.txt
91
92
# - label: LM Eval Large Models (4xA100)
93
# key: lm-eval-large-models-4xa100
@@ -203,6 +225,33 @@ steps:
225
autorun_on_main: true
226
commands:
227
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-watermark-dspark-tp2.txt
228
+ mirror:
229
+ amd:
230
+ label: ":amd: (MI355) LM Eval Watermark Feature Combination"
231
+ dind: false
232
+ device: mi355_2
233
+ num_devices: 2
234
+ timeout_in_minutes: 60
235
+ working_dir: /vllm-workspace/tests
236
+ depends_on:
237
+ - image-build-amd
238
+ source_file_dependencies:
239
+ - .buildkite/test_areas/lm_eval.yaml
240
+ - tests/evals/gsm8k/
241
+ - vllm/config/speculative.py
242
+ - vllm/config/vllm.py
243
+ - vllm/config/watermarking.py
244
+ - vllm/sampling_params.py
245
+ - vllm/model_executor/models/qwen3_dspark.py
246
+ - vllm/model_executor/models/qwen3_dflash.py
247
+ - vllm/model_executor/warmup/
248
+ - vllm/v1/watermarking/
249
+ - vllm/v1/worker/gpu/
250
+ - vllm/platforms/rocm.py
251
+ env:
252
+ VLLM_WORKER_MULTIPROC_METHOD: "spawn"
253
+ commands:
254
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-watermark-dspark-tp2.txt
255
256
- label: ":nvidia: (B200) LM Eval Spec Decode"
257
key: lm-eval-spec-decode-4xb200
@@ -329,7 +378,27 @@ steps:
378
# build 83851): shards ~12.4/13.6/12.4/14.5m of eval time. New configs must
379
# be added to exactly one config-h100-shard-*.txt.
380
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor/config-h100-shard-$$BUILDKITE_PARALLEL_JOB.txt
332
-
381
+ mirror:
382
+ amd:
383
+ label: ":amd: (MI355) MoE Refactor Integration Shard %N"
384
+ dind: false
385
+ device: mi355_2
386
+ num_devices: 2
387
+ timeout_in_minutes: 105
388
+ working_dir: /vllm-workspace/tests
389
+ depends_on:
390
+ - image-build-amd
391
+ source_file_dependencies:
392
+ - vllm/platforms/rocm.py
393
+ - tests/evals/gsm8k/
394
+ - vllm/model_executor/layers/fused_moe/
395
+ - vllm/model_executor/layers/quantization/
396
+ - vllm/_aiter_ops.py
397
+ env:
398
+ VLLM_WORKER_MULTIPROC_METHOD: "spawn"
399
+ commands:
400
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor/config-rocm-shard-$$BUILDKITE_PARALLEL_JOB.txt
401
+
402
- label: ":nvidia: (B200) MoE Refactor Integration TEMPORARY Shard %N"
403
key: moe-refactor-integration-test-b200-temporary
404
device: b200-k8s
@@ -578,6 +647,29 @@ steps:
647
commands:
648
- uv pip install --system 'evalscope==1.10.0'
649
- pytest -s -v evals/qwen4_exp/test_accuracy.py --config-list-file=configs/models-h200.txt
650
+ mirror:
651
+ amd:
652
+ label: ":amd: (MI355) Qwen3.8-Flash-Next-FP8 Accuracy Eval"
653
+ dind: false
654
+ device: mi355_4
655
+ num_devices: 4
656
+ timeout_in_minutes: 105
657
+ working_dir: /vllm-workspace/tests
658
+ depends_on:
659
+ - image-build-amd
660
+ source_file_dependencies:
661
+ - csrc/libtorch_stable/gdn/
662
+ - tests/evals/qwen4_exp/
663
+ - vllm/model_executor/layers/mamba/
664
+ - vllm/models/qwen4_exp/
665
+ - vllm/transformers_utils/configs/qwen4_exp.py
666
+ - vllm/v1/attention/backends/short_conv_attn.py
667
+ - vllm/platforms/rocm.py
668
+ env:
669
+ VLLM_WORKER_MULTIPROC_METHOD: "spawn"
670
+ commands:
671
+ - uv pip install --system 'evalscope==1.10.0'
672
+ - pytest -s -v evals/qwen4_exp/test_accuracy.py --config-list-file=configs/models-rocm.txt
673
674
- label: ":nvidia: (DGX) Spark GPQA Eval (GPT-OSS)"
675
key: gpqa-eval-gpt-oss-spark
.buildkite/test_areas/misc.yaml
+21
@@ -237,6 +237,27 @@ steps:
237
commands:
238
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
239
- pytest -v -s tests/v1/kv_connector/extract_hidden_states_integration
240
+ mirror:
241
+ amd:
242
+ label: ":amd: (MI355) Extract Hidden States Integration Single GPU"
243
+ dind: false
244
+ device: mi355_1
245
+ num_devices: 1
246
+ timeout_in_minutes: 45
247
+ working_dir: /vllm-workspace
248
+ depends_on:
249
+ - image-build-amd
250
+ source_file_dependencies:
251
+ - vllm/v1/spec_decode/extract_hidden_states.py
252
+ - vllm/model_executor/models/extract_hidden_states.py
253
+ - vllm/transformers_utils/configs/extract_hidden_states.py
254
+ - vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py
255
+ - tests/v1/kv_connector/extract_hidden_states_integration
256
+ - vllm/platforms/rocm.py
257
+ env:
258
+ VLLM_WORKER_MULTIPROC_METHOD: "spawn"
259
+ commands:
260
+ - pytest -v -s -m 'not distributed' tests/v1/kv_connector/extract_hidden_states_integration
261
262
- label: ":nvidia: (L4) Extract Hidden States Integration"
263
key: extract-hidden-states-integration-2-gpus
tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-BF16-triton-rocm.yaml
new
+5
@@ -0,0 +1,5 @@
1
+model_name: "Qwen/Qwen3-30B-A3B"
2
+accuracy_threshold: 0.88
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --moe-backend=triton"
tests/evals/gsm8k/configs/moe-refactor/config-rocm-shard-0.txt
new
+1
@@ -0,0 +1 @@
1
+Qwen3-30B-A3B-BF16-triton-rocm.yaml
tests/evals/gsm8k/configs/moe-refactor/config-rocm-shard-1.txt
new
+1
@@ -0,0 +1 @@
1
+Qwen3-30B-A3B-Fp8-AutoFp8-triton.yaml
tests/evals/gsm8k/configs/moe-refactor/config-rocm-shard-2.txt
new
+1
@@ -0,0 +1 @@
1
+Qwen3-30B-A3B-Fp8-CT-Block-triton.yaml
tests/evals/gsm8k/configs/moe-refactor/config-rocm-shard-3.txt
new
+1
@@ -0,0 +1 @@
1
+Mixtral-8x7B-Fp8-AutoFp8-triton.yaml
tests/evals/qwen4_exp/configs/Qwen3.8-Flash-Next-FP8-rocm.yaml
new
+26
@@ -0,0 +1,26 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+model_name: "Qwen/Qwen3.8-Flash-Next-FP8"
4
+datasets:
5
+ gsm8k:
6
+ metric_threshold: 0.98
7
+ tolerance: 0.02
8
+ aime25:
9
+ metric_threshold: 0.90
10
+ tolerance: 0.05
11
+eval_batch_size: 32
12
+startup_max_wait_seconds: 1800
13
+generation_config:
14
+ seed: 1236
15
+ do_sample: true
16
+ temperature: 0.7
17
+ top_p: 0.8
18
+ top_k: 20
19
+ repetition_penalty: 1.0
20
+ presence_penalty: 1.5
21
+ max_tokens: 32768
22
+ stream: true
23
+ extra_body:
24
+ chat_template_kwargs:
25
+ enable_thinking: true
26
+server_args: "--tensor-parallel-size 4 --mamba-cache-mode align"
tests/evals/qwen4_exp/configs/models-rocm.txt
new
+1
@@ -0,0 +1 @@
1
+Qwen3.8-Flash-Next-FP8-rocm.yaml
tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
+14
-2
@@ -96,7 +96,7 @@ def apply_rope_gptj_last_k(
96
# `e*c - o*s` fused form. This keeps the reference close to the kernel, but
97
# the fp32 reference and the fp32 GPU kernel can still round to bf16 on
98
# opposite sides of a round-to-nearest tie for a tiny number of elements at
99
- # high positions, so callers compare the RoPE region within 1 bf16 ULP.
99
+ # high positions, so callers allow for bf16 rounding in the RoPE region.
100
new_even = torch.addcmul(-odd * sin, even, cos)
101
new_odd = torch.addcmul(odd * cos, even, sin)
102
rope_rotated = torch.stack((new_even, new_odd), dim=-1).reshape(shape)
@@ -314,7 +314,19 @@ def test_q_path_without_qnorm_matches_rope_only_reference(num_tokens: int):
314
apply_q_norm=False,
315
)
316
317
- torch.testing.assert_close(q_out[:, :n_heads], q_ref, rtol=0, atol=0)
317
+ torch.testing.assert_close(
318
+ q_out[:, :n_heads, :NOPE_DIM], q_ref[..., :NOPE_DIM], rtol=0, atol=0
319
+ )
320
+ # HIP and PyTorch can contract different FP32 products into an FMA.
321
+ # Allow bf16 rounding and small cancellation errors only in the RoPE region
322
+ # on ROCm; the unrotated dimensions and padded heads must remain exact.
323
+ rope_rtol, rope_atol = (1e-2, 1e-6) if current_platform.is_rocm() else (0, 0)
324
+ torch.testing.assert_close(
325
+ q_out[:, :n_heads, NOPE_DIM:],
326
+ q_ref[..., NOPE_DIM:],
327
+ rtol=rope_rtol,
328
+ atol=rope_atol,
329
+ )
330
assert q_out[:, n_heads:].count_nonzero().item() == 0
331
332