@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 225d1090a0

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 7, 02:45 UTC

Enabling some B200-specific tests on MI355 (#35253)

Signed-off-by: Alexei V. Ivanov <alexei.ivanov@amd.com> Signed-off-by: Alexei-V-Ivanov-AMD <156011006+Alexei-V-Ivanov-AMD@users.noreply.github.com>

Alexei-V-Ivanov-AMD committed Mar 6, 2026 at 13:27 UTC 225d1090a0996710a23d58cfcd1d4d2b089cc553

4 files changed +89 -110

.buildkite/test-amd.yaml

+71 -110

@@ -499,17 +499,6 @@ steps:

499 - pytest -v -s v1/determinism/test_batch_invariance.py

500 - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py

501

502 -- label: V1 Test attention (B200) # 10min

503 - timeout_in_minutes: 30

504 - gpu: b200

505 - source_file_dependencies:

506 - - vllm/config/attention.py

507 - - vllm/model_executor/layers/attention

508 - - vllm/v1/attention

509 - - tests/v1/attention

510 - commands:

511 - - pytest -v -s v1/attention

512 -

502 - label: V1 Test others (CPU) # 5 mins

503 mirror_hardwares: [amdexperimental, amdproduction, amdtentative]

504 agent_pool: mi325_1

@@ -1185,47 +1174,40 @@ steps:

1174 # Whisper needs spawn method to avoid deadlock

1175 - VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/offline_inference/audio_language.py --model-type whisper

1176

1188 -- label: Blackwell Test # 21 min

1189 - timeout_in_minutes: 30

1177 +- label: Blackwell Fusion and Compile Tests # 30 min

1178 + timeout_in_minutes: 40

1179 working_dir: "/vllm-workspace/"

1180 gpu: b200

1192 - # optional: true

1181 source_file_dependencies:

1182 - csrc/quantization/fp4/

1195 - - csrc/attention/mla/

1196 - - csrc/quantization/cutlass_w8a8/moe/

1197 - - vllm/model_executor/layers/fused_moe/cutlass_moe.py

1198 - - vllm/model_executor/layers/fused_moe/flashinfer_cutlass_moe.py

1199 - - vllm/model_executor/layers/fused_moe/flashinfer_a2a_prepare_finalize.py

1183 - vllm/model_executor/layers/quantization/utils/flashinfer_utils.py

1184 - vllm/v1/attention/backends/flashinfer.py

1202 - - vllm/v1/attention/backends/mla/cutlass_mla.py

1203 - - vllm/v1/attention/backends/mla/flashinfer_mla.py

1204 - - vllm/v1/attention/selector.py

1205 - - vllm/platforms/cuda.py

1185 + - vllm/v1/worker/

1186 + - vllm/v1/cudagraph_dispatcher.py

1187 + - vllm/compilation/

1188 + # can affect pattern matching

1189 + - vllm/model_executor/layers/layernorm.py

1190 + - vllm/model_executor/layers/activation.py

1191 + - vllm/model_executor/layers/quantization/input_quant_fp8.py

1192 + - tests/compile/passes/test_fusion_attn.py

1193 + - tests/compile/passes/test_silu_mul_quant_fusion.py

1194 + - tests/compile/passes/distributed/test_fusion_all_reduce.py

1195 + - tests/compile/fullgraph/test_full_graph.py

1196 commands:

1197 - nvidia-smi

1208 - - python3 examples/offline_inference/basic/chat.py

1209 - # Attention

1210 - # num_heads2 broken by https://github.com/flashinfer-ai/flashinfer/issues/1353

1211 - - pytest -v -s tests/kernels/attention/test_attention_selector.py

1212 - - pytest -v -s tests/kernels/attention/test_flashinfer.py -k 'not num_heads2'

1213 - - pytest -v -s tests/kernels/attention/test_flashinfer_trtllm_attention.py

1214 - - pytest -v -s tests/kernels/attention/test_cutlass_mla_decode.py

1215 - - pytest -v -s tests/kernels/attention/test_flashinfer_mla_decode.py

1216 - # Quantization

1217 - - pytest -v -s tests/kernels/quantization/test_cutlass_scaled_mm.py -k 'fp8'

1218 - - pytest -v -s tests/kernels/quantization/test_nvfp4_quant.py

1219 - - pytest -v -s tests/kernels/quantization/test_silu_mul_nvfp4_quant.py

1220 - - pytest -v -s tests/kernels/quantization/test_nvfp4_scaled_mm.py

1221 - - pytest -v -s tests/kernels/quantization/test_flashinfer_scaled_mm.py

1222 - - pytest -v -s tests/kernels/quantization/test_flashinfer_nvfp4_scaled_mm.py

1223 - - pytest -v -s tests/kernels/quantization/test_nvfp4_qutlass.py

1224 - - pytest -v -s tests/kernels/quantization/test_mxfp4_qutlass.py

1225 - - pytest -v -s tests/kernels/moe/test_nvfp4_moe.py

1226 - - pytest -v -s tests/kernels/moe/test_ocp_mx_moe.py

1227 - - pytest -v -s tests/kernels/moe/test_flashinfer.py

1228 - - pytest -v -s tests/kernels/moe/test_cutedsl_moe.py

1198 + - pytest -v -s tests/compile/passes/test_fusion_attn.py

1199 + - pytest -v -s tests/compile/passes/test_silu_mul_quant_fusion.py

1200 + # this runner has 2 GPUs available even though num_gpus=2 is not set

1201 + - pytest -v -s tests/compile/passes/distributed/test_fusion_all_reduce.py

1202 +

1203 + # # Limit to Inductor partition, no custom ops, and allreduce & attn fusion to reduce running time

1204 + # # Wrap with quotes to escape yaml

1205 + # - "pytest -v -s tests/compile/distributed/test_fusions_e2e.py::test_tp2_attn_quant_allreduce_rmsnorm -k 'True and not +quant_fp8 and not +rms_norm'"

1206 + # Old E2E tests were removed in https://github.com/vllm-project/vllm/pull/33293

1207 + # in favor of new tests in fusions_e2e. We avoid replicating the new jobs in this file as it's deprecated.

1208 +

1209 + # test_fp8_kv_scale_compile requires FlashAttention (not supported on default L4/L40)

1210 + - pytest -v -s tests/compile/fullgraph/test_full_graph.py::test_fp8_kv_scale_compile

1211

1212 - label: Blackwell GPT-OSS Eval

1213 timeout_in_minutes: 60

@@ -1258,16 +1240,6 @@ steps:

1240 commands:

1241 - pytest -s -v tests/quantization/test_blackwell_moe.py

1242

1261 -- label: Blackwell LM Eval Small Models

1262 - timeout_in_minutes: 120

1263 - gpu: b200

1264 - optional: true # run on nightlies

1265 - source_file_dependencies:

1266 - - csrc/

1267 - - vllm/model_executor/layers/quantization

1268 - commands:

1269 - - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt

1270 -

1243 ##### 1 GPU test #####

1244 ##### multi gpus test #####

1245

@@ -1681,16 +1653,6 @@ steps:

1653 commands:

1654 - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020

1655

1684 -- label: Qwen3-30B-A3B-FP8-block Accuracy (B200)

1685 - timeout_in_minutes: 60

1686 - gpu: b200

1687 - optional: true

1688 - num_gpus: 2

1689 - working_dir: "/vllm-workspace"

1690 - commands:

1691 - - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1

1692 -

1693 -

1656 - label: Qwen3-Next-80B-A3B-Instruct MTP Async EPLB Accuracy

1657 timeout_in_minutes: 60

1658 mirror_hardwares: [amdexperimental, amdproduction]

@@ -2176,19 +2138,6 @@ steps:

2138

2139 # TODO: Add the "V1 Test attention (MI300)" test group

2140

2179 -- label: V1 Test attention (H100) # 10min

2180 - mirror_hardwares: [amdexperimental]

2181 - agent_pool: mi355_1

2182 - timeout_in_minutes: 30

2183 - gpu: h100

2184 - source_file_dependencies:

2185 - - vllm/config/attention.py

2186 - - vllm/model_executor/layers/attention

2187 - - vllm/v1/attention

2188 - - tests/v1/attention

2189 - commands:

2190 - - pytest -v -s v1/attention

2191 -

2141 - label: Batch Invariance Tests (H100) # 10min

2142 mirror_hardwares: [amdexperimental]

2143 agent_pool: mi355_1

@@ -2205,6 +2154,8 @@ steps:

2154 - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py

2155

2156 - label: V1 Test attention (B200) # 10min

2157 + mirror_hardwares: [amdexperimental, amdmi355]

2158 + agent_pool: mi355_1

2159 timeout_in_minutes: 30

2160 gpu: b200

2161 source_file_dependencies:

@@ -2829,7 +2780,9 @@ steps:

2780 # Whisper needs spawn method to avoid deadlock

2781 - VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/offline_inference/audio_language.py --model-type whisper

2782

2832 -- label: Blackwell Test # 21 min

2783 +- label: Blackwell Test (MI355) # 21 min

2784 + mirror_hardwares: [amdexperimental, amdmi355]

2785 + agent_pool: mi355_1

2786 timeout_in_minutes: 30

2787 working_dir: "/vllm-workspace/"

2788 gpu: b200

@@ -2848,28 +2801,28 @@ steps:

2801 - vllm/v1/attention/selector.py

2802 - vllm/platforms/cuda.py

2803 commands:

2851 - - nvidia-smi

2804 + rocm-smi

2805 - python3 examples/offline_inference/basic/chat.py

2806 # Attention

2807 # num_heads2 broken by https://github.com/flashinfer-ai/flashinfer/issues/1353

2855 - - pytest -v -s tests/kernels/attention/test_attention_selector.py

2856 - - pytest -v -s tests/kernels/attention/test_flashinfer.py -k 'not num_heads2'

2857 - - pytest -v -s tests/kernels/attention/test_flashinfer_trtllm_attention.py

2858 - - pytest -v -s tests/kernels/attention/test_cutlass_mla_decode.py

2859 - - pytest -v -s tests/kernels/attention/test_flashinfer_mla_decode.py

2860 - # Quantization

2861 - - pytest -v -s tests/kernels/quantization/test_cutlass_scaled_mm.py -k 'fp8'

2862 - - pytest -v -s tests/kernels/quantization/test_nvfp4_quant.py

2863 - - pytest -v -s tests/kernels/quantization/test_silu_mul_nvfp4_quant.py

2864 - - pytest -v -s tests/kernels/quantization/test_nvfp4_scaled_mm.py

2865 - - pytest -v -s tests/kernels/quantization/test_flashinfer_scaled_mm.py

2866 - - pytest -v -s tests/kernels/quantization/test_flashinfer_nvfp4_scaled_mm.py

2867 - - pytest -v -s tests/kernels/quantization/test_nvfp4_qutlass.py

2868 - - pytest -v -s tests/kernels/quantization/test_mxfp4_qutlass.py

2869 - - pytest -v -s tests/kernels/moe/test_nvfp4_moe.py

2870 - - pytest -v -s tests/kernels/moe/test_ocp_mx_moe.py

2871 - - pytest -v -s tests/kernels/moe/test_flashinfer.py

2872 - - pytest -v -s tests/kernels/moe/test_cutedsl_moe.py

2808 + - pytest -v -s tests/kernels/attention/test_attention_selector.py

2809 + #- pytest -v -s tests/kernels/attention/test_flashinfer.py -k 'not num_heads2'

2810 + #- pytest -v -s tests/kernels/attention/test_flashinfer_trtllm_attention.py

2811 + #- pytest -v -s tests/kernels/attention/test_cutlass_mla_decode.py

2812 + #- pytest -v -s tests/kernels/attention/test_flashinfer_mla_decode.py

2813 + ## Quantization

2814 + #- pytest -v -s tests/kernels/quantization/test_cutlass_scaled_mm.py -k 'fp8'

2815 + #- pytest -v -s tests/kernels/quantization/test_nvfp4_quant.py

2816 + #- pytest -v -s tests/kernels/quantization/test_silu_mul_nvfp4_quant.py

2817 + #- pytest -v -s tests/kernels/quantization/test_nvfp4_scaled_mm.py

2818 + #- pytest -v -s tests/kernels/quantization/test_flashinfer_scaled_mm.py

2819 + #- pytest -v -s tests/kernels/quantization/test_flashinfer_nvfp4_scaled_mm.py

2820 + #- pytest -v -s tests/kernels/quantization/test_nvfp4_qutlass.py

2821 + #- pytest -v -s tests/kernels/quantization/test_mxfp4_qutlass.py

2822 + #- pytest -v -s tests/kernels/moe/test_nvfp4_moe.py

2823 + #- pytest -v -s tests/kernels/moe/test_ocp_mx_moe.py

2824 + #- pytest -v -s tests/kernels/moe/test_flashinfer.py

2825 + #- pytest -v -s tests/kernels/moe/test_cutedsl_moe.py

2826

2827 - label: Blackwell Fusion and Compile Tests # 30 min

2828 timeout_in_minutes: 40

@@ -2939,13 +2892,15 @@ steps:

2892

2893 - label: Blackwell LM Eval Small Models

2894 timeout_in_minutes: 120

2895 + mirror_hardwares: [amdexperimental, amdproduction, amdmi355]

2896 + agent_pool: mi355_2

2897 gpu: b200

2898 optional: true # run on nightlies

2899 source_file_dependencies:

2900 - csrc/

2901 - vllm/model_executor/layers/quantization

2902 commands:

2948 - - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt

2903 + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355.txt

2904

2905 ##### 1 GPU test #####

2906 ##### multi gpus test #####

@@ -3328,18 +3283,9 @@ steps:

3283 commands:

3284 - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010

3285

3331 -- label: Qwen3-30B-A3B-FP8-block Accuracy (H100)

3332 - mirror_hardwares: [amdexperimental, amdproduction]

3333 - agent_pool: mi355_4

3334 - timeout_in_minutes: 60

3335 - gpu: h100

3336 - optional: true

3337 - num_gpus: 4

3338 - working_dir: "/vllm-workspace"

3339 - commands:

3340 - - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020

3341 -

3342 -- label: Qwen3-30B-A3B-FP8-block Accuracy (B200)

3286 +- label: Qwen3-30B-A3B-FP8-block Accuracy (B200/MI355)

3287 + mirror_hardwares: [amdexperimental, amdproduction, amdmi355]

3288 + agent_pool: mi355_2

3289 timeout_in_minutes: 60

3290 gpu: b200

3291 optional: true

@@ -3358,3 +3304,18 @@ steps:

3304 working_dir: "/vllm-workspace"

3305 commands:

3306 - bash .buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh 0.8 1319 8040

3307 +

3308 +- label: Attention Benchmarks Smoke Test (B200/MI355)

3309 + device: b200

3310 + mirror_hardwares: [amdexperimental, amdmi355]

3311 + agent_pool: mi355_2

3312 + num_gpus: 2

3313 + optional: true

3314 + working_dir: "/vllm-workspace/"

3315 + timeout_in_minutes: 10

3316 + source_file_dependencies:

3317 + - benchmarks/attention_benchmarks/

3318 + - vllm/v1/attention/

3319 + commands:

3320 + - python3 benchmarks/attention_benchmarks/benchmark.py --backends ROCM_ATTN ROCM_AITER_FA ROCM_AITER_UNIFIED_ATTN --batch-specs "8q1s1k" --repeats 1 --warmup-iters 1

3321 +

tests/evals/gsm8k/configs/Qwen3-Next-FP8-EP2_MI355.yaml new

@@ -0,0 +1,9 @@

1 +model_name: "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8"

2 +accuracy_threshold: 0.85

3 +num_questions: 1319

4 +num_fewshot: 5

5 +server_args: >-

6 + --max-model-len 4096

7 + --tensor-parallel-size 2

8 + --enable-expert-parallel

9 + --async-scheduling

tests/evals/gsm8k/configs/models-mi355.txt new

@@ -0,0 +1,5 @@

1 +Qwen3-0.6B-FP8.yaml

2 +Qwen2.5-VL-3B-Instruct-FP8-dynamic.yaml

3 +Qwen1.5-MoE-W4A16-CT.yaml

4 +DeepSeek-V2-Lite-Instruct-FP8.yaml

5 +Qwen3-Next-FP8-EP2_MI355.yaml

tests/kernels/attention/test_attention_selector.py

@@ -343,6 +343,10 @@ def test_auto_backend_selection_behavior():

343 ("FLEX_ATTENTION", None, False), # Flex does not support

344 ],

345 )

346 +@pytest.mark.skipif(

347 + current_platform.is_rocm(),

348 + reason="Attention backend FA3 is not supported on ROCm. This test can't succeed.",

349 +)

350 def test_per_head_quant_scales_backend_selection(

351 backend_name: str, flash_attn_version: int | None, should_succeed: bool

352 ):