@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 1b922ffcbe

[ROCm][CI] Add quantized MoE serving test for gfx950 (#58748)

Signed-off-by: Stefan Koncarevic <Stefan.Koncarevic@amd.com> Co-authored-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>

stefankoncarevic committed Sep 25, 2026 at 20:02 UTC 1b922ffcbe75cd411fab98a773f47b8f0294e43b
3 files changed +206 -4
.buildkite/test-amd.yaml
+32 -2
@@ -2579,7 +2579,7 @@ steps:
2579 - unset VLLM_USE_V2_MODEL_RUNNER
2580 - uv pip install --system torchao==0.18.0
2581 - uv pip install --system conch-triton-kernels
2582 - - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
2582 + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --ignore quantization/test_rocm_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
2583 - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi
2584
2585 #--------------------------------------------------------- mi300 · samplers ----------------------------------------------------------#
@@ -4943,7 +4943,7 @@ steps:
4943 - unset VLLM_USE_V2_MODEL_RUNNER
4944 - uv pip install --system torchao==0.17.0
4945 - uv pip install --system conch-triton-kernels
4946 - - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
4946 + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --ignore quantization/test_rocm_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
4947 - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi
4948
4949 # - label: Quantized MoE Test (B200-MI355) # TBD
@@ -5445,6 +5445,36 @@ steps:
5445 - pip install 'modelscope<1.38'
5446 - pytest -v -s test_regression.py
5447
5448 +- label: ':amd: (MI355) Quantized MoE'
5449 + timeout_in_minutes: 45
5450 + working_dir: /vllm-workspace/
5451 + source_file_dependencies:
5452 + - tests/quantization/test_rocm_moe.py
5453 + - tests/utils.py
5454 + - vllm/model_executor/models/deepseek_v2.py
5455 + - vllm/model_executor/models/qwen3_moe.py
5456 + - vllm/model_executor/models/gpt_oss.py
5457 + - vllm/model_executor/model_loader/
5458 + - vllm/model_executor/layers/fused_moe/
5459 + - vllm/model_executor/layers/quantization/
5460 + - vllm/model_executor/kernels/linear/
5461 + - vllm/v1/attention/
5462 + - vllm/v1/worker/gpu/
5463 + - vllm/_aiter_ops.py
5464 + - vllm/platforms/rocm.py
5465 + - vllm/envs.py
5466 + commands:
5467 + - pytest -v -s tests/quantization/test_rocm_moe.py
5468 + dind: false
5469 + agent_pool: mi355_1
5470 + num_gpus: 1
5471 + mirror_hardwares:
5472 + - amdexperimental
5473 + - amdproduction
5474 + - amdgfx950nightly
5475 + - amdmi355
5476 + optional: true
5477 +
5478 - label: ':amd: (MI355) Fusion E2E TP2 AsyncTP BF16 Config Sweep'
5479 timeout_in_minutes: 30
5480 working_dir: /vllm-workspace/
.buildkite/test_areas/quantization.yaml
+28 -2
@@ -19,7 +19,7 @@ steps:
19 - uv pip install --system conch-triton-kernels
20 # The SM90-only checkpoint currently contains a removed weight_chan_scale
21 # parameter. It was not exercised by the previous L4 job.
22 - - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py -k 'not test_compressed_tensors_w4a8_fp8' --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
22 + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --ignore quantization/test_rocm_moe.py -k 'not test_compressed_tensors_w4a8_fp8' --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
23 mirror:
24 amd:
25 label: ":amd: (MI355) Quantization Shard %N"
@@ -43,7 +43,7 @@ steps:
43 - unset VLLM_USE_V2_MODEL_RUNNER
44 - uv pip install --system torchao==0.17.0
45 - uv pip install --system conch-triton-kernels
46 - - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
46 + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --ignore quantization/test_rocm_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
47 - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi
48
49 - label: ":nvidia: (H200 MIG 35GB) Quantized Fusions"
@@ -76,6 +76,32 @@ steps:
76 - vllm/v1/attention/backends/flashinfer.py
77 commands:
78 - pytest -s -v tests/quantization/test_blackwell_moe.py
79 + mirror:
80 + amd:
81 + label: ":amd: (MI355) Quantized MoE"
82 + dind: false
83 + device: mi355_1
84 + timeout_in_minutes: 45
85 + working_dir: "/vllm-workspace/"
86 + depends_on:
87 + - image-build-amd
88 + source_file_dependencies:
89 + - tests/quantization/test_rocm_moe.py
90 + - tests/utils.py
91 + - vllm/model_executor/models/deepseek_v2.py
92 + - vllm/model_executor/models/qwen3_moe.py
93 + - vllm/model_executor/models/gpt_oss.py
94 + - vllm/model_executor/model_loader/
95 + - vllm/model_executor/layers/fused_moe/
96 + - vllm/model_executor/layers/quantization/
97 + - vllm/model_executor/kernels/linear/
98 + - vllm/v1/attention/
99 + - vllm/v1/worker/gpu/
100 + - vllm/_aiter_ops.py
101 + - vllm/platforms/rocm.py
102 + - vllm/envs.py
103 + commands:
104 + - pytest -v -s tests/quantization/test_rocm_moe.py
105
106 - label: ":nvidia: (H200 MIG 35GB) Quantized Models"
107 device: h200_35gb
tests/quantization/test_rocm_moe.py new
+146
@@ -0,0 +1,146 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""ROCm equivalents of the quantized-MoE model startup/generation contract.
4 +
5 +Like test_blackwell_moe.py, these tests use reduced models and dummy weights.
6 +They validate model/backend integration, not trained-model accuracy.
7 +"""
8 +
9 +import math
10 +
11 +import pytest
12 +
13 +from tests.utils import RemoteOpenAIServer
14 +from vllm.platforms import current_platform
15 +from vllm.platforms.rocm import on_gfx950
16 +
17 +pytestmark = pytest.mark.skipif(
18 + not current_platform.is_rocm() or not on_gfx950(),
19 + reason="ROCm FP8/MXFP4 model integration requires gfx950",
20 +)
21 +
22 +COMMON_OVERRIDES = {
23 + "num_hidden_layers": 4,
24 + "hidden_size": 512,
25 + "intermediate_size": 1024,
26 + "num_attention_heads": 8,
27 + "num_key_value_heads": 2,
28 +}
29 +
30 +CASES = [
31 + pytest.param(
32 + "RedHatAI/DeepSeek-Coder-V2-Lite-Instruct-FP8",
33 + "efe1ced428db63e7ccbcf367334596f77e9af140",
34 + "triton",
35 + "TRITON_MLA",
36 + {
37 + "n_routed_experts": 8,
38 + "n_shared_experts": 1,
39 + "num_experts_per_tok": 2,
40 + "moe_intermediate_size": 512,
41 + },
42 + ["--code-revision", "e434a23f91ba5b4923cf6c9d9a238eb4a08e3a11"],
43 + id="deepseek-fp8-per-tensor-triton",
44 + ),
45 + pytest.param(
46 + "Qwen/Qwen3-30B-A3B-FP8",
47 + "d206ba732169f29bb77fbf80fc2c4b81d4d30782",
48 + "triton",
49 + "TRITON_ATTN",
50 + {
51 + "num_experts": 8,
52 + "num_experts_per_tok": 2,
53 + "moe_intermediate_size": 512,
54 + "head_dim": 64,
55 + },
56 + [],
57 + id="qwen3-fp8-block-triton",
58 + ),
59 + pytest.param(
60 + "openai/gpt-oss-20b",
61 + "6cee5e81ee83917806bbde320786a8fb61efebee",
62 + "aiter",
63 + "TRITON_ATTN",
64 + {
65 + "num_local_experts": 8,
66 + "num_experts_per_tok": 2,
67 + "experts_per_token": 2,
68 + "layer_types": ["sliding_attention", "full_attention"] * 2,
69 + },
70 + [],
71 + id="gptoss-mxfp4-aiter",
72 + ),
73 +]
74 +
75 +
76 +@pytest.mark.parametrize("enforce_eager", [True, False], ids=["eager", "compiled"])
77 +@pytest.mark.parametrize(
78 + "model,revision,moe_backend,attention_backend,overrides,extra_args", CASES
79 +)
80 +def test_rocm_quantized_moe_generation(
81 + model,
82 + revision,
83 + moe_backend,
84 + attention_backend,
85 + overrides,
86 + extra_args,
87 + enforce_eager,
88 +):
89 + """Quantized MoE models warm up and return finite logits through serving."""
90 + args = [
91 + "--revision",
92 + revision,
93 + "--load-format",
94 + "dummy",
95 + "--trust-remote-code",
96 + "--dtype",
97 + "bfloat16",
98 + "--max-model-len",
99 + "2048",
100 + "--max-num-batched-tokens",
101 + "256",
102 + "--max-num-seqs",
103 + "8",
104 + "--gpu-memory-utilization",
105 + "0.02",
106 + "--kv-cache-memory-bytes",
107 + str(256 * 1024 * 1024),
108 + "--moe-backend",
109 + moe_backend,
110 + "--attention-backend",
111 + attention_backend,
112 + *extra_args,
113 + ]
114 + if enforce_eager:
115 + args.append("--enforce-eager")
116 +
117 + with RemoteOpenAIServer(
118 + model,
119 + args,
120 + env_dict={"VLLM_ROCM_USE_AITER": "1", "VLLM_USE_V2_MODEL_RUNNER": "1"},
121 + override_hf_configs={**COMMON_OVERRIDES, **overrides},
122 + max_wait_seconds=1200,
123 + ) as server:
124 + prompts = [[1, 2, 3], [1, 2, 3, 4] * 4 + [5]]
125 + completion = server.get_client().completions.create(
126 + model=model,
127 + prompt=prompts,
128 + temperature=0,
129 + max_tokens=2,
130 + logprobs=1,
131 + extra_body={"ignore_eos": True, "skip_special_tokens": False},
132 + )
133 + assert len(completion.choices) == len(prompts)
134 + assert completion.usage is not None
135 + assert completion.usage.prompt_tokens == sum(map(len, prompts))
136 + assert completion.usage.completion_tokens == 2 * len(prompts)
137 + for choice in completion.choices:
138 + assert choice.finish_reason == "length"
139 + assert choice.text
140 + assert choice.logprobs is not None
141 + assert choice.logprobs.token_logprobs is not None
142 + assert len(choice.logprobs.token_logprobs) == 2
143 + assert all(
144 + p is not None and math.isfinite(p)
145 + for p in choice.logprobs.token_logprobs
146 + )