Support DeepSeek-V4 AMD Quark NVFP4 with emulation kernel (#47972)
Signed-off-by: Jimmy Adams <Adam_ji@outlook.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: huji <huidong.ji@amd.com> Signed-off-by: jimmy-adams <41593649+jimmy-adams@users.noreply.github.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: huji <huidong.ji@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
jimmy-adams committed
Aug 7, 2026 at 16:19 UTC
da788334bc0683cc44a58b4624e3f5a3c09a09e0
11 files changed
+332
-17
.buildkite/test-amd.yaml
+23
@@ -3341,6 +3341,29 @@ steps:
3341
- export VLLM_USE_DEEP_GEMM=0
3342
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-fp8.txt --tp-size=4
3343
3344
+- label: LM Eval Large Models (8xB200-8xMI355) # TBD
3345
+ timeout_in_minutes: 180
3346
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3347
+ dind: false
3348
+ agent_pool: mi355_8
3349
+ optional: true
3350
+ num_gpus: 8
3351
+ working_dir: "/vllm-workspace/tests"
3352
+ source_file_dependencies:
3353
+ - vllm/model_executor/models/
3354
+ - vllm/model_executor/model_loader/
3355
+ - vllm/model_executor/layers/quantization/
3356
+ - vllm/v1/attention/backends/
3357
+ - vllm/v1/attention/selector.py
3358
+ - vllm/model_executor/layers/layernorm.py
3359
+ - csrc/
3360
+ - vllm/_aiter_ops.py
3361
+ - vllm/platforms/rocm.py
3362
+ - tests/evals/
3363
+ commands:
3364
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
3365
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-gfx950-large.txt
3366
+
3367
#--------------------------------------------------------- mi355 · examples ----------------------------------------------------------#
3368
3369
- label: Examples # TBD
tests/evals/gsm8k/configs/DeepSeek-V4-Flash-NVFP4.yaml
new
+15
@@ -0,0 +1,15 @@
1
+model_name: "amd/DeepSeek-V4-Flash-NVFP4"
2
+accuracy_threshold: 0.92
3
+num_questions: 1319
4
+num_fewshot: 8
5
+startup_max_wait_seconds: 1800
6
+server_args: >-
7
+ --tensor-parallel-size 8
8
+ --max-model-len 4096
9
+ --trust-remote-code
10
+ --kv-cache-dtype fp8
11
+ --tokenizer-mode deepseek_v4
12
+ --compilation-config '{"mode": 3, "cudagraph_mode": "FULL_DECODE_ONLY"}'
13
+env:
14
+ VLLM_ROCM_USE_AITER: "1"
15
+ VLLM_ROCM_USE_AITER_MOE: "1"
tests/evals/gsm8k/configs/DeepSeek-V4-Pro-NVFP4.yaml
new
+15
@@ -0,0 +1,15 @@
1
+model_name: "amd/DeepSeek-V4-Pro-NVFP4"
2
+accuracy_threshold: 0.92
3
+num_questions: 1319
4
+num_fewshot: 8
5
+startup_max_wait_seconds: 3600
6
+server_args: >-
7
+ --tensor-parallel-size 8
8
+ --max-model-len 4096
9
+ --trust-remote-code
10
+ --kv-cache-dtype fp8
11
+ --tokenizer-mode deepseek_v4
12
+ --compilation-config '{"mode": 3, "cudagraph_mode": "FULL_DECODE_ONLY"}'
13
+env:
14
+ VLLM_ROCM_USE_AITER: "1"
15
+ VLLM_ROCM_USE_AITER_MOE: "1"
tests/evals/gsm8k/configs/models-gfx950-large.txt
new
+8
@@ -0,0 +1,8 @@
1
+# GFX950 model configurations for GSM8K evaluation
2
+# TP8 models only
3
+# DeepSeek-V4-Pro-NVFP4.yaml is temporarily disabled: repeated CI failures due to
4
+# cold NFS reads of the 850GiB checkpoint exceeding startup_max_wait_seconds
5
+# (see https://github.com/vllm-project/vllm/pull/47972). Re-enable once the
6
+# startup timeout fix has landed and is verified green.
7
+# DeepSeek-V4-Pro-NVFP4.yaml
8
+DeepSeek-V4-Flash-NVFP4.yaml
tests/quantization/test_quark.py
+97
@@ -19,8 +19,10 @@ from packaging import version
19
20
from vllm._aiter_ops import is_aiter_found_and_supported
21
from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501
22
+ QuarkConfig,
23
QuarkLinearMethod,
24
QuarkW8A8Fp8,
25
+ QuarkW8A8Fp8PerBlock,
26
QuarkW8A8Int8,
27
)
28
from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E501
@@ -80,6 +82,101 @@ def enable_pickle(monkeypatch):
82
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
83
84
85
+def test_quark_config_has_no_model_specific_fused_mappings():
86
+ config = QuarkConfig({})
87
+
88
+ assert "gate_up_proj" not in config.packed_modules_mapping
89
+ assert "fused_wqa_wkv" not in config.packed_modules_mapping
90
+
91
+
92
+def test_quark_config_preserves_existing_packed_modules_mapping():
93
+ class CustomQuarkConfig(QuarkConfig):
94
+ packed_modules_mapping = {"custom_proj": ["a", "b"]}
95
+
96
+ config = CustomQuarkConfig({})
97
+
98
+ assert config.packed_modules_mapping["custom_proj"] == ["a", "b"]
99
+
100
+
101
+def test_quark_fp8_w8a8_detects_per_block_config():
102
+ config = QuarkConfig({})
103
+ weight_config = {
104
+ "dtype": "fp8_e4m3",
105
+ "qscheme": "per_block",
106
+ "is_dynamic": False,
107
+ "block_size": [128, 128],
108
+ "symmetric": True,
109
+ }
110
+ input_config = {
111
+ "dtype": "fp8_e4m3",
112
+ "qscheme": "per_group",
113
+ "is_dynamic": True,
114
+ "group_size": 128,
115
+ "symmetric": True,
116
+ }
117
+
118
+ assert config._is_fp8_w8a8(weight_config, input_config)
119
+
120
+
121
+def test_quark_fp8_w8a8_rejects_per_block_static_input():
122
+ config = QuarkConfig({})
123
+ weight_config = {
124
+ "dtype": "fp8_e4m3",
125
+ "qscheme": "per_block",
126
+ "is_dynamic": False,
127
+ "block_size": [128, 128],
128
+ "symmetric": True,
129
+ }
130
+ input_config = {
131
+ "dtype": "fp8_e4m3",
132
+ "qscheme": "per_group",
133
+ "is_dynamic": False,
134
+ "group_size": 128,
135
+ "symmetric": True,
136
+ }
137
+
138
+ assert not config._is_fp8_w8a8(weight_config, input_config)
139
+
140
+
141
+def test_quark_fp8_w8a8_rejects_per_block_group_size_mismatch():
142
+ config = QuarkConfig({})
143
+ weight_config = {
144
+ "dtype": "fp8_e4m3",
145
+ "qscheme": "per_block",
146
+ "is_dynamic": False,
147
+ "block_size": [128, 128],
148
+ "symmetric": True,
149
+ }
150
+ input_config = {
151
+ "dtype": "fp8_e4m3",
152
+ "qscheme": "per_group",
153
+ "is_dynamic": True,
154
+ "group_size": 64,
155
+ "symmetric": True,
156
+ }
157
+
158
+ assert not config._is_fp8_w8a8(weight_config, input_config)
159
+
160
+
161
+def test_quark_w8a8_fp8_per_block_requires_block_size():
162
+ weight_config = {
163
+ "dtype": "fp8_e4m3",
164
+ "qscheme": "per_block",
165
+ "is_dynamic": False,
166
+ "symmetric": True,
167
+ }
168
+ input_config = {
169
+ "dtype": "fp8_e4m3",
170
+ "qscheme": "per_group",
171
+ "is_dynamic": True,
172
+ "group_size": 128,
173
+ "symmetric": True,
174
+ }
175
+
176
+ with pytest.raises(ValueError, match="requires `block_size`"):
177
+ QuarkW8A8Fp8PerBlock(weight_config, input_config)
178
+
179
+
180
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
181
@pytest.mark.parametrize("tp", [1])
182
def test_quark_fp8_w_per_tensor_a_per_tensor(vllm_runner, kv_cache_dtype, tp):
vllm/model_executor/layers/fused_moe/oracle/nvfp4.py
+1
@@ -193,6 +193,7 @@ def select_nvfp4_moe_backend(
193
NvFp4MoeBackend.FLASHINFER_CUTEDSL,
194
NvFp4MoeBackend.VLLM_CUTLASS,
195
NvFp4MoeBackend.MARLIN,
196
+ NvFp4MoeBackend.EMULATION,
197
NvFp4MoeBackend.HUMMING,
198
}
199
vllm/model_executor/layers/quantization/quark/quark.py
+32
-2
@@ -33,6 +33,7 @@ from vllm.model_executor.layers.quantization.quark.schemes import (
33
QuarkScheme,
34
QuarkW4A8_MXFP4_FP8,
35
QuarkW8A8Fp8,
36
+ QuarkW8A8Fp8PerBlock,
37
QuarkW8A8Int8,
38
)
39
from vllm.model_executor.layers.quantization.quark.utils import (
@@ -66,6 +67,14 @@ class QuarkConfig(QuantizationConfig):
67
if kv_cache_group is None:
68
kv_cache_group = []
69
self.quant_config = quant_config
70
+ # Copy the class-level default (which a subclass may override, e.g. the
71
+ # DeepSeek-V4 Quark config) so per-instance edits don't mutate the class.
72
+ # Read from the class rather than ``self`` because the base
73
+ # ``QuantizationConfig.__init__`` sets an empty instance-level
74
+ # ``packed_modules_mapping`` that would otherwise shadow the override.
75
+ self.packed_modules_mapping = dict(
76
+ getattr(type(self), "packed_modules_mapping", {})
77
+ )
78
self.kv_cache_group = kv_cache_group
79
self.kv_cache_config = kv_cache_config
80
self.pack_method = pack_method
@@ -341,11 +350,30 @@ class QuarkConfig(QuantizationConfig):
350
"per_tensor",
351
"per_channel",
352
]
353
+ block_size = list(weight_quant.get("block_size") or [])
354
+ is_per_block_weight = (
355
+ weight_quant.get("qscheme") == "per_block"
356
+ and block_size == [128, 128]
357
+ and weight_quant.get("symmetric") is True
358
+ )
359
345
- if not (is_fp8_dtype and is_static_weight and is_per_tensor_or_channel_weight):
360
+ if not is_fp8_dtype or not is_static_weight:
361
return False
362
348
- # Dynamic quantization is always supported if weights supported.
363
+ if is_per_block_weight:
364
+ if not input_quant.get("is_dynamic"):
365
+ return False
366
+ return (
367
+ input_quant.get("qscheme") == "per_group"
368
+ and input_quant.get("group_size") == block_size[1]
369
+ and input_quant.get("symmetric") is True
370
+ )
371
+
372
+ if not is_per_tensor_or_channel_weight:
373
+ return False
374
+
375
+ # Dynamic quantization is always supported if tensor/channel weights
376
+ # are supported.
377
if input_quant.get("is_dynamic"):
378
return True
379
@@ -640,6 +668,8 @@ class QuarkConfig(QuantizationConfig):
668
QuarkW8A8Fp8.get_min_capability(), error=False
669
)
670
if is_fp8_w8a8_supported:
671
+ if weight_config.get("qscheme") == "per_block":
672
+ return QuarkW8A8Fp8PerBlock(weight_config, input_config)
673
return QuarkW8A8Fp8(weight_config, input_config)
674
elif self._is_static_tensor_w8a8(weight_config, input_config):
675
weight_qscheme = cast(str, weight_config.get("qscheme"))
vllm/model_executor/layers/quantization/quark/quark_moe.py
+7
-6
@@ -1590,15 +1590,16 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
1590
Convert NVFP4 MoE weights into kernel format and setup the kernel.
1591
"""
1592
1593
- if not torch.allclose(
1593
+ # Match existing NVFP4 MoE paths: fused w13 uses the w1 global scale.
1594
+ if self.moe.is_act_and_mul and not torch.allclose(
1595
layer.w13_weight_scale_2[:, 0], layer.w13_weight_scale_2[:, 1]
1596
):
1596
- raise ValueError("Different global scales for w1 and w3 is not supported.")
1597
+ logger.warning_once(
1598
+ "w1_weight_scale_2 must match w3_weight_scale_2. "
1599
+ "Accuracy may be affected."
1600
+ )
1601
1598
- # Use a single gscale for w13
1599
- w13_weight_scale_2 = torch.maximum(
1600
- layer.w13_weight_scale_2[:, 0], layer.w13_weight_scale_2[:, 1]
1601
- ).contiguous()
1602
+ w13_weight_scale_2 = layer.w13_weight_scale_2[:, 0].contiguous()
1603
1604
w2_weight_scale_2 = layer.w2_weight_scale_2
1605
vllm/model_executor/layers/quantization/quark/schemes/__init__.py
+2
-1
@@ -5,12 +5,13 @@ from .quark_nvfp4 import QuarkNVFP4
5
from .quark_ocp_mx import QuarkOCP_MX
6
from .quark_scheme import QuarkScheme
7
from .quark_w4a8_mxfp4_fp8 import QuarkW4A8_MXFP4_FP8
8
-from .quark_w8a8_fp8 import QuarkW8A8Fp8
8
+from .quark_w8a8_fp8 import QuarkW8A8Fp8, QuarkW8A8Fp8PerBlock
9
from .quark_w8a8_int8 import QuarkW8A8Int8
10
11
__all__ = [
12
"QuarkScheme",
13
"QuarkW8A8Fp8",
14
+ "QuarkW8A8Fp8PerBlock",
15
"QuarkW8A8Int8",
16
"QuarkOCP_MX",
17
"QuarkW4A8_MXFP4_FP8",
vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py
+115
-4
@@ -9,13 +9,18 @@ from torch.nn import Parameter
9
10
from vllm.config import get_current_vllm_config
11
from vllm.logger import init_logger
12
-from vllm.model_executor.kernels.linear import (
13
- init_fp8_linear_kernel,
14
-)
12
+from vllm.model_executor.kernels.linear import init_fp8_linear_kernel
13
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
14
+from vllm.model_executor.layers.quantization.utils.fp8_utils import (
15
+ create_fp8_scale_parameter,
16
+ create_fp8_weight_parameter,
17
+ validate_fp8_block_shape,
18
+)
19
from vllm.model_executor.layers.quantization.utils.quant_utils import (
20
GroupShape,
21
+ kFp8Dynamic128Sym,
22
kFp8DynamicTokenSym,
23
+ kFp8Static128BlockSym,
24
kFp8StaticChannelSym,
25
kFp8StaticTensorSym,
26
)
@@ -24,13 +29,14 @@ from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
29
requantize_with_max_scale,
30
)
31
from vllm.model_executor.parameter import (
32
+ BlockQuantScaleParameter,
33
ChannelQuantScaleParameter,
34
ModelWeightParameter,
35
PerTensorScaleParameter,
36
)
37
from vllm.platforms import current_platform
38
33
-__all__ = ["QuarkW8A8Fp8"]
39
+__all__ = ["QuarkW8A8Fp8", "QuarkW8A8Fp8PerBlock"]
40
41
logger = init_logger(__name__)
42
@@ -46,6 +52,11 @@ class QuarkW8A8Fp8(QuarkScheme):
52
self.is_static_input_scheme = not cast(bool, input_config.get("is_dynamic"))
53
self.input_qscheme = cast(str, input_config.get("qscheme"))
54
55
+ if self.weight_qscheme == "per_block":
56
+ raise ValueError(
57
+ "Quark W8A8 FP8 per-block weight quantization should use "
58
+ "QuarkW8A8Fp8PerBlock."
59
+ )
60
per_token_activation = (
61
not self.is_static_input_scheme and self.input_qscheme == "per_channel"
62
)
@@ -196,3 +207,103 @@ class QuarkW8A8Fp8(QuarkScheme):
207
bias: torch.Tensor | None = None,
208
) -> torch.Tensor:
209
return self.fp8_linear.apply_weights(layer, x, bias)
210
+
211
+
212
+class QuarkW8A8Fp8PerBlock(QuarkScheme):
213
+ def __init__(
214
+ self, weight_config: dict[str, Any], input_config: dict[str, Any] | None
215
+ ):
216
+ self.weight_qscheme = cast(str, weight_config.get("qscheme"))
217
+ if self.weight_qscheme != "per_block":
218
+ raise ValueError("QuarkW8A8Fp8PerBlock requires per-block weights.")
219
+ block_size = weight_config.get("block_size")
220
+ if not block_size:
221
+ raise ValueError(
222
+ "Quark W8A8 FP8 per-block weight quantization requires "
223
+ "`block_size` in the weight quantization config."
224
+ )
225
+ self.weight_config = weight_config
226
+ self.weight_block_size = list(block_size)
227
+ self.activation_quant_key = kFp8Dynamic128Sym
228
+ self.weight_quant_key = kFp8Static128BlockSym
229
+ self.out_dtype = torch.get_default_dtype()
230
+ self.input_dtype = get_current_vllm_config().model_config.dtype
231
+
232
+ @classmethod
233
+ def get_min_capability(cls) -> int:
234
+ return QuarkW8A8Fp8.get_min_capability()
235
+
236
+ def process_weights_after_loading(self, layer) -> None:
237
+ self.fp8_linear.process_weights_after_loading(layer)
238
+
239
+ def create_weights(
240
+ self,
241
+ layer: torch.nn.Module,
242
+ output_partition_sizes: list[int],
243
+ input_size_per_partition: int,
244
+ params_dtype: torch.dtype,
245
+ weight_loader: Callable,
246
+ **kwargs,
247
+ ):
248
+ input_size = kwargs.get("input_size", input_size_per_partition)
249
+ output_size = kwargs.get("output_size", sum(output_partition_sizes))
250
+ output_size_per_partition = sum(output_partition_sizes)
251
+
252
+ layer.logical_widths = output_partition_sizes
253
+ layer.input_size_per_partition = input_size_per_partition
254
+ layer.output_size_per_partition = output_size_per_partition
255
+ layer.orig_dtype = params_dtype
256
+ layer.weight_block_size = self.weight_block_size
257
+
258
+ validate_fp8_block_shape(
259
+ layer,
260
+ input_size,
261
+ output_size,
262
+ input_size_per_partition,
263
+ output_partition_sizes,
264
+ self.weight_block_size,
265
+ )
266
+
267
+ weight = create_fp8_weight_parameter(
268
+ output_size_per_partition,
269
+ input_size_per_partition,
270
+ weight_loader,
271
+ )
272
+ layer.register_parameter("weight", weight)
273
+
274
+ scale_dtype = (
275
+ torch.float8_e8m0fnu
276
+ if self.weight_config.get("scale_type") == "float8_e8m0fnu"
277
+ else None
278
+ )
279
+ weight_scale = create_fp8_scale_parameter(
280
+ BlockQuantScaleParameter,
281
+ output_partition_sizes,
282
+ input_size_per_partition,
283
+ self.weight_block_size,
284
+ weight_loader,
285
+ scale_dtype=scale_dtype,
286
+ )
287
+ # DeepSeek V4 weight mappers route checkpoint ".scale" tensors here.
288
+ layer.register_parameter("weight_scale_inv", weight_scale)
289
+
290
+ self.fp8_linear = init_fp8_linear_kernel(
291
+ activation_quant_key=self.activation_quant_key,
292
+ weight_quant_key=self.weight_quant_key,
293
+ weight_shape=layer.weight.shape,
294
+ input_dtype=self.input_dtype,
295
+ out_dtype=self.out_dtype,
296
+ module_name=self.__class__.__name__,
297
+ )
298
+ logger.info_once(
299
+ "Selected %s for QuarkW8A8Fp8PerBlock",
300
+ type(self.fp8_linear).__name__,
301
+ )
302
+
303
+ def apply_weights(
304
+ self,
305
+ layer: torch.nn.Module,
306
+ x: torch.Tensor,
307
+ bias: torch.Tensor | None = None,
308
+ ) -> torch.Tensor:
309
+ return self.fp8_linear.apply_weights(layer, x, bias)
vllm/models/deepseek_v4/amd/model.py
+17
-4
@@ -750,6 +750,12 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
750
params_dict = dict(self.named_parameters())
751
loaded_params: set[str] = set()
752
753
+ def _resolve_param_name(name: str) -> str:
754
+ inv_name = f"{name}_inv"
755
+ if name not in params_dict and inv_name in params_dict:
756
+ return inv_name
757
+ return name
758
+
759
# TP for attention
760
tp_size = get_tensor_model_parallel_world_size()
761
tp_rank = get_tensor_model_parallel_rank()
@@ -803,10 +809,11 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
809
810
if is_pp_missing_parameter(name, self):
811
break
806
- param = params_dict[name]
812
+ param_name = _resolve_param_name(name)
813
+ param = params_dict[param_name]
814
weight_loader = param.weight_loader
815
weight_loader(param, loaded_weight, shard_id)
809
- loaded_params.add(name)
816
+ loaded_params.add(param_name)
817
break
818
else:
819
if ".experts." in name:
@@ -857,12 +864,13 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
864
else:
865
if is_pp_missing_parameter(name, self):
866
continue
860
- param = params_dict[name]
867
+ param_name = _resolve_param_name(name)
868
+ param = params_dict[param_name]
869
weight_loader = getattr(
870
param, "weight_loader", default_weight_loader
871
)
872
weight_loader(param, loaded_weight)
865
- loaded_params.add(name)
873
+ loaded_params.add(param_name)
874
continue
875
876
return loaded_params
@@ -934,6 +942,7 @@ def _make_deepseek_v4_weights_mapper(
942
"head.weight": "lm_head.weight",
943
"embed.weight": "embed_tokens.weight",
944
".ffn.gate.bias": ".ffn.gate.e_score_correction_bias",
945
+ ".input_scale": ".input_scale_2",
946
},
947
orig_to_new_substr=substr_map,
948
)
@@ -945,6 +954,10 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
954
# Default mapper assumes the original FP4-expert checkpoint layout.
955
# Overridden per-instance in __init__ when expert_dtype != "fp4".
956
hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper("fp4")
957
+ packed_modules_mapping = {
958
+ "gate_up_proj": ["w1", "w3"],
959
+ "fused_wqa_wkv": ["wq_a", "wkv"],
960
+ }
961
962
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
963
super().__init__()