[Bugfix][Quantization] Detect NVFP4 in ModelOpt mixed-precision checkpoints (#56050)
Signed-off-by: Stefano Castagnetta <scastagnetta@nvidia.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Stefano Castagnetta committed
Oct 2, 2026 at 14:30 UTC
f0b4b36890ad0465c0564a31ec34fc2bb12ab745
4 files changed
+125
-1
tests/compile/fusions_e2e/models.py
+14
@@ -101,6 +101,20 @@ llama3_8b_fp4 = ModelFusionInfo(
101
),
102
)
103
104
+# ModelOpt MIXED_PRECISION: FP8 attention projections, NVFP4 MLPs. It resolves to
105
+# modelopt_mixed, not modelopt_fp4, and its MLP has no manual act+quant fusion, so
106
+# every MLP boundary is left to ActivationQuantFusionPass.
107
+qwen3_8_27b_mixed_fp4 = ModelFusionInfo(
108
+ model_name="nvidia/Qwen3.8-27B-NVFP4",
109
+ hf_overrides=lambda n_layers: {
110
+ "text_config": {
111
+ "num_hidden_layers": n_layers,
112
+ "layer_types": ["linear_attention"] * (n_layers - 1) + ["full_attention"],
113
+ }
114
+ },
115
+ matches=lambda n_layers: Matches(act_quant_fusion=n_layers),
116
+)
117
+
118
# MoEs cannot do act+quant fusion because those ops are hidden from torch.compile.
119
# MoEs also only expose 1 rms+quant fusion because the quant for up_proj is hidden.
120
# TODO(luka): https://github.com/vllm-project/vllm/issues/31985
tests/compile/fusions_e2e/test_tp1_quant.py
+44
@@ -30,6 +30,7 @@ from .models import (
30
llama3_8b_fp8,
31
llama4_scout_fp4,
32
llama4_scout_fp8,
33
+ qwen3_8_27b_mixed_fp4,
34
qwen3_a3b_fp8,
35
)
36
@@ -202,3 +203,46 @@ def test_tp1_fp4_fusions(
203
compilation_config,
204
matches_check,
205
)
206
+
207
+
208
+@pytest.mark.parametrize(
209
+ "model_name, matches_fn, model_kwargs, hf_overrides", [qwen3_8_27b_mixed_fp4]
210
+)
211
+@pytest.mark.parametrize("attn_backend", [FLASHINFER_ATTN])
212
+@pytest.mark.parametrize("n_layers", [4])
213
+@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
214
+@pytest.mark.skipif(not is_blackwell(), reason="Blackwell required for fp4")
215
+def test_tp1_fp4_mixed_precision_act_fusion(
216
+ model_name: str,
217
+ matches_fn: Callable[[int], Matches],
218
+ model_kwargs: dict,
219
+ hf_overrides: Callable[[int], dict],
220
+ attn_backend: AttentionBackendCase,
221
+ n_layers: int,
222
+ inductor_graph_partition: bool,
223
+ run_e2e_fusion_test,
224
+):
225
+ """Act+quant fusion matches on a MIXED_PRECISION NVFP4 checkpoint."""
226
+ model_kwargs["hf_overrides"] = hf_overrides(n_layers)
227
+ model_kwargs["load_format"] = "dummy"
228
+ model_kwargs["max_model_len"] = 1024
229
+ model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
230
+
231
+ compilation_config = dict(
232
+ use_inductor_graph_partition=inductor_graph_partition,
233
+ pass_config=PassConfig(
234
+ fuse_norm_quant=False,
235
+ fuse_act_quant=True,
236
+ fuse_attn_quant=False,
237
+ enable_qk_norm_rope_fusion=False,
238
+ ),
239
+ )
240
+
241
+ run_e2e_fusion_test(
242
+ model_name,
243
+ matches_fn(n_layers),
244
+ model_kwargs,
245
+ attn_backend,
246
+ compilation_config,
247
+ ["act_quant_fusion"],
248
+ )
tests/config/test_modelopt_mixed_nvfp4.py
new
+43
@@ -0,0 +1,43 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""NVFP4 detection for ModelOpt MIXED_PRECISION checkpoints.
4
+
5
+A checkpoint that quantizes some layers to NVFP4 and others to FP8 declares
6
+``quant_algo: MIXED_PRECISION`` and resolves to ``modelopt_mixed``, so NVFP4
7
+has to be detected from the per-layer algorithms.
8
+"""
9
+
10
+import pytest
11
+
12
+from vllm.config.model import _modelopt_mixed_has_nvfp4
13
+
14
+
15
+def _quantized_layers(*algos: str) -> dict:
16
+ return {
17
+ "quantized_layers": {
18
+ f"model.layers.{i}.proj": {"quant_algo": algo}
19
+ for i, algo in enumerate(algos)
20
+ }
21
+ }
22
+
23
+
24
+def test_detects_nvfp4_alongside_another_algorithm():
25
+ """The shipped shape: FP8 attention projections, NVFP4 MLP."""
26
+ assert _modelopt_mixed_has_nvfp4(_quantized_layers("FP8", "NVFP4", "FP8"))
27
+
28
+
29
+def test_ignores_a_checkpoint_with_no_nvfp4_layer():
30
+ assert not _modelopt_mixed_has_nvfp4(_quantized_layers("FP8", "MXFP8"))
31
+
32
+
33
+def test_weight_only_nvfp4_does_not_count():
34
+ """W4A16_NVFP4 leaves activations in high precision, so nothing to fuse."""
35
+ assert not _modelopt_mixed_has_nvfp4(_quantized_layers("FP8", "W4A16_NVFP4"))
36
+
37
+
38
+@pytest.mark.parametrize(
39
+ "quant_config",
40
+ [None, {}, {"quantized_layers": None}, {"quantized_layers": ["NVFP4"]}],
41
+)
42
+def test_absent_or_malformed_config_is_not_nvfp4(quant_config):
43
+ assert not _modelopt_mixed_has_nvfp4(quant_config)
vllm/config/model.py
+24
-1
@@ -124,6 +124,22 @@ _RUNNER_CONVERTS: dict[RunnerType, list[ConvertType]] = {
124
"draft": [],
125
}
126
127
+
128
+def _modelopt_mixed_has_nvfp4(quant_config: dict[str, Any] | None) -> bool:
129
+ """Whether a ModelOpt MIXED_PRECISION checkpoint quantizes any layer to NVFP4.
130
+
131
+ ``W4A16_NVFP4`` is excluded on purpose: it quantizes weights only, so there
132
+ is no activation quantization for the fusion passes to act on.
133
+ """
134
+ layers = (quant_config or {}).get("quantized_layers")
135
+ if not isinstance(layers, dict):
136
+ return False
137
+ return any(
138
+ isinstance(info, dict) and info.get("quant_algo") == "NVFP4"
139
+ for info in layers.values()
140
+ )
141
+
142
+
143
AttnTypeStr = Literal[
144
"decoder", "encoder", "encoder_only", "encoder_decoder", "attention_free", "hybrid"
145
]
@@ -2249,13 +2265,20 @@ class ModelConfig:
2265
return getattr(self.hf_config, "quantization_config", None) is not None
2266
2267
def is_nvfp4_quantized(self) -> bool:
2268
+ quant_config = self.model_arch_config.quantization_config
2269
+
2270
# ModelOpt NVFP4 checkpoints resolve to modelopt_fp4 quantization method
2271
if self.quantization in ("modelopt_fp4",):
2272
return True
2273
2274
+ # A checkpoint mixing NVFP4 with another algorithm declares
2275
+ # quant_algo MIXED_PRECISION and resolves to modelopt_mixed, so the
2276
+ # per-layer algorithms decide.
2277
+ if self.quantization == "modelopt_mixed":
2278
+ return _modelopt_mixed_has_nvfp4(quant_config)
2279
+
2280
# For Compressed Tensors we look for `"format": "nvfp4-pack-quantized"`
2281
# in the quantization config
2258
- quant_config = self.model_arch_config.quantization_config
2282
return (
2283
self.quantization == "compressed-tensors"
2284
and quant_config is not None