[Quantization][INC]Add MXFP8 Linear Support (#47514)
Signed-off-by: Zhenzhong1 <zhenzhong.xu@intel.com> Signed-off-by: Zhenzhong Xu <zhenzhong.xu@intel.com> Co-authored-by: Yi Liu <yi4.liu@intel.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Zhenzhong Xu committed
Jul 27, 2026 at 14:26 UTC
5d07e268b11ae0240d3af2582516861dfe52e86a
7 files changed
+322
-9
tests/quantization/test_auto_round.py
+138
-3
@@ -18,9 +18,13 @@ from vllm.model_executor.layers.quantization.inc import INCConfig
18
from vllm.model_executor.layers.quantization.inc.config_parser import INCLayerConfig
19
from vllm.model_executor.layers.quantization.inc.inc_linear import INCLinearMethod
20
from vllm.model_executor.layers.quantization.inc.schemes import (
21
+ INCMxfp8Scheme,
22
INCWna16Scheme,
23
resolve_scheme,
24
)
25
+from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear import (
26
+ INCMxfp8LinearScheme,
27
+)
28
from vllm.model_executor.layers.quantization.inc.schemes.inc_scheme import (
29
INCLinearScheme,
30
)
@@ -52,12 +56,19 @@ MODELS = [
56
pytest.param(
57
"Intel/Qwen3-8B-w2g64-for-ut",
58
marks=pytest.mark.skipif(
55
- not (current_platform.is_cuda() or current_platform.is_xpu())
56
- or current_platform.device_count() < 2,
57
- reason="72B INT2 AutoRound model requires XPU with at least 2 devices.",
59
+ not (current_platform.is_xpu()),
60
+ reason="INC int2 on XPU requires the ARK backend.",
61
),
62
id="auto_round:auto_gptq_int2_tp2",
63
),
64
+ pytest.param(
65
+ "INC4AI/Qwen3-8B-MXFP8-AR",
66
+ marks=pytest.mark.skipif(
67
+ not (current_platform.is_cuda() or current_platform.is_xpu()),
68
+ reason="MXFP8 AutoRound model only supports CUDA/XPU backend for now.",
69
+ ),
70
+ id="auto_round:llm_compressor_mxfp8",
71
+ ),
72
]
73
74
MODEL_RUNNER_KWARGS = {
@@ -66,6 +77,11 @@ MODEL_RUNNER_KWARGS = {
77
"gpu_memory_utilization": 0.8,
78
"max_model_len": 512,
79
},
80
+ "INC4AI/Qwen3-8B-MXFP8-AR": {
81
+ "block_size": 64,
82
+ "gpu_memory_utilization": 0.8,
83
+ "max_model_len": 512,
84
+ },
85
}
86
87
@@ -273,6 +289,39 @@ def test_inc_config_parser_fused_module_requires_consistent_configs() -> None:
289
config.config_parser.resolve(DummyLayer(), "layers.0.self_attn.qkv_proj")
290
291
292
+def test_inc_mxfp8() -> None:
293
+ config = make_config(
294
+ weight_bits=8,
295
+ group_size=32,
296
+ sym=True,
297
+ packing_format="auto_round:llm_compressor",
298
+ data_type="mx_fp",
299
+ )
300
+
301
+ assert config.weight_bits == 8
302
+ assert config.group_size == 32
303
+ assert config.data_type == "mx_fp"
304
+ assert config.packing_format == "auto_round:llm_compressor"
305
+
306
+
307
+def test_inc_config_rejects_invalid_mxfp8_activation_config() -> None:
308
+ with pytest.raises(AssertionError, match="act_dynamic=True"):
309
+ INCConfig.from_config(
310
+ {
311
+ "bits": 8,
312
+ "group_size": 32,
313
+ "sym": True,
314
+ "packing_format": "auto_round:llm_compressor",
315
+ "data_type": "mx_fp",
316
+ "act_bits": 8,
317
+ "act_data_type": "mx_fp",
318
+ "act_group_size": 32,
319
+ "act_sym": True,
320
+ "act_dynamic": False,
321
+ }
322
+ )
323
+
324
+
325
def test_inc_layer_config_mx_fp_helpers() -> None:
326
layer_config = INCLayerConfig(
327
bits=4,
@@ -304,6 +353,22 @@ def test_inc_resolve_scheme_selects_wna16() -> None:
353
assert isinstance(scheme, INCWna16Scheme)
354
355
356
+def test_inc_resolve_scheme_selects_mxfp8() -> None:
357
+ layer_config = INCLayerConfig(
358
+ bits=8,
359
+ group_size=32,
360
+ sym=True,
361
+ packing_format="auto_round:llm_compressor",
362
+ backend="auto",
363
+ data_type="mx_fp",
364
+ quantized=True,
365
+ )
366
+
367
+ scheme = resolve_scheme(layer_config)
368
+
369
+ assert isinstance(scheme, INCMxfp8Scheme)
370
+
371
+
372
class DummyLinearScheme(INCLinearScheme):
373
def __init__(self) -> None:
374
self.calls: list[tuple] = []
@@ -323,6 +388,76 @@ class DummyLinearScheme(INCLinearScheme):
388
return "applied"
389
390
391
+def test_inc_mxfp8_linear_scheme_delegates_to_kernel(monkeypatch) -> None:
392
+ class DummyKernel:
393
+ def __init__(self) -> None:
394
+ self.calls: list[tuple] = []
395
+
396
+ def process_weights_after_loading(self, layer) -> None:
397
+ self.calls.append(("process", layer))
398
+
399
+ def apply_weights(self, layer, x, bias=None):
400
+ self.calls.append(("apply", layer, x, bias))
401
+ return "applied"
402
+
403
+ kernel = DummyKernel()
404
+ monkeypatch.setattr(
405
+ "vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear.init_mxfp8_linear_kernel",
406
+ lambda: kernel,
407
+ )
408
+ monkeypatch.setattr(
409
+ "vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear.ModelWeightParameter",
410
+ lambda **kwargs: torch.nn.Parameter(kwargs["data"], requires_grad=False),
411
+ )
412
+ monkeypatch.setattr(
413
+ "vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear.GroupQuantScaleParameter",
414
+ lambda **kwargs: torch.nn.Parameter(kwargs["data"], requires_grad=False),
415
+ )
416
+
417
+ scheme = INCMxfp8LinearScheme()
418
+ layer = torch.nn.Module()
419
+
420
+ scheme.create_weights(
421
+ layer=layer,
422
+ input_size_per_partition=64,
423
+ output_partition_sizes=[48, 16],
424
+ input_size=64,
425
+ output_size=64,
426
+ params_dtype=torch.bfloat16,
427
+ weight_loader=lambda *args, **kwargs: None,
428
+ )
429
+
430
+ assert layer.weight.shape == (64, 64)
431
+ assert layer.weight.dtype == torch.float8_e4m3fn
432
+ assert layer.weight_scale.shape == (64, 2)
433
+ assert layer.weight_scale.dtype == torch.uint8
434
+
435
+ scheme.process_weights_after_loading(layer)
436
+ result = scheme.apply_weights(layer, torch.randn(1, 64), None)
437
+
438
+ assert result == "applied"
439
+ assert [call[0] for call in kernel.calls] == ["process", "apply"]
440
+
441
+
442
+def test_inc_mxfp8_linear_scheme_requires_block_32_input(monkeypatch) -> None:
443
+ monkeypatch.setattr(
444
+ "vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear.init_mxfp8_linear_kernel",
445
+ lambda: object(),
446
+ )
447
+ scheme = INCMxfp8LinearScheme()
448
+
449
+ with pytest.raises(ValueError, match="divisible by 32"):
450
+ scheme.create_weights(
451
+ layer=torch.nn.Module(),
452
+ input_size_per_partition=48,
453
+ output_partition_sizes=[32],
454
+ input_size=48,
455
+ output_size=32,
456
+ params_dtype=torch.bfloat16,
457
+ weight_loader=lambda *args, **kwargs: None,
458
+ )
459
+
460
+
461
def test_inc_linear_method_delegates() -> None:
462
scheme = DummyLinearScheme()
463
method = INCLinearMethod(scheme)
vllm/model_executor/layers/quantization/inc/inc.py
+64
-6
@@ -35,8 +35,12 @@ class INCConfig(QuantizationConfig):
35
"""
36
37
SUPPORTED_BITS = {2, 3, 4, 8}
38
- SUPPORTED_DTYPES = {"int"}
39
- SUPPORTED_FORMATS = {"auto_round:auto_gptq", "auto_round:auto_awq"}
38
+ SUPPORTED_DTYPES = {"int", "mx_fp"}
39
+ SUPPORTED_FORMATS = {
40
+ "auto_round:auto_gptq",
41
+ "auto_round:auto_awq",
42
+ "auto_round:llm_compressor",
43
+ }
44
SUPPORTED_BACKENDS = {
45
"auto",
46
"gptq",
@@ -45,6 +49,11 @@ class INCConfig(QuantizationConfig):
49
"awq:marlin",
50
"marlin",
51
}
52
+ MXFP8_BITS = 8
53
+ MXFP8_GROUP_SIZE = 32
54
+ MXFP8_DATA_TYPE = "mx_fp"
55
+ MXFP8_PACKING_FORMAT = "auto_round:llm_compressor"
56
+ MXFP8_SUPPORTED_ACT_DTYPES = {"mx_fp", "mx_fp_rceil"}
57
58
def __init__(
59
self,
@@ -65,8 +74,8 @@ class INCConfig(QuantizationConfig):
74
)
75
if data_type not in self.SUPPORTED_DTYPES:
76
raise ValueError(
68
- f"Unsupported data_type: {data_type},"
69
- f" currently only support {self.SUPPORTED_DTYPES}."
77
+ f"Unsupported data_type: {data_type}, "
78
+ f"currently only support {self.SUPPORTED_DTYPES}."
79
)
80
if packing_format not in self.SUPPORTED_FORMATS:
81
raise ValueError(
@@ -75,7 +84,7 @@ class INCConfig(QuantizationConfig):
84
)
85
if backend not in self.SUPPORTED_BACKENDS:
86
raise ValueError(
78
- f"Unsupported backend: {backend}, "
87
+ f"Unsupported backend: {backend}, "
88
f"currently only support {self.SUPPORTED_BACKENDS}."
89
)
90
@@ -94,12 +103,59 @@ class INCConfig(QuantizationConfig):
103
self.pack_factor = Fraction(32, weight_bits)
104
self.config_parser = INCConfigParser(self)
105
106
+ self._validate_supported_quantization()
107
+
108
def __repr__(self) -> str:
109
return (
110
f"INCConfig(weight_bits={self.weight_bits}, "
111
f"group_size={self.group_size}, sym={self.sym})"
112
)
113
114
+ def _validate_supported_quantization(self) -> None:
115
+ if self.data_type == self.MXFP8_DATA_TYPE:
116
+ assert self.weight_bits == self.MXFP8_BITS, (
117
+ f"INC MXFP8 only supports bits=8, but found bits={self.weight_bits}."
118
+ )
119
+ assert self.group_size == self.MXFP8_GROUP_SIZE, (
120
+ "INC MXFP8 only supports group_size=32, "
121
+ f"but found group_size={self.group_size}."
122
+ )
123
+ assert self.sym, "INC MXFP8 only supports symmetric weights."
124
+ assert self.packing_format == self.MXFP8_PACKING_FORMAT, (
125
+ "INC MXFP8 only supports "
126
+ f"packing_format={self.MXFP8_PACKING_FORMAT!r}, "
127
+ f"but found {self.packing_format!r}."
128
+ )
129
+ assert self.backend == "auto", (
130
+ "INC MXFP8 only supports backend='auto', "
131
+ f"but found backend={self.backend!r}."
132
+ )
133
+ elif self.packing_format == self.MXFP8_PACKING_FORMAT:
134
+ raise ValueError(
135
+ f"packing_format={self.MXFP8_PACKING_FORMAT!r} requires "
136
+ f"data_type={self.MXFP8_DATA_TYPE!r}."
137
+ )
138
+
139
+ def _validate_raw_config(self, config: dict[str, Any]) -> None:
140
+ if self.data_type != self.MXFP8_DATA_TYPE:
141
+ return
142
+
143
+ expected_fields = {
144
+ "act_bits": self.MXFP8_BITS,
145
+ "act_data_type": self.MXFP8_DATA_TYPE,
146
+ "act_group_size": self.MXFP8_GROUP_SIZE,
147
+ "act_sym": True,
148
+ "act_dynamic": True,
149
+ "enable_quanted_input": False,
150
+ }
151
+ for field_name, expected_value in expected_fields.items():
152
+ actual_value = self.get_from_keys_or(config, [field_name], expected_value)
153
+ assert actual_value == expected_value, (
154
+ "INC MXFP8 only supports "
155
+ f"{field_name}={expected_value!r}, "
156
+ f"but found {field_name}={actual_value!r}."
157
+ )
158
+
159
@classmethod
160
def get_name(cls) -> QuantizationMethods:
161
return "inc"
@@ -118,7 +174,7 @@ class INCConfig(QuantizationConfig):
174
175
@classmethod
176
def from_config(cls, config: dict[str, Any]) -> "INCConfig":
121
- return cls(
177
+ quant_config = cls(
178
weight_bits=cls.get_from_keys(config, ["bits"]),
179
group_size=cls.get_from_keys(config, ["group_size"]),
180
sym=cls.get_from_keys(config, ["sym"]),
@@ -132,6 +188,8 @@ class INCConfig(QuantizationConfig):
188
data_type=cls.get_from_keys_or(config, ["data_type"], "int"),
189
backend=cls.get_from_keys_or(config, ["backend", "vllm_backend"], "auto"),
190
)
191
+ quant_config._validate_raw_config(config)
192
+ return quant_config
193
194
def get_layer_config(self, layer, layer_name: str):
195
return self.config_parser.get_layer_config(layer, layer_name)
vllm/model_executor/layers/quantization/inc/schemes/__init__.py
+2
@@ -2,12 +2,14 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
from .factory import resolve_scheme
5
+from .inc_mxfp8_scheme import INCMxfp8Scheme
6
from .inc_scheme import INCLinearScheme, INCScheme
7
from .inc_wna16_scheme import INCWna16Scheme
8
9
__all__ = [
10
"INCScheme",
11
"INCLinearScheme",
12
+ "INCMxfp8Scheme",
13
"INCWna16Scheme",
14
"resolve_scheme",
15
]
vllm/model_executor/layers/quantization/inc/schemes/factory.py
+2
@@ -9,9 +9,11 @@ if TYPE_CHECKING:
9
10
11
def resolve_scheme(layer_config: "INCLayerConfig") -> "INCScheme":
12
+ from .inc_mxfp8_scheme import INCMxfp8Scheme
13
from .inc_wna16_scheme import INCWna16Scheme
14
15
scheme_list: list[type[INCScheme]] = [
16
+ INCMxfp8Scheme,
17
INCWna16Scheme,
18
]
19
vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_linear.py
new
+84
@@ -0,0 +1,84 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import torch
4
+
5
+from vllm.model_executor.kernels.linear import init_mxfp8_linear_kernel
6
+from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
7
+ MXFP8_BLOCK_SIZE,
8
+ MXFP8_SCALE_DTYPE,
9
+ MXFP8_VALUE_DTYPE,
10
+)
11
+from vllm.model_executor.parameter import (
12
+ GroupQuantScaleParameter,
13
+ ModelWeightParameter,
14
+)
15
+
16
+from .inc_scheme import INCLinearScheme
17
+
18
+
19
+class INCMxfp8LinearScheme(INCLinearScheme):
20
+ def __init__(self) -> None:
21
+ self.kernel = init_mxfp8_linear_kernel()
22
+
23
+ @classmethod
24
+ def get_min_capability(cls) -> int:
25
+ return 75
26
+
27
+ def create_weights(
28
+ self,
29
+ layer: torch.nn.Module,
30
+ input_size_per_partition: int,
31
+ output_partition_sizes: list[int],
32
+ input_size: int,
33
+ output_size: int,
34
+ params_dtype: torch.dtype,
35
+ **extra_weight_attrs,
36
+ ) -> None:
37
+ del input_size, output_size
38
+ if input_size_per_partition % MXFP8_BLOCK_SIZE != 0:
39
+ raise ValueError(
40
+ "INC MXFP8 requires input_size_per_partition "
41
+ f"({input_size_per_partition}) to be divisible by "
42
+ f"{MXFP8_BLOCK_SIZE}."
43
+ )
44
+
45
+ output_size_per_partition = sum(output_partition_sizes)
46
+ layer.logical_widths = output_partition_sizes
47
+ layer.input_size_per_partition = input_size_per_partition
48
+ layer.output_size_per_partition = output_size_per_partition
49
+ layer.params_dtype = params_dtype
50
+
51
+ weight = ModelWeightParameter(
52
+ data=torch.empty(
53
+ output_size_per_partition,
54
+ input_size_per_partition,
55
+ dtype=MXFP8_VALUE_DTYPE,
56
+ ),
57
+ input_dim=1,
58
+ output_dim=0,
59
+ weight_loader=extra_weight_attrs.get("weight_loader"),
60
+ )
61
+ layer.register_parameter("weight", weight)
62
+
63
+ weight_scale = GroupQuantScaleParameter(
64
+ data=torch.empty(
65
+ output_size_per_partition,
66
+ input_size_per_partition // MXFP8_BLOCK_SIZE,
67
+ dtype=MXFP8_SCALE_DTYPE,
68
+ ),
69
+ input_dim=1,
70
+ output_dim=0,
71
+ weight_loader=extra_weight_attrs.get("weight_loader"),
72
+ )
73
+ layer.register_parameter("weight_scale", weight_scale)
74
+
75
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
76
+ self.kernel.process_weights_after_loading(layer)
77
+
78
+ def apply_weights(
79
+ self,
80
+ layer: torch.nn.Module,
81
+ x: torch.Tensor,
82
+ bias: torch.Tensor | None = None,
83
+ ) -> torch.Tensor:
84
+ return self.kernel.apply_weights(layer, x, bias)
vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_scheme.py
new
+31
@@ -0,0 +1,31 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from typing import TYPE_CHECKING
5
+
6
+from ..inc_linear import INCLinearMethod
7
+from .inc_scheme import INCScheme
8
+
9
+if TYPE_CHECKING:
10
+ import torch
11
+
12
+ from ..config_parser import INCLayerConfig
13
+ from ..inc import INCConfig
14
+
15
+
16
+class INCMxfp8Scheme(INCScheme):
17
+ @staticmethod
18
+ def can_handle(layer_config: "INCLayerConfig") -> bool:
19
+ return layer_config.is_mxfp8
20
+
21
+ def get_linear_method(
22
+ self,
23
+ config: "INCConfig",
24
+ layer: "torch.nn.Module",
25
+ prefix: str,
26
+ layer_config: "INCLayerConfig",
27
+ ):
28
+ del config, layer, prefix, layer_config
29
+ from .inc_mxfp8_linear import INCMxfp8LinearScheme
30
+
31
+ return INCLinearMethod(INCMxfp8LinearScheme())
vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_linear.py
+1
@@ -166,6 +166,7 @@ class INCXPULinearBase(INCLinearScheme):
166
def __init__(self, layer_config: "INCLayerConfig") -> None:
167
self.weight_bits = layer_config.bits
168
self.group_size = layer_config.group_size
169
+
170
self.sym = layer_config.sym
171
self.pack_factor = 32 // self.weight_bits
172
self.is_awq_packed = layer_config.is_awq