[AutoRound] Support AutoRound Format Block-Wise FP8 in vLLM (#47434)
Signed-off-by: Zhenzhong1 <zhenzhong.xu@intel.com> Signed-off-by: Zhenzhong Xu <zhenzhong.xu@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
Zhenzhong Xu committed
Aug 31, 2026 at 22:56 UTC
bd575a0d0bdf482557648e5278bbba98b6a939b9
10 files changed
+503
-62
tests/quantization/test_auto_round.py
+167
-1
@@ -22,11 +22,15 @@ from vllm.model_executor.layers.quantization.inc import INCConfig
22
from vllm.model_executor.layers.quantization.inc.config_parser import INCLayerConfig
23
from vllm.model_executor.layers.quantization.inc.inc_linear import INCLinearMethod
24
from vllm.model_executor.layers.quantization.inc.schemes import (
25
+ INCFp8Scheme,
26
INCMxfp4Scheme,
27
INCMxfp8Scheme,
28
INCWna16Scheme,
29
resolve_scheme,
30
)
31
+from vllm.model_executor.layers.quantization.inc.schemes.inc_fp8_linear import (
32
+ INCFp8LinearScheme,
33
+)
34
from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear import (
35
INCMxfp8LinearScheme,
36
)
@@ -77,6 +81,14 @@ MODELS = [
81
),
82
id="auto_round:llm_compressor_mxfp8",
83
),
84
+ pytest.param(
85
+ "INC4AI/Qwen3-8B-fp-w8g128x128-for-ut",
86
+ marks=pytest.mark.skipif(
87
+ not (current_platform.is_cuda()),
88
+ reason="Block-wise AutoRound model only supports CUDA backend for now.",
89
+ ),
90
+ id="auto_round:block_wise_fp8_on_cuda",
91
+ ),
92
]
93
94
QWEN3_AUTOROUND_MODELS = [
@@ -128,6 +140,10 @@ MODEL_RUNNER_KWARGS: dict[str, dict[str, Any]] = {
140
"gpu_memory_utilization": 0.8,
141
"max_model_len": 512,
142
},
143
+ "INC4AI/Qwen3-8B-fp-w8g128x128-for-ut": {
144
+ "gpu_memory_utilization": 0.8,
145
+ "max_model_len": 512,
146
+ },
147
}
148
149
@@ -190,6 +206,27 @@ def make_layer_config(**overrides) -> INCLayerConfig:
206
return INCLayerConfig(**kwargs)
207
208
209
+def make_block_fp8_raw_config(**overrides) -> dict[str, object]:
210
+ kwargs = {
211
+ "bits": 8,
212
+ "group_size": [128, 128],
213
+ "sym": True,
214
+ "packing_format": "auto_round:fp8",
215
+ "quant_method": "auto-round",
216
+ "data_type": "fp",
217
+ "act_bits": 8,
218
+ "act_data_type": "fp",
219
+ "act_group_size": 128,
220
+ "act_sym": True,
221
+ "act_dynamic": True,
222
+ "activation_scheme": "dynamic",
223
+ "fmt": "e4m3",
224
+ "weight_block_size": [128, 128],
225
+ }
226
+ kwargs.update(overrides)
227
+ return kwargs
228
+
229
+
230
def make_qwen3_autoround_config(kind: str) -> INCConfig:
231
configs = {
232
"qwen3_1p7b_mxfp4": {
@@ -480,6 +517,52 @@ def test_inc_layer_config_mx_fp_helpers() -> None:
517
assert layer_config.is_mxfp8 is False
518
519
520
+def test_inc_config_accepts_block_fp8() -> None:
521
+ config = INCConfig.from_config(make_block_fp8_raw_config())
522
+
523
+ assert config.weight_bits == 8
524
+ assert config.group_size == (128, 128)
525
+ assert config.sym is True
526
+ assert config.data_type == "fp"
527
+ assert config.packing_format == "auto_round:fp8"
528
+
529
+
530
+def test_inc_config_rejects_invalid_block_fp8_activation_config() -> None:
531
+ with pytest.raises(AssertionError, match="act_dynamic=True"):
532
+ INCConfig.from_config(make_block_fp8_raw_config(act_dynamic=False))
533
+
534
+
535
+def test_inc_layer_config_fp8_block_helper() -> None:
536
+ layer_config = INCLayerConfig(
537
+ bits=8,
538
+ group_size=(128, 128),
539
+ sym=True,
540
+ packing_format="auto_round:fp8",
541
+ backend="auto",
542
+ data_type="fp",
543
+ quantized=True,
544
+ )
545
+
546
+ assert layer_config.is_fp8_block is True
547
+ assert layer_config.is_wna16_int is False
548
+
549
+
550
+def test_inc_resolve_scheme_selects_fp8_block() -> None:
551
+ layer_config = INCLayerConfig(
552
+ bits=8,
553
+ group_size=(128, 128),
554
+ sym=True,
555
+ packing_format="auto_round:fp8",
556
+ backend="auto",
557
+ data_type="fp",
558
+ quantized=True,
559
+ )
560
+
561
+ scheme = resolve_scheme(layer_config)
562
+
563
+ assert isinstance(scheme, INCFp8Scheme)
564
+
565
+
566
def test_inc_resolve_scheme_selects_wna16() -> None:
567
layer_config = INCLayerConfig(
568
bits=4,
@@ -1053,6 +1136,89 @@ def test_inc_linear_method_delegates() -> None:
1136
]
1137
1138
1139
+def test_inc_fp8_linear_scheme_delegates_to_fp8_linear_method(
1140
+ monkeypatch,
1141
+) -> None:
1142
+ captured = {}
1143
+
1144
+ class DummyMethod:
1145
+ def __init__(self, quant_config) -> None:
1146
+ captured["quant_config"] = quant_config
1147
+ captured["method"] = self
1148
+ self.calls: list[tuple] = []
1149
+ self.marlin_input_dtype = None
1150
+
1151
+ def create_weights(
1152
+ self,
1153
+ layer,
1154
+ input_size_per_partition,
1155
+ output_partition_sizes,
1156
+ input_size,
1157
+ output_size,
1158
+ params_dtype,
1159
+ **extra_weight_attrs,
1160
+ ) -> None:
1161
+ self.calls.append(
1162
+ (
1163
+ "create_weights",
1164
+ layer,
1165
+ input_size_per_partition,
1166
+ output_partition_sizes,
1167
+ input_size,
1168
+ output_size,
1169
+ params_dtype,
1170
+ extra_weight_attrs,
1171
+ )
1172
+ )
1173
+
1174
+ def process_weights_after_loading(self, layer) -> None:
1175
+ self.calls.append(("process_weights_after_loading", layer))
1176
+
1177
+ def apply(self, layer, x, bias=None):
1178
+ self.calls.append(("apply", layer, x, bias))
1179
+ return "applied"
1180
+
1181
+ monkeypatch.setattr(
1182
+ "vllm.model_executor.layers.quantization.inc.schemes.inc_fp8_linear."
1183
+ "Fp8LinearMethod",
1184
+ DummyMethod,
1185
+ )
1186
+ monkeypatch.setattr(
1187
+ "vllm.model_executor.layers.quantization.inc.schemes.inc_fp8_linear."
1188
+ "get_marlin_input_dtype",
1189
+ lambda _prefix: "fp8-marlin",
1190
+ )
1191
+
1192
+ scheme = INCFp8LinearScheme(
1193
+ prefix="layers.0.mlp.down_proj",
1194
+ weight_block_size=(128, 128),
1195
+ )
1196
+ layer = torch.nn.Module()
1197
+
1198
+ scheme.create_weights(
1199
+ layer=layer,
1200
+ input_size_per_partition=128,
1201
+ output_partition_sizes=[64, 64],
1202
+ input_size=128,
1203
+ output_size=128,
1204
+ params_dtype=torch.bfloat16,
1205
+ weight_loader=lambda *args, **kwargs: None,
1206
+ )
1207
+ scheme.process_weights_after_loading(layer)
1208
+ result = scheme.apply_weights(layer, torch.randn(1, 128), None)
1209
+
1210
+ assert captured["quant_config"].is_checkpoint_fp8_serialized is True
1211
+ assert captured["quant_config"].activation_scheme == "dynamic"
1212
+ assert captured["quant_config"].weight_block_size == [128, 128]
1213
+ assert captured["method"].marlin_input_dtype == "fp8-marlin"
1214
+ assert result == "applied"
1215
+ assert [call[0] for call in captured["method"].calls] == [
1216
+ "create_weights",
1217
+ "process_weights_after_loading",
1218
+ "apply",
1219
+ ]
1220
+
1221
+
1222
def test_wna16_xpu_prefers_ark_when_available(monkeypatch) -> None:
1223
class DummyQuantLinear:
1224
pass
@@ -1284,7 +1450,7 @@ def test_wna16_linear_gptq_uses_auto_gptq_when_supported(monkeypatch) -> None:
1450
1451
scheme = INCWNA16LinearScheme(make_layer_config())
1452
1287
- assert isinstance(scheme.inner_method, DummyMethod)
1453
+ assert isinstance(scheme.linear_method, DummyMethod)
1454
assert isinstance(captured["cfg"], AutoGPTQConfig)
1455
assert captured["cfg"].weight_bits == 4
1456
assert captured["cfg"].group_size == 128
vllm/model_executor/layers/quantization/inc/config_parser.py
+51
-10
@@ -17,7 +17,7 @@ if TYPE_CHECKING:
17
@dataclass(frozen=True)
18
class INCLayerConfig:
19
bits: int
20
- group_size: int
20
+ group_size: int | tuple[int, int]
21
sym: bool
22
packing_format: str
23
backend: str
@@ -44,6 +44,18 @@ class INCLayerConfig:
44
def is_mxfp8(self) -> bool:
45
return "mx_fp" in self.data_type and self.bits == 8
46
47
+ @property
48
+ def is_fp8_block(self) -> bool:
49
+ return (
50
+ self.data_type == "fp"
51
+ and self.bits == 8
52
+ and self.sym
53
+ and self.quantized
54
+ and self.packing_format == "auto_round:fp8"
55
+ and isinstance(self.group_size, tuple)
56
+ and len(self.group_size) == 2
57
+ )
58
+
59
60
class INCConfigParser:
61
def __init__(self, config: "INCConfig") -> None:
@@ -63,13 +75,13 @@ class INCConfigParser:
75
76
def get_layer_config(
77
self, layer: "torch.nn.Module", layer_name: str
66
- ) -> tuple[int, int, bool]:
78
+ ) -> tuple[int, int | tuple[int, int], bool]:
79
layer_config = self.resolve(layer, layer_name)
80
return layer_config.bits, layer_config.group_size, layer_config.sym
81
82
def _resolve_raw(
83
self, layer: "torch.nn.Module", layer_name: str
72
- ) -> tuple[int, int, bool]:
84
+ ) -> tuple[int, int | tuple[int, int], bool]:
85
REGEX_SPECIAL_CHARS = set(r"*+?^$()[]{}|\\")
86
87
def is_explicitly_configured(name: str) -> bool:
@@ -91,7 +103,9 @@ class INCConfigParser:
103
continue
104
return False
105
94
- def get_config(name: str, quantized: bool = True) -> tuple[int, int, bool]:
106
+ def get_config(
107
+ name: str, quantized: bool = True
108
+ ) -> tuple[int, int | tuple[int, int], bool]:
109
if not self._config.extra_config:
110
return (
111
self._config.weight_bits if quantized else 16,
@@ -103,9 +117,11 @@ class INCConfigParser:
117
cfg = self._config.extra_config[name]
118
return (
119
cfg.get("bits", self._config.weight_bits if quantized else 16),
106
- cfg.get(
107
- "group_size",
108
- self._config.group_size if quantized else -1,
120
+ self._normalize_group_size(
121
+ cfg.get(
122
+ "group_size",
123
+ self._config.group_size if quantized else -1,
124
+ )
125
),
126
cfg.get("sym", self._config.sym if quantized else True),
127
)
@@ -124,9 +140,11 @@ class INCConfigParser:
140
"bits",
141
self._config.weight_bits if quantized else 16,
142
),
127
- cfg.get(
128
- "group_size",
129
- self._config.group_size if quantized else -1,
143
+ self._normalize_group_size(
144
+ cfg.get(
145
+ "group_size",
146
+ self._config.group_size if quantized else -1,
147
+ )
148
),
149
cfg.get("sym", self._config.sym if quantized else True),
150
)
@@ -194,3 +212,26 @@ class INCConfigParser:
212
)
213
214
return get_config(layer_name, quantized)
215
+
216
+ @staticmethod
217
+ def _normalize_group_size(
218
+ group_size: int | list[int] | tuple[int, int],
219
+ ) -> int | tuple[int, int]:
220
+ """Normalize INC group_size into either an int or a 2-D int tuple."""
221
+ if isinstance(group_size, (list, tuple)):
222
+ if len(group_size) != 2 or not all(
223
+ isinstance(value, int) for value in group_size
224
+ ):
225
+ raise ValueError(
226
+ "INC block-wise FP8 requires group_size to be a 2-D "
227
+ f"integer sequence, but found {group_size!r}."
228
+ )
229
+ return (group_size[0], group_size[1])
230
+
231
+ if not isinstance(group_size, int):
232
+ raise ValueError(
233
+ "INC group_size must be an int or a 2-D integer sequence, "
234
+ f"but found {type(group_size).__name__}."
235
+ )
236
+
237
+ return group_size
vllm/model_executor/layers/quantization/inc/inc.py
+154
-47
@@ -34,11 +34,14 @@ class INCConfig(QuantizationConfig):
34
Repo: https://github.com/intel/neural-compressor
35
"""
36
37
+ DEFAULT_INT_PACKING_FORMAT = "auto_round:auto_gptq"
38
+
39
SUPPORTED_BITS = {2, 3, 4, 8}
38
- SUPPORTED_DTYPES = {"int", "mx_fp"}
40
+ SUPPORTED_DTYPES = {"int", "mx_fp", "fp"}
41
SUPPORTED_FORMATS = {
42
"auto_round:auto_gptq",
43
"auto_round:auto_awq",
44
+ "auto_round:fp8",
45
"auto_round:llm_compressor",
46
}
47
SUPPORTED_BACKENDS = {
@@ -49,23 +52,39 @@ class INCConfig(QuantizationConfig):
52
"awq:marlin",
53
"marlin",
54
}
55
+
56
+ FP8_BLOCK_BITS = 8
57
+ FP8_BLOCK_DATA_TYPE = "fp"
58
+ FP8_BLOCK_PACKING_FORMAT = "auto_round:fp8"
59
+ FP8_BLOCK_FMT = "e4m3"
60
+ FP8_BLOCK_ACTIVATION_SCHEME = "dynamic"
61
+
62
MXFP8_BITS = 8
63
MXFP8_GROUP_SIZE = 32
64
MXFP8_DATA_TYPE = "mx_fp"
65
MXFP8_PACKING_FORMAT = "auto_round:llm_compressor"
66
67
+ @staticmethod
68
+ def _normalize_data_type(data_type: str) -> str:
69
+ # Keep a small alias for branches that already experimented with "fp8".
70
+ return "fp" if data_type == "fp8" else data_type
71
+
72
def __init__(
73
self,
74
weight_bits: int,
60
- group_size: int,
75
+ group_size: int | tuple[int, int],
76
sym: bool = True,
62
- packing_format: str = "auto_round:auto_gptq",
77
+ packing_format: str = DEFAULT_INT_PACKING_FORMAT,
78
block_name_to_quantize: str | list[str] | None = None,
79
extra_config: dict[str, Any] | None = None,
80
data_type: str = "int",
81
backend: str = "auto",
82
) -> None:
83
super().__init__()
84
+
85
+ data_type = self._normalize_data_type(data_type)
86
+ group_size = INCConfigParser._normalize_group_size(group_size)
87
+
88
if weight_bits not in self.SUPPORTED_BITS:
89
raise ValueError(
90
f"Unsupported weight_bits: {weight_bits}, "
@@ -124,48 +143,118 @@ class INCConfig(QuantizationConfig):
143
# MXFP4 and MXFP8 share data_type "mx_fp" and differ only by bit width.
144
return self.is_mxfp and self.weight_bits == self.MXFP8_BITS
145
127
- def _validate_supported_quantization(self) -> None:
128
- if self.is_mxfp8:
129
- assert self.group_size == self.MXFP8_GROUP_SIZE, (
130
- "INC MXFP8 only supports group_size=32, "
131
- f"but found group_size={self.group_size}."
132
- )
133
- assert self.sym, "INC MXFP8 only supports symmetric weights."
134
- assert self.packing_format == self.MXFP8_PACKING_FORMAT, (
135
- "INC MXFP8 only supports "
136
- f"packing_format={self.MXFP8_PACKING_FORMAT!r}, "
137
- f"but found {self.packing_format!r}."
138
- )
139
- assert self.backend == "auto", (
140
- "INC MXFP8 only supports backend='auto', "
141
- f"but found backend={self.backend!r}."
142
- )
143
- elif self.packing_format == self.MXFP8_PACKING_FORMAT and not self.is_mxfp:
144
- raise ValueError(
145
- f"packing_format={self.MXFP8_PACKING_FORMAT!r} requires "
146
- f"an {self.MXFP8_DATA_TYPE!r} data_type."
147
- )
148
-
149
- def _validate_raw_config(self, config: dict[str, Any]) -> None:
150
- if not self.is_mxfp8:
151
- return
152
-
153
- expected_fields = {
154
- "act_bits": self.MXFP8_BITS,
155
- "act_data_type": self.MXFP8_DATA_TYPE,
156
- "act_group_size": self.MXFP8_GROUP_SIZE,
157
- "act_sym": True,
158
- "act_dynamic": True,
159
- "enable_quanted_input": False,
160
- }
146
+ @classmethod
147
+ def _check_expected(
148
+ cls,
149
+ values: dict[str, Any],
150
+ expected_fields: dict[str, Any],
151
+ quantization_name: str,
152
+ ) -> None:
153
for field_name, expected_value in expected_fields.items():
162
- actual_value = self.get_from_keys_or(config, [field_name], expected_value)
154
+ actual_value = cls.get_from_keys_or(values, [field_name], expected_value)
155
assert actual_value == expected_value, (
164
- "INC MXFP8 only supports "
156
+ f"{quantization_name} only supports "
157
f"{field_name}={expected_value!r}, "
158
f"but found {field_name}={actual_value!r}."
159
)
160
161
+ def _validate_int_quantization(self) -> None:
162
+ int_packing_formats = {
163
+ self.DEFAULT_INT_PACKING_FORMAT,
164
+ "auto_round:auto_awq",
165
+ }
166
+ assert isinstance(self.group_size, int), (
167
+ "INC int quantization requires scalar group_size, "
168
+ f"but found group_size={self.group_size!r}."
169
+ )
170
+ assert self.packing_format in int_packing_formats, (
171
+ "INC int quantization only supports "
172
+ f"packing_format in {sorted(int_packing_formats)!r}, "
173
+ f"but found packing_format={self.packing_format!r}."
174
+ )
175
+
176
+ def _validate_fp8_block_quantization(self) -> None:
177
+ assert self.weight_bits == self.FP8_BLOCK_BITS, (
178
+ "INC block-wise FP8 only supports bits=8, "
179
+ f"but found bits={self.weight_bits}."
180
+ )
181
+ assert isinstance(self.group_size, tuple), (
182
+ "INC block-wise FP8 requires a 2-D group_size, "
183
+ f"but found group_size={self.group_size!r}."
184
+ )
185
+ assert self.sym, "INC block-wise FP8 only supports symmetric weights."
186
+ assert self.packing_format == self.FP8_BLOCK_PACKING_FORMAT, (
187
+ "INC block-wise FP8 only supports "
188
+ f"packing_format={self.FP8_BLOCK_PACKING_FORMAT!r}, "
189
+ f"but found {self.packing_format!r}."
190
+ )
191
+ assert self.backend == "auto", (
192
+ "INC block-wise FP8 only supports backend='auto', "
193
+ f"but found backend={self.backend!r}."
194
+ )
195
+
196
+ def _validate_mxfp_quantization(self) -> None:
197
+ assert self.weight_bits in {4, self.MXFP8_BITS}, (
198
+ "INC MXFP only supports bits=4 or bits=8, "
199
+ f"but found bits={self.weight_bits}."
200
+ )
201
+ assert self.group_size == self.MXFP8_GROUP_SIZE, (
202
+ "INC MXFP only supports group_size=32, "
203
+ f"but found group_size={self.group_size}."
204
+ )
205
+ assert self.sym, "INC MXFP only supports symmetric weights."
206
+ assert self.packing_format == self.MXFP8_PACKING_FORMAT, (
207
+ "INC MXFP only supports "
208
+ f"packing_format={self.MXFP8_PACKING_FORMAT!r}, "
209
+ f"but found {self.packing_format!r}."
210
+ )
211
+ assert self.backend == "auto", (
212
+ "INC MXFP only supports backend='auto', "
213
+ f"but found backend={self.backend!r}."
214
+ )
215
+
216
+ def _validate_supported_quantization(self) -> None:
217
+ if self.data_type == "int":
218
+ self._validate_int_quantization()
219
+ elif self.data_type == self.FP8_BLOCK_DATA_TYPE:
220
+ self._validate_fp8_block_quantization()
221
+ elif self.data_type == self.MXFP8_DATA_TYPE:
222
+ self._validate_mxfp_quantization()
223
+
224
+ def _validate_raw_config(self, config: dict[str, Any]) -> None:
225
+ if self.data_type == self.FP8_BLOCK_DATA_TYPE:
226
+ assert isinstance(self.group_size, tuple), (
227
+ "INC block-wise FP8 group_size must be a tuple."
228
+ )
229
+
230
+ expected_fields = {
231
+ "act_bits": self.FP8_BLOCK_BITS,
232
+ "act_group_size": self.group_size[0],
233
+ "act_data_type": self.FP8_BLOCK_DATA_TYPE,
234
+ "act_sym": True,
235
+ "act_dynamic": True,
236
+ "activation_scheme": self.FP8_BLOCK_ACTIVATION_SCHEME,
237
+ "fmt": self.FP8_BLOCK_FMT,
238
+ }
239
+ self._check_expected(
240
+ config,
241
+ expected_fields,
242
+ "INC block-wise FP8",
243
+ )
244
+ elif self.is_mxfp8:
245
+ expected_fields = {
246
+ "act_bits": self.MXFP8_BITS,
247
+ "act_data_type": self.MXFP8_DATA_TYPE,
248
+ "act_group_size": self.MXFP8_GROUP_SIZE,
249
+ "act_sym": True,
250
+ "act_dynamic": True,
251
+ }
252
+ self._check_expected(
253
+ config,
254
+ expected_fields,
255
+ "INC MXFP8",
256
+ )
257
+
258
@classmethod
259
def get_name(cls) -> QuantizationMethods:
260
return "inc"
@@ -184,18 +273,37 @@ class INCConfig(QuantizationConfig):
273
274
@classmethod
275
def from_config(cls, config: dict[str, Any]) -> "INCConfig":
276
+ data_type = cls._normalize_data_type(
277
+ cls.get_from_keys_or(config, ["data_type"], "int")
278
+ )
279
+ weight_bits = cls.get_from_keys(config, ["bits"])
280
+ group_size = INCConfigParser._normalize_group_size(
281
+ cls.get_from_keys(config, ["group_size"])
282
+ )
283
+ packing_format = cls.get_from_keys_or(config, ["packing_format"], None)
284
+
285
+ if packing_format is None:
286
+ if (
287
+ data_type == cls.FP8_BLOCK_DATA_TYPE
288
+ and weight_bits == cls.FP8_BLOCK_BITS
289
+ and isinstance(group_size, tuple)
290
+ ):
291
+ packing_format = cls.FP8_BLOCK_PACKING_FORMAT
292
+ elif "mx_fp" in data_type:
293
+ packing_format = cls.MXFP8_PACKING_FORMAT
294
+ else:
295
+ packing_format = cls.DEFAULT_INT_PACKING_FORMAT
296
+
297
quant_config = cls(
188
- weight_bits=cls.get_from_keys(config, ["bits"]),
189
- group_size=cls.get_from_keys(config, ["group_size"]),
190
- sym=cls.get_from_keys(config, ["sym"]),
191
- packing_format=cls.get_from_keys_or(
192
- config, ["packing_format"], "auto_round:auto_gptq"
193
- ),
298
+ weight_bits=weight_bits,
299
+ group_size=group_size,
300
+ sym=cls.get_from_keys_or(config, ["sym"], True),
301
+ packing_format=packing_format,
302
block_name_to_quantize=cls.get_from_keys_or(
303
config, ["block_name_to_quantize", "to_quant_block_names"], None
304
),
305
extra_config=cls.get_from_keys_or(config, ["extra_config"], None),
198
- data_type=cls.get_from_keys_or(config, ["data_type"], "int"),
306
+ data_type=data_type,
307
backend=cls.get_from_keys_or(config, ["backend", "vllm_backend"], "auto"),
308
)
309
quant_config._validate_raw_config(config)
@@ -215,7 +323,6 @@ class INCConfig(QuantizationConfig):
323
def get_quant_method(self, layer: torch.nn.Module, prefix: str):
324
from .schemes.factory import resolve_scheme
325
218
- # Match original: check model.-prefixed names for unquantized layers
326
if prefix and self.extra_config:
327
for layer_name in self.extra_config:
328
if (
vllm/model_executor/layers/quantization/inc/schemes/__init__.py
+2
@@ -2,6 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
from .factory import resolve_scheme
5
+from .inc_fp8_scheme import INCFp8Scheme
6
from .inc_mxfp4_scheme import INCMxfp4Scheme
7
from .inc_mxfp8_scheme import INCMxfp8Scheme
8
from .inc_scheme import INCLinearScheme, INCScheme
@@ -11,6 +12,7 @@ __all__ = [
12
"INCScheme",
13
"INCLinearScheme",
14
"INCMxfp8Scheme",
15
+ "INCFp8Scheme",
16
"INCWna16Scheme",
17
"INCMxfp4Scheme",
18
"resolve_scheme",
vllm/model_executor/layers/quantization/inc/schemes/factory.py
+2
@@ -9,11 +9,13 @@ if TYPE_CHECKING:
9
10
11
def resolve_scheme(layer_config: "INCLayerConfig") -> "INCScheme":
12
+ from .inc_fp8_scheme import INCFp8Scheme
13
from .inc_mxfp4_scheme import INCMxfp4Scheme
14
from .inc_mxfp8_scheme import INCMxfp8Scheme
15
from .inc_wna16_scheme import INCWna16Scheme
16
17
scheme_list: list[type[INCScheme]] = [
18
+ INCFp8Scheme,
19
INCMxfp8Scheme,
20
INCWna16Scheme,
21
INCMxfp4Scheme,
vllm/model_executor/layers/quantization/inc/schemes/inc_fp8_linear.py
new
+57
@@ -0,0 +1,57 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import torch
5
+
6
+from vllm.model_executor.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod
7
+from vllm.model_executor.layers.quantization.utils.marlin_utils import (
8
+ get_marlin_input_dtype,
9
+)
10
+
11
+from .inc_scheme import INCLinearScheme
12
+
13
+
14
+class INCFp8LinearScheme(INCLinearScheme):
15
+ def __init__(self, prefix: str, weight_block_size: tuple[int, int]) -> None:
16
+ self.quant_config = Fp8Config(
17
+ is_checkpoint_fp8_serialized=True,
18
+ activation_scheme="dynamic",
19
+ weight_block_size=list(weight_block_size),
20
+ )
21
+ self.linear_method = Fp8LinearMethod(self.quant_config)
22
+ self.linear_method.marlin_input_dtype = get_marlin_input_dtype(prefix)
23
+
24
+ @classmethod
25
+ def get_min_capability(cls) -> int:
26
+ return Fp8Config.get_min_capability()
27
+
28
+ def create_weights(
29
+ self,
30
+ layer: torch.nn.Module,
31
+ input_size_per_partition: int,
32
+ output_partition_sizes: list[int],
33
+ input_size: int,
34
+ output_size: int,
35
+ params_dtype: torch.dtype,
36
+ **extra_weight_attrs,
37
+ ) -> None:
38
+ self.linear_method.create_weights(
39
+ layer=layer,
40
+ input_size_per_partition=input_size_per_partition,
41
+ output_partition_sizes=output_partition_sizes,
42
+ input_size=input_size,
43
+ output_size=output_size,
44
+ params_dtype=params_dtype,
45
+ **extra_weight_attrs,
46
+ )
47
+
48
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
49
+ self.linear_method.process_weights_after_loading(layer)
50
+
51
+ def apply_weights(
52
+ self,
53
+ layer: torch.nn.Module,
54
+ x: torch.Tensor,
55
+ bias: torch.Tensor | None = None,
56
+ ) -> torch.Tensor:
57
+ return self.linear_method.apply(layer, x, bias)
vllm/model_executor/layers/quantization/inc/schemes/inc_fp8_scheme.py
new
+39
@@ -0,0 +1,39 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from typing import TYPE_CHECKING
5
+
6
+from ..inc_linear import INCLinearMethod
7
+from .inc_scheme import INCScheme
8
+
9
+if TYPE_CHECKING:
10
+ import torch
11
+
12
+ from ..config_parser import INCLayerConfig
13
+ from ..inc import INCConfig
14
+
15
+
16
+class INCFp8Scheme(INCScheme):
17
+ @staticmethod
18
+ def can_handle(layer_config: "INCLayerConfig") -> bool:
19
+ return layer_config.is_fp8_block
20
+
21
+ def get_linear_method(
22
+ self,
23
+ config: "INCConfig",
24
+ layer: "torch.nn.Module",
25
+ prefix: str,
26
+ layer_config: "INCLayerConfig",
27
+ ):
28
+ del config, layer
29
+
30
+ assert isinstance(layer_config.group_size, tuple)
31
+
32
+ from .inc_fp8_linear import INCFp8LinearScheme
33
+
34
+ return INCLinearMethod(
35
+ INCFp8LinearScheme(
36
+ prefix=prefix,
37
+ weight_block_size=layer_config.group_size,
38
+ )
39
+ )
vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp4_linear.py
+5
@@ -29,6 +29,11 @@ class INCMxfp4LinearMethod(INCLinearScheme):
29
"""
30
31
def __init__(self, layer_config: "INCLayerConfig") -> None:
32
+ if not isinstance(layer_config.group_size, int):
33
+ raise ValueError(
34
+ "INC MXFP4 requires scalar group_size, "
35
+ f"but found group_size={layer_config.group_size!r}."
36
+ )
37
self.group_size = layer_config.group_size or 32
38
self.kernel = init_mxfp4_linear_kernel(activation_quant_key=kMxfp4Dynamic)
39
vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_linear.py
+15
-4
@@ -27,7 +27,7 @@ if TYPE_CHECKING:
27
class INCWNA16LinearScheme(INCLinearScheme):
28
def __init__(self, layer_config: "INCLayerConfig") -> None:
29
self.layer_config = layer_config
30
- self.inner_method = self._build_inner_method()
30
+ self.linear_method = self._build_inner_method()
31
32
@classmethod
33
def get_min_capability(cls) -> int:
@@ -43,6 +43,10 @@ class INCWNA16LinearScheme(INCLinearScheme):
43
)
44
45
def _build_gptq_method(self):
46
+ assert isinstance(self.layer_config.group_size, int), (
47
+ "WNA16 only supports integer group_size."
48
+ )
49
+
50
gptq_type_map = {
51
(4, True): scalar_types.uint4b8,
52
(8, True): scalar_types.uint8b128,
@@ -82,6 +86,10 @@ class INCWNA16LinearScheme(INCLinearScheme):
86
)
87
88
def _build_awq_method(self):
89
+ assert isinstance(self.layer_config.group_size, int), (
90
+ "WNA16 only supports integer group_size."
91
+ )
92
+
93
awq_type_map = {
94
4: scalar_types.uint4,
95
8: scalar_types.uint8,
@@ -135,7 +143,7 @@ class INCWNA16LinearScheme(INCLinearScheme):
143
params_dtype: "torch.dtype",
144
**extra_weight_attrs,
145
) -> None:
138
- return self.inner_method.create_weights(
146
+ return self.linear_method.create_weights(
147
layer=layer,
148
input_size_per_partition=input_size_per_partition,
149
output_partition_sizes=output_partition_sizes,
@@ -146,7 +154,7 @@ class INCWNA16LinearScheme(INCLinearScheme):
154
)
155
156
def process_weights_after_loading(self, layer: "torch.nn.Module") -> None:
149
- return self.inner_method.process_weights_after_loading(layer)
157
+ return self.linear_method.process_weights_after_loading(layer)
158
159
def apply_weights(
160
self,
@@ -154,7 +162,7 @@ class INCWNA16LinearScheme(INCLinearScheme):
162
x: "torch.Tensor",
163
bias: "torch.Tensor | None" = None,
164
) -> "torch.Tensor":
157
- return self.inner_method.apply(layer, x, bias)
165
+ return self.linear_method.apply(layer, x, bias)
166
167
168
class INCXPULinearBase(INCLinearScheme):
@@ -165,6 +173,9 @@ class INCXPULinearBase(INCLinearScheme):
173
174
def __init__(self, layer_config: "INCLayerConfig") -> None:
175
self.weight_bits = layer_config.bits
176
+ assert isinstance(layer_config.group_size, int), (
177
+ "INCXPULinearBase requires integer group_size."
178
+ )
179
self.group_size = layer_config.group_size
180
181
self.sym = layer_config.sym
vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_scheme.py
+11
@@ -41,6 +41,9 @@ def _check_xpu_w4a8_supported(layer_config: "INCLayerConfig", prefix: str) -> No
41
"which this build of vllm-xpu-kernels does not provide. "
42
f"Layer: {prefix}."
43
)
44
+ assert isinstance(layer_config.group_size, int), (
45
+ "WNA16 only supports integer group_size."
46
+ )
47
if layer_config.group_size <= 0 or layer_config.group_size % 32 != 0:
48
raise NotImplementedError(
49
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires a group size that is a "
@@ -168,6 +171,10 @@ def _resolve_gptq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
171
check_moe_marlin_supports_layer,
172
)
173
174
+ assert isinstance(layer_config.group_size, int), (
175
+ "WNA16 only supports integer group_size."
176
+ )
177
+
178
# AutoGPTQMoEMethod selects its fused-MoE backend through the WNA16 oracle
179
# (Marlin on CUDA, XPUExpertsWNA16 on XPU). Gate only on the layer-shape
180
# check like compressed-tensors does; the capability-based
@@ -213,6 +220,10 @@ def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
220
check_moe_marlin_supports_layer,
221
)
222
223
+ assert isinstance(layer_config.group_size, int), (
224
+ "WNA16 only supports integer group_size."
225
+ )
226
+
227
use_marlin = layer_config.bits in (4, 8) and check_moe_marlin_supports_layer(
228
layer, layer_config.group_size
229
)