add 2/3/5/6/7 CUDA support in AutoRound format (#52890)
Signed-off-by: Wenhua Cheng <wenhua.cheng@intel.com>
Wenhua Cheng committed
Sep 7, 2026 at 09:59 UTC
294fbb4f595f61743124fc2e56a99e128cbe10da
3 files changed
+152
-2
tests/quantization/test_auto_round.py
+86
@@ -666,6 +666,92 @@ def test_inc_resolve_scheme_selects_wna16() -> None:
666
assert isinstance(scheme, INCWna16Scheme)
667
668
669
+@pytest.mark.parametrize("bits", [2, 3])
670
+@pytest.mark.parametrize(
671
+ "packing_format", ["auto_round:auto_gptq", "auto_round:auto_awq"]
672
+)
673
+def test_wna16_cuda_low_bit_linear_routes_to_humming(
674
+ monkeypatch, bits, packing_format
675
+) -> None:
676
+ expected_method = object()
677
+ captured = {}
678
+
679
+ monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
680
+ monkeypatch.setattr(current_platform, "is_xpu", lambda: False)
681
+ monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
682
+ monkeypatch.setattr(
683
+ "vllm.model_executor.layers.quantization.inc.schemes."
684
+ "inc_wna16_scheme._build_humming_linear_method",
685
+ lambda layer_config: captured.update({"layer_config": layer_config})
686
+ or expected_method,
687
+ )
688
+
689
+ layer_config = make_layer_config(bits=bits, packing_format=packing_format)
690
+ method = INCWna16Scheme().get_linear_method(
691
+ make_config(), object(), "model.layers.0.mlp.down_proj", layer_config
692
+ )
693
+
694
+ assert method is expected_method
695
+ assert captured["layer_config"] is layer_config
696
+
697
+
698
+@pytest.mark.parametrize("bits", [2, 3])
699
+def test_wna16_cuda_low_bit_moe_routes_to_humming(monkeypatch, bits) -> None:
700
+ expected_method = object()
701
+ captured = {}
702
+
703
+ class DummyMoeConfig:
704
+ pass
705
+
706
+ monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
707
+ monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
708
+ monkeypatch.setattr(
709
+ "vllm.model_executor.layers.quantization.inc.schemes."
710
+ "inc_wna16_scheme._build_humming_moe_method",
711
+ lambda layer, layer_config: captured.update(
712
+ {"layer": layer, "layer_config": layer_config}
713
+ )
714
+ or expected_method,
715
+ )
716
+
717
+ layer = object.__new__(RoutedExperts)
718
+ layer.moe_config = DummyMoeConfig()
719
+ layer_config = make_layer_config(bits=bits)
720
+ method = INCWna16Scheme().get_moe_method(
721
+ make_config(), layer, "model.layers.0.mlp", layer_config
722
+ )
723
+
724
+ assert method is expected_method
725
+ assert captured["layer"] is layer
726
+ assert captured["layer_config"] is layer_config
727
+
728
+
729
+@pytest.mark.parametrize("bits", [4, 8])
730
+def test_wna16_cuda_high_bit_skips_humming(monkeypatch, bits) -> None:
731
+ """4/8-bit int stays on the Marlin/GPTQ/AWQ path even on CUDA so a single
732
+ model can mix high-bit Marlin and low-bit humming layers."""
733
+ called = {"humming": False}
734
+
735
+ monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
736
+ monkeypatch.setattr(current_platform, "is_xpu", lambda: False)
737
+ monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
738
+ monkeypatch.setattr(
739
+ "vllm.model_executor.layers.quantization.inc.schemes."
740
+ "inc_wna16_scheme._build_humming_linear_method",
741
+ lambda layer_config: called.update({"humming": True}),
742
+ )
743
+
744
+ method = INCWna16Scheme().get_linear_method(
745
+ make_config(),
746
+ object(),
747
+ "model.layers.0.mlp.down_proj",
748
+ make_layer_config(bits=bits),
749
+ )
750
+
751
+ assert called["humming"] is False
752
+ assert isinstance(method, INCLinearMethod)
753
+
754
+
755
def test_inc_config_accepts_mxfp_family_llm_compressor() -> None:
756
config = INCConfig.from_config(
757
{
vllm/model_executor/layers/quantization/inc/inc.py
+1
-2
@@ -34,9 +34,8 @@ class INCConfig(QuantizationConfig):
34
Repo: https://github.com/intel/neural-compressor
35
"""
36
37
+ SUPPORTED_BITS = {2, 3, 4, 5, 6, 7, 8}
38
DEFAULT_INT_PACKING_FORMAT = "auto_round:auto_gptq"
38
-
39
- SUPPORTED_BITS = {2, 3, 4, 8}
39
SUPPORTED_DTYPES = {"int", "mx_fp", "fp"}
40
SUPPORTED_FORMATS = {
41
"auto_round:auto_gptq",
vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_scheme.py
+65
@@ -21,6 +21,11 @@ if TYPE_CHECKING:
21
logger = init_logger(__name__)
22
23
XPU_WNA16_SUPPORTED_BITS = {2, 4}
24
+# On CUDA the Marlin/GPTQ/AWQ kernels only cover 4/8-bit. The remaining
25
+# widths (2/3/5/6/7) have no dedicated CUDA kernel, so they are dispatched to
26
+# the humming kernel instead. This mirrors compressed-tensors WNA16, whose
27
+# 5/6/7-bit biased scalar types also fall back to humming at kernel selection.
28
+CUDA_HUMMING_SUPPORTED_BITS = {2, 3, 5, 6, 7}
29
30
# Backends selectable through VLLM_XPU_INC_WNA16_BACKEND that are served by the
31
# oneDNN int4 GEMMs rather than by ARK. These only cover int4.
@@ -133,6 +138,15 @@ class INCWna16Scheme(INCScheme):
138
return INCLinearMethod(INCWNA16LinearScheme(layer_config))
139
raise NotImplementedError(f"INC on CPU: unsupported config {layer_config}")
140
141
+ # CUDA low-bit (2/3/5/6/7): no Marlin/GPTQ/AWQ kernel, route to humming
142
+ # so a single model can mix 4/8-bit (Marlin) and 2/3/5/6/7-bit (humming)
143
+ # layers.
144
+ if (
145
+ current_platform.is_cuda()
146
+ and layer_config.bits in CUDA_HUMMING_SUPPORTED_BITS
147
+ ):
148
+ return _build_humming_linear_method(layer_config)
149
+
150
from .inc_wna16_linear import INCWNA16LinearScheme
151
152
return INCLinearMethod(INCWNA16LinearScheme(layer_config))
@@ -152,6 +166,12 @@ class INCWna16Scheme(INCScheme):
166
)
167
168
return UnquantizedFusedMoEMethod(layer.moe_config)
169
+ # CUDA low-bit (2/3/5/6/7): route to the humming MoE kernel (see above).
170
+ if (
171
+ current_platform.is_cuda()
172
+ and layer_config.bits in CUDA_HUMMING_SUPPORTED_BITS
173
+ ):
174
+ return _build_humming_moe_method(layer, layer_config)
175
if layer_config.is_gptq:
176
return _resolve_gptq_moe(layer, layer_config)
177
if layer_config.is_awq:
@@ -251,3 +271,48 @@ def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
271
}
272
)
273
return MoeWNA16Method(moe_config, layer.moe_config)
274
+
275
+
276
+def _humming_weight_config(layer_config: "INCLayerConfig") -> dict:
277
+ """Build the humming weight-schema config for a WNA16 int checkpoint."""
278
+ if layer_config.is_gptq:
279
+ return {
280
+ "quant_method": "gptq",
281
+ "bits": layer_config.bits,
282
+ "group_size": layer_config.group_size,
283
+ "desc_act": False,
284
+ "sym": layer_config.sym,
285
+ }
286
+ if layer_config.is_awq:
287
+ return {
288
+ "quant_method": "awq",
289
+ "bits": layer_config.bits,
290
+ "group_size": layer_config.group_size,
291
+ "zero_point": not layer_config.sym,
292
+ }
293
+ raise NotImplementedError(
294
+ "INC humming dispatch only supports gptq/awq packed int checkpoints, "
295
+ f"but found {layer_config}."
296
+ )
297
+
298
+
299
+def _build_humming_quant_config(layer_config: "INCLayerConfig"):
300
+ from vllm.model_executor.layers.quantization.humming import (
301
+ HummingLayerQuantizationConfig,
302
+ )
303
+ from vllm.utils.humming import BaseWeightSchema
304
+
305
+ weight_schema = BaseWeightSchema.from_config(_humming_weight_config(layer_config))
306
+ return HummingLayerQuantizationConfig(weight_schema=weight_schema)
307
+
308
+
309
+def _build_humming_linear_method(layer_config: "INCLayerConfig"):
310
+ from vllm.model_executor.layers.quantization.humming import HummingLinearMethod
311
+
312
+ return HummingLinearMethod(_build_humming_quant_config(layer_config))
313
+
314
+
315
+def _build_humming_moe_method(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
316
+ from vllm.model_executor.layers.quantization.humming import HummingMoEMethod
317
+
318
+ return HummingMoEMethod(_build_humming_quant_config(layer_config), layer.moe_config)