A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 03:30 UTC
[Quantization][Autoround][CPU] Add W4A16 Support (#38192)
Signed-off-by: Zhenzhong1 <zhenzhong.xu@intel.com> Signed-off-by: Zhenzhong Xu <zhenzhong.xu@intel.com>
Zhenzhong Xu committed
Apr 15, 2026 at 18:38 UTC
60995c05b4ca3a26b92dfa7abed8f5db850301cc
2 files changed
+34
-1
tests/quantization/test_cpu_wna16.py
+1
@@ -12,6 +12,7 @@ MODELS = [
12
"TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", # with g_idx
13
"Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4", # without g_idx
14
"RedHatAI/Qwen3-1.7B-quantized.w4a16", # with zp
15
+ "OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc",
16
]
17
DTYPE = ["bfloat16"]
18
vllm/model_executor/layers/quantization/inc.py
+33
-1
@@ -414,6 +414,7 @@ class INCConfig(QuantizationConfig):
414
415
def apply_xpu_w4a16_quant_layer(self, layer, prefix: str):
416
weight_bits, group_size, sym = self.get_layer_config(layer, prefix)
417
+
418
if not self.check_quantized(weight_bits):
419
if isinstance(layer, (LinearBase, ParallelLMHead)):
420
return UnquantizedLinearMethod()
@@ -437,6 +438,27 @@ class INCConfig(QuantizationConfig):
438
)
439
return None
440
441
+ def apply_cpu_w4a16_quant_layer(self, layer, prefix: str):
442
+ weight_bits, group_size, sym = self.get_layer_config(layer, prefix)
443
+ if not self.check_quantized(weight_bits):
444
+ if isinstance(layer, (LinearBase, ParallelLMHead)):
445
+ return UnquantizedLinearMethod()
446
+ else:
447
+ return None
448
+
449
+ if weight_bits != 4:
450
+ raise NotImplementedError(
451
+ f"INC on CPU only supports 4-bit quantization, "
452
+ f"got weight_bits={weight_bits}."
453
+ )
454
+ if not sym:
455
+ raise NotImplementedError(
456
+ "INC W4A16 on CPU only supports symmetric quantization for now."
457
+ )
458
+ if isinstance(layer, (LinearBase, ParallelLMHead)):
459
+ return self.apply_gptq_quant_layer(layer, prefix)
460
+ return None
461
+
462
def get_quant_method(self, layer: torch.nn.Module, prefix: str):
463
if prefix and self.extra_config:
464
for layer_name in self.extra_config:
@@ -446,11 +468,21 @@ class INCConfig(QuantizationConfig):
468
return UnquantizedLinearMethod()
469
if current_platform.is_xpu():
470
return self.apply_xpu_w4a16_quant_layer(layer, prefix)
449
- if "gptq" in self.packing_format or "gptq" in self.backend:
471
+ is_gptq = "gptq" in self.packing_format or "gptq" in self.backend
472
+ if current_platform.is_cpu() and is_gptq:
473
+ return self.apply_cpu_w4a16_quant_layer(layer, prefix)
474
+ if is_gptq:
475
return self.apply_gptq_quant_layer(layer, prefix)
476
if "awq" in self.packing_format or "awq" in self.backend:
477
return self.apply_awq_quant_layer(layer, prefix)
478
479
+ raise NotImplementedError(
480
+ f"Unsupported quantization configuration for layer '{prefix}'. "
481
+ f"Platform: CPU={current_platform.is_cpu()}. "
482
+ f"Platform: XPU={current_platform.is_xpu()}. "
483
+ f"Format: {self.packing_format}, Backend: {self.backend}."
484
+ )
485
+
486
@classmethod
487
def override_quantization_method(
488
cls, hf_quant_cfg, user_quant, hf_config=None