A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Feature][Quantization] auto_round support for mixed bits quantization (#23812)
Signed-off-by: n1ck-guo <heng.guo@intel.com> Signed-off-by: Heng Guo <heng.guo@intel.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Heng Guo committed
Oct 21, 2025 at 06:23 UTC
87778d5f00ab09f88d0b677c34ae7d33e6296ce2
1 file changed
+6
vllm/model_executor/layers/quantization/auto_round.py
+6
@@ -436,6 +436,12 @@ class AutoRoundConfig(QuantizationConfig):
436
return None
437
438
def get_quant_method(self, layer: torch.nn.Module, prefix: str):
439
+ if prefix and self.extra_config:
440
+ for layer_name in self.extra_config:
441
+ if (
442
+ layer_name == prefix or layer_name == f"model.{prefix}"
443
+ ) and self.extra_config[layer_name].get("bits", 16) >= 16:
444
+ return UnquantizedLinearMethod()
445
if (
446
current_platform.is_cpu()
447
or current_platform.is_xpu()