@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8f0a9ca890
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Bugfix] Respect modules_to_not_convert within awq_marlin (#9895)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Nov 4, 2024 at 18:57 UTC 8f0a9ca890a125f2b0fef49ba042ecf5b37830a8
1 file changed +24 -11
vllm/model_executor/layers/quantization/awq_marlin.py
+24 -11
@@ -9,7 +9,9 @@ from vllm.logger import init_logger
9 from vllm.model_executor.layers.fused_moe.layer import (
10 FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
11 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
12 + UnquantizedLinearMethod,
13 set_weight_attrs)
14 +from vllm.model_executor.layers.quantization.awq import is_layer_skipped_awq
15 from vllm.model_executor.layers.quantization.base_config import (
16 QuantizationConfig, QuantizeMethodBase)
17 from vllm.model_executor.layers.quantization.utils import replace_parameter
@@ -36,13 +38,18 @@ class AWQMarlinConfig(QuantizationConfig):
38 8: scalar_types.uint8,
39 }
40
39 - def __init__(self, weight_bits: int, group_size: int, has_zp: bool,
40 - lm_head_quantized: bool) -> None:
41 + def __init__(self,
42 + weight_bits: int,
43 + group_size: int,
44 + zero_point: bool,
45 + lm_head_quantized: bool,
46 + modules_to_not_convert: Optional[List[str]] = None) -> None:
47 self.pack_factor = 32 // weight_bits # packed into int32
48 self.group_size = group_size
43 - self.has_zp = has_zp
49 + self.zero_point = zero_point
50 self.lm_head_quantized = lm_head_quantized
51 self.weight_bits = weight_bits
52 + self.modules_to_not_convert = modules_to_not_convert or []
53
54 if self.weight_bits not in self.TYPE_MAP:
55 raise ValueError(f"Unsupported num_bits = {self.weight_bits}. "
@@ -52,13 +59,14 @@ class AWQMarlinConfig(QuantizationConfig):
59
60 verify_marlin_supported(self.quant_type,
61 group_size=self.group_size,
55 - has_zp=self.has_zp)
62 + has_zp=self.zero_point)
63
64 def __repr__(self) -> str:
65 return (f"AWQMarlinConfig(quant_type={self.quant_type}, "
66 f"group_size={self.group_size}, "
60 - f"has_zp={self.has_zp}, "
61 - f"lm_head_quantized={self.lm_head_quantized})")
67 + f"zero_point={self.zero_point}, "
68 + f"lm_head_quantized={self.lm_head_quantized}, "
69 + f"modules_to_not_convert={self.modules_to_not_convert})")
70
71 @classmethod
72 def get_name(cls) -> str:
@@ -80,10 +88,13 @@ class AWQMarlinConfig(QuantizationConfig):
88 def from_config(cls, config: Dict[str, Any]) -> "AWQMarlinConfig":
89 weight_bits = cls.get_from_keys(config, ["bits"])
90 group_size = cls.get_from_keys(config, ["group_size"])
83 - has_zp = cls.get_from_keys(config, ["zero_point"])
91 + zero_point = cls.get_from_keys(config, ["zero_point"])
92 lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
93 default=False)
86 - return cls(weight_bits, group_size, has_zp, lm_head_quantized)
94 + modules_to_not_convert = cls.get_from_keys_or(
95 + config, ["modules_to_not_convert"], None)
96 + return cls(weight_bits, group_size, zero_point, lm_head_quantized,
97 + modules_to_not_convert)
98
99 @classmethod
100 def override_quantization_method(cls, hf_quant_cfg,
@@ -109,6 +120,8 @@ class AWQMarlinConfig(QuantizationConfig):
120 prefix: str) -> Optional["QuantizeMethodBase"]:
121 if (isinstance(layer, LinearBase) or
122 (isinstance(layer, ParallelLMHead) and self.lm_head_quantized)):
123 + if is_layer_skipped_awq(prefix, self.modules_to_not_convert):
124 + return UnquantizedLinearMethod()
125 return AWQMarlinLinearMethod(self)
126 elif isinstance(layer, FusedMoE):
127 return AWQMoEMethod(self)
@@ -123,7 +136,7 @@ class AWQMarlinConfig(QuantizationConfig):
136 quant_method = quant_config.get("quant_method", "").lower()
137 num_bits = quant_config.get("bits")
138 group_size = quant_config.get("group_size")
126 - has_zp = quant_config.get("zero_point")
139 + zero_point = quant_config.get("zero_point")
140
141 if not current_platform.is_cuda():
142 return False
@@ -132,7 +145,7 @@ class AWQMarlinConfig(QuantizationConfig):
145 return False
146
147 # If we cannot find the info needed in the config, cannot convert.
135 - if (num_bits is None or group_size is None or has_zp is None):
148 + if (num_bits is None or group_size is None or zero_point is None):
149 return False
150
151 if num_bits not in cls.TYPE_MAP:
@@ -140,7 +153,7 @@ class AWQMarlinConfig(QuantizationConfig):
153
154 return check_marlin_supported(quant_type=cls.TYPE_MAP[num_bits],
155 group_size=group_size,
143 - has_zp=has_zp)
156 + has_zp=zero_point)
157
158
159 class AWQMarlinLinearMethod(LinearMethodBase):