A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Fix] Fix quantization="gptq" when using Marlin (#3319)
Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Bo-Wen Wang committed
Mar 13, 2024 at 13:51 UTC
b167109ba12f18d028d2be8a61d3dce950eb2724
1 file changed
+6
-1
vllm/config.py
+6
-1
@@ -168,13 +168,18 @@ class ModelConfig:
168
# Parse quantization method from the HF model config, if available.
169
hf_quant_config = getattr(self.hf_config, "quantization_config", None)
170
if hf_quant_config is not None:
171
-
171
hf_quant_method = str(hf_quant_config["quant_method"]).lower()
172
+
173
# If the GPTQ model is serialized in marlin format, use marlin.
174
if (hf_quant_method == "gptq"
175
and "is_marlin_format" in hf_quant_config
176
and hf_quant_config["is_marlin_format"]):
177
+ logger.info("The model is serialized in Marlin format. "
178
+ "Using Marlin kernel.")
179
hf_quant_method = "marlin"
180
+ if self.quantization == "gptq":
181
+ self.quantization = hf_quant_method
182
+
183
if self.quantization is None:
184
self.quantization = hf_quant_method
185
elif self.quantization != hf_quant_method: