@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b167109ba1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Fix] Fix quantization="gptq" when using Marlin (#3319)

Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Bo-Wen Wang committed Mar 13, 2024 at 13:51 UTC b167109ba12f18d028d2be8a61d3dce950eb2724
1 file changed +6 -1
vllm/config.py
+6 -1
@@ -168,13 +168,18 @@ class ModelConfig:
168 # Parse quantization method from the HF model config, if available.
169 hf_quant_config = getattr(self.hf_config, "quantization_config", None)
170 if hf_quant_config is not None:
171 -
171 hf_quant_method = str(hf_quant_config["quant_method"]).lower()
172 +
173 # If the GPTQ model is serialized in marlin format, use marlin.
174 if (hf_quant_method == "gptq"
175 and "is_marlin_format" in hf_quant_config
176 and hf_quant_config["is_marlin_format"]):
177 + logger.info("The model is serialized in Marlin format. "
178 + "Using Marlin kernel.")
179 hf_quant_method = "marlin"
180 + if self.quantization == "gptq":
181 + self.quantization = hf_quant_method
182 +
183 if self.quantization is None:
184 self.quantization = hf_quant_method
185 elif self.quantization != hf_quant_method: