@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a0dce9383a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Misc] Add compressed-tensors to optimized quant list (#7006)

Michael Goin committed Jul 31, 2024 at 17:40 UTC a0dce9383ab7de0015060fb9fedadeb7d8ffdfb9
1 file changed +6 -4
vllm/config.py
+6 -4
@@ -197,13 +197,17 @@ class ModelConfig:
197 def _parse_quant_hf_config(self):
198 quant_cfg = getattr(self.hf_config, "quantization_config", None)
199 if quant_cfg is None:
200 - # compress-tensors uses a "compression_config" key
200 + # compressed-tensors uses a "compression_config" key
201 quant_cfg = getattr(self.hf_config, "compression_config", None)
202 return quant_cfg
203
204 def _verify_quantization(self) -> None:
205 supported_quantization = [*QUANTIZATION_METHODS]
206 rocm_supported_quantization = ["gptq", "squeezellm"]
207 + optimized_quantization_methods = [
208 + "fp8", "marlin", "gptq_marlin_24", "gptq_marlin", "awq_marlin",
209 + "fbgemm_fp8", "compressed_tensors", "compressed-tensors"
210 + ]
211 if self.quantization is not None:
212 self.quantization = self.quantization.lower()
213
@@ -242,9 +246,7 @@ class ModelConfig:
246 raise ValueError(
247 f"{self.quantization} quantization is currently not "
248 f"supported in ROCm.")
245 - if (self.quantization
246 - not in ("fp8", "marlin", "gptq_marlin_24", "gptq_marlin",
247 - "awq_marlin", "fbgemm_fp8", "compressed_tensors")):
249 + if self.quantization not in optimized_quantization_methods:
250 logger.warning(
251 "%s quantization is not fully "
252 "optimized yet. The speed can be slower than "