A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Misc] Add compressed-tensors to optimized quant list (#7006)
Michael Goin committed
Jul 31, 2024 at 17:40 UTC
a0dce9383ab7de0015060fb9fedadeb7d8ffdfb9
1 file changed
+6
-4
vllm/config.py
+6
-4
@@ -197,13 +197,17 @@ class ModelConfig:
197
def _parse_quant_hf_config(self):
198
quant_cfg = getattr(self.hf_config, "quantization_config", None)
199
if quant_cfg is None:
200
- # compress-tensors uses a "compression_config" key
200
+ # compressed-tensors uses a "compression_config" key
201
quant_cfg = getattr(self.hf_config, "compression_config", None)
202
return quant_cfg
203
204
def _verify_quantization(self) -> None:
205
supported_quantization = [*QUANTIZATION_METHODS]
206
rocm_supported_quantization = ["gptq", "squeezellm"]
207
+ optimized_quantization_methods = [
208
+ "fp8", "marlin", "gptq_marlin_24", "gptq_marlin", "awq_marlin",
209
+ "fbgemm_fp8", "compressed_tensors", "compressed-tensors"
210
+ ]
211
if self.quantization is not None:
212
self.quantization = self.quantization.lower()
213
@@ -242,9 +246,7 @@ class ModelConfig:
246
raise ValueError(
247
f"{self.quantization} quantization is currently not "
248
f"supported in ROCm.")
245
- if (self.quantization
246
- not in ("fp8", "marlin", "gptq_marlin_24", "gptq_marlin",
247
- "awq_marlin", "fbgemm_fp8", "compressed_tensors")):
249
+ if self.quantization not in optimized_quantization_methods:
250
logger.warning(
251
"%s quantization is not fully "
252
"optimized yet. The speed can be slower than "