A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Quant] Fix use_mla TypeError and support loading pure-sparsity Compressed Tensors configs (#12711)
Kyle Sayers committed
Feb 4, 2025 at 02:27 UTC
4896d0c2dd367efbdf8387028322bf5f74359930
2 files changed
+8
-2
vllm/config.py
+3
-2
@@ -1000,8 +1000,9 @@ class ModelConfig:
1000
# have fp8 for both weights and activations.
1001
if self.quantization == "compressed-tensors":
1002
quant_config = self._parse_quant_hf_config()
1003
- for group_name, cfg in quant_config.get("config_groups",
1004
- ("", {})).items():
1003
+ for group_name, cfg in quant_config.get("config_groups", {
1004
+ "": {}
1005
+ }).items():
1006
act_cfg = cfg.get("input_activations", {})
1007
act_type = None if act_cfg is None else act_cfg.get("type", "")
1008
w_cfg = cfg.get("weights", {})
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+5
@@ -424,6 +424,11 @@ class CompressedTensorsConfig(QuantizationConfig):
424
or input_quant is not None,
425
weight_quant=weight_quant,
426
input_quant=input_quant)
427
+ elif weight_quant is None:
428
+ logger.warning_once("Acceleration for non-quantized schemes is "
429
+ "not supported by Compressed Tensors. "
430
+ "Falling back to UnquantizedLinearMethod")
431
+ return None
432
else:
433
# Find the quant_scheme
434
scheme = self._get_scheme_from_parts( # type: ignore