@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4896d0c2dd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Quant] Fix use_mla TypeError and support loading pure-sparsity Compressed Tensors configs (#12711)

Kyle Sayers committed Feb 4, 2025 at 02:27 UTC 4896d0c2dd367efbdf8387028322bf5f74359930
2 files changed +8 -2
vllm/config.py
+3 -2
@@ -1000,8 +1000,9 @@ class ModelConfig:
1000 # have fp8 for both weights and activations.
1001 if self.quantization == "compressed-tensors":
1002 quant_config = self._parse_quant_hf_config()
1003 - for group_name, cfg in quant_config.get("config_groups",
1004 - ("", {})).items():
1003 + for group_name, cfg in quant_config.get("config_groups", {
1004 + "": {}
1005 + }).items():
1006 act_cfg = cfg.get("input_activations", {})
1007 act_type = None if act_cfg is None else act_cfg.get("type", "")
1008 w_cfg = cfg.get("weights", {})
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+5
@@ -424,6 +424,11 @@ class CompressedTensorsConfig(QuantizationConfig):
424 or input_quant is not None,
425 weight_quant=weight_quant,
426 input_quant=input_quant)
427 + elif weight_quant is None:
428 + logger.warning_once("Acceleration for non-quantized schemes is "
429 + "not supported by Compressed Tensors. "
430 + "Falling back to UnquantizedLinearMethod")
431 + return None
432 else:
433 # Find the quant_scheme
434 scheme = self._get_scheme_from_parts( # type: ignore