A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
Mistral-format support for compressed-tensors (#16803)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Apr 23, 2025 at 06:46 UTC
aa72d9a4ea6b31a845bf4fbd5a97d3175a8c329a
1 file changed
+6
vllm/transformers_utils/config.py
+6
@@ -690,6 +690,9 @@ def load_params_config(model: Union[str, Path], revision: Optional[str],
690
"quant_method": "fp8",
691
"activation_scheme": "static"
692
}
693
+ elif quantization.get("quant_method") == "compressed-tensors":
694
+ # Pass through the quantization config to compressed-tensors
695
+ quantization_config = quantization
696
else:
697
raise ValueError(
698
f"Found unknown quantization='{quantization}' in config")
@@ -707,6 +710,7 @@ def load_params_config(model: Union[str, Path], revision: Optional[str],
710
711
if config_type == "multimodal":
712
multimodal_config = config_dict.pop("vision_encoder")
713
+ quantization_config = config_dict.get("quantization_config", {})
714
715
config_dict = {
716
"text_config": config_dict,
@@ -714,6 +718,8 @@ def load_params_config(model: Union[str, Path], revision: Optional[str],
718
}
719
config_dict["architectures"] = ["PixtralForConditionalGeneration"]
720
config_dict["model_type"] = "pixtral"
721
+ if quantization_config:
722
+ config_dict["quantization_config"] = quantization_config
723
724
config_dict.update(kwargs)
725