A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 10:00 UTC
Fix Mistral config remap to accept compressed-tensors quantization #34028 (#34104)
Signed-off-by: baonudesifeizhai <baonudesifeizhai@gmail.com>
baonudesifeizhai committed
Feb 12, 2026 at 03:22 UTC
f5897613fb270fb478cda362868713cc338f6be9
1 file changed
+8
vllm/transformers_utils/configs/mistral.py
+8
@@ -198,6 +198,14 @@ def _remap_mistral_quantization_args(config: dict) -> dict:
198
"quant_method": "fp8",
199
"activation_scheme": "dynamic" if is_dynamic else "static",
200
}
201
+ elif (
202
+ str(quantization.get("quant_method", "")).lower().replace("_", "-")
203
+ == "compressed-tensors"
204
+ ):
205
+ # Pass through compressed-tensors config, while normalizing
206
+ # quant_method to the canonical community spelling.
207
+ quantization["quant_method"] = "compressed-tensors"
208
+ config["quantization_config"] = quantization
209
else:
210
raise ValueError(f"Found unknown quantization='{quantization}' in config")
211