A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
[Bugfix] Fix transformers model impl ignored for mixtral quant (#18602)
Signed-off-by: Tristan Leclercq <tristanleclercq@gmail.com>
Tristan Leclercq committed
May 23, 2025 at 14:54 UTC
6220f3c6b0a1f96bc494a8f927ff490e8b12dde0
1 file changed
+4
-5
vllm/model_executor/model_loader/utils.py
+4
-5
@@ -225,17 +225,16 @@ def get_model_architecture(
225
"fp8", "compressed-tensors", "gptq_marlin", "awq_marlin", "quark"
226
]
227
228
- if (model_config.quantization is not None
229
- and model_config.quantization not in mixtral_supported
230
- and "MixtralForCausalLM" in architectures):
231
- architectures = ["QuantMixtralForCausalLM"]
232
-
228
vllm_supported_archs = ModelRegistry.get_supported_archs()
229
vllm_not_supported = not any(arch in vllm_supported_archs
230
for arch in architectures)
231
if (model_config.model_impl == ModelImpl.TRANSFORMERS or
232
model_config.model_impl != ModelImpl.VLLM and vllm_not_supported):
233
architectures = resolve_transformers_arch(model_config, architectures)
234
+ elif (model_config.quantization is not None
235
+ and model_config.quantization not in mixtral_supported
236
+ and "MixtralForCausalLM" in architectures):
237
+ architectures = ["QuantMixtralForCausalLM"]
238
239
model_cls, arch = ModelRegistry.resolve_model_cls(architectures)
240
if model_config.task == "embed":