A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 04:45 UTC
[Bugfix][Model] Fix Devstral Small 2 HF format weight loading (#39293)
Signed-off-by: thomasmaindron <thomasmaindron@users.noreply.github.com> Co-authored-by: thomasmaindron <thomasmaindron@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Thomas committed
Apr 14, 2026 at 12:11 UTC
6f786f2c506cb07f4566771fdc62e640e2c4a176
3 files changed
+11
-7
tests/models/registry.py
+1
@@ -416,6 +416,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
416
"MiniMaxAI/MiniMax-M2",
417
trust_remote_code=True,
418
),
419
+ "Ministral3ForCausalLM": _HfExamplesInfo("mistralai/Ministral-3-3B-Instruct-2512"),
420
"MistralForCausalLM": _HfExamplesInfo("mistralai/Mistral-7B-Instruct-v0.1"),
421
"MistralLarge3ForCausalLM": _HfExamplesInfo(
422
"mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4"
vllm/model_executor/models/mistral3.py
+9
-7
@@ -382,7 +382,14 @@ class Mistral3ForConditionalGeneration(
382
# Some PEFT LoRAs are trained against the text submodule directly
383
# and produce names like `base_model.model.model.layers.*`.
384
"model.": "language_model.model.",
385
- }
385
+ },
386
+ orig_to_new_suffix={
387
+ # FP8 quantized HF checkpoints use "activation_scale" and
388
+ # "weight_scale_inv" but vLLM's FP8 linear layers register
389
+ # them as "input_scale" and "weight_scale"
390
+ ".activation_scale": ".input_scale",
391
+ ".weight_scale_inv": ".weight_scale",
392
+ },
393
)
394
395
@classmethod
@@ -402,13 +409,8 @@ class Mistral3ForConditionalGeneration(
409
self.config = config
410
self.multimodal_config = multimodal_config
411
405
- # NOTE: These are special cases for Pixtral-12B in the HF-format
412
+ # NOTE: This is a special case for Pixtral-12B in the HF-format
413
# https://huggingface.co/mistral-community/pixtral-12b/blob/main/config.json # noqa
407
- if (
408
- config.text_config.architectures is None
409
- and config.text_config.model_type == "mistral"
410
- ):
411
- config.text_config.architectures = ["MistralForCausalLM"]
414
if (
415
config.projector_hidden_act is None
416
and config.vision_config.hidden_act == "gelu"
vllm/model_executor/models/registry.py
+1
@@ -160,6 +160,7 @@ _TEXT_GENERATION_MODELS = {
160
"MiniMaxText01ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
161
"MiniMaxM1ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
162
"MiniMaxM2ForCausalLM": ("minimax_m2", "MiniMaxM2ForCausalLM"),
163
+ "Ministral3ForCausalLM": ("mistral", "MistralForCausalLM"),
164
"MistralForCausalLM": ("mistral", "MistralForCausalLM"),
165
"MistralLarge3ForCausalLM": ("mistral_large_3", "MistralLarge3ForCausalLM"),
166
"MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),