@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d8c6210eea
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 12, 06:00 UTC

Add Mistral Large 3 and Ministral 3 (#29757)

Signed-off-by: Julien Denize <julien.denize@mistral.ai> Signed-off-by: Julien Denize <40604584+juliendenize@users.noreply.github.com> Signed-off-by: Mickael Seznec <mickael@mistral.ai> Signed-off-by: Roger Wang <hey@rogerw.io> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Mickael Seznec <mickael@mistral.ai>

Julien Denize committed Dec 2, 2025 at 11:29 UTC d8c6210eeaa7f3b474e50cf74926f77a8dc79adf
16 files changed +724 -30
docs/models/supported_models.md
+3 -2
@@ -417,7 +417,8 @@ th {
417 | `MiniCPMForCausalLM` | MiniCPM | `openbmb/MiniCPM-2B-sft-bf16`, `openbmb/MiniCPM-2B-dpo-bf16`, `openbmb/MiniCPM-S-1B-sft`, etc. | ✅︎ | ✅︎ |
418 | `MiniCPM3ForCausalLM` | MiniCPM3 | `openbmb/MiniCPM3-4B`, etc. | ✅︎ | ✅︎ |
419 | `MiniMaxM2ForCausalLM` | MiniMax-M2 |`MiniMaxAI/MiniMax-M2`, etc. | | ✅︎ |
420 -| `MistralForCausalLM` | Mistral, Mistral-Instruct | `mistralai/Mistral-7B-v0.1`, `mistralai/Mistral-7B-Instruct-v0.1`, etc. | ✅︎ | ✅︎ |
420 +| `MistralForCausalLM` | Ministral-3, Mistral, Mistral-Instruct | `mistralai/Ministral-3-3B-Instruct-2512`, `mistralai/Mistral-7B-v0.1`, `mistralai/Mistral-7B-Instruct-v0.1`, etc. | ✅︎ | ✅︎ |
421 +| `MistralLarge3ForCausalLM` | Mistral-Large-3-675B-Base-2512, Mistral-Large-3-675B-Instruct-2512 | `mistralai/Mistral-Large-3-675B-Base-2512`, `mistralai/Mistral-Large-3-675B-Instruct-2512`, etc. | ✅︎ | ✅︎ |
422 | `MixtralForCausalLM` | Mixtral-8x7B, Mixtral-8x7B-Instruct | `mistralai/Mixtral-8x7B-v0.1`, `mistralai/Mixtral-8x7B-Instruct-v0.1`, `mistral-community/Mixtral-8x22B-v0.1`, etc. | ✅︎ | ✅︎ |
423 | `MPTForCausalLM` | MPT, MPT-Instruct, MPT-Chat, MPT-StoryWriter | `mosaicml/mpt-7b`, `mosaicml/mpt-7b-storywriter`, `mosaicml/mpt-30b`, etc. | | ✅︎ |
424 | `NemotronForCausalLM` | Nemotron-3, Nemotron-4, Minitron | `nvidia/Minitron-8B-Base`, `mgoin/Nemotron-4-340B-Base-hf-FP8`, etc. | ✅︎ | ✅︎ |
@@ -711,7 +712,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
712 | `Phi3VForCausalLM` | Phi-3-Vision, Phi-3.5-Vision | T + I<sup>E+</sup> | `microsoft/Phi-3-vision-128k-instruct`, `microsoft/Phi-3.5-vision-instruct`, etc. | | ✅︎ |
713 | `Phi4MMForCausalLM` | Phi-4-multimodal | T + I<sup>+</sup> / T + A<sup>+</sup> / I<sup>+</sup> + A<sup>+</sup> | `microsoft/Phi-4-multimodal-instruct`, etc. | ✅︎ | ✅︎ |
714 | `Phi4MultimodalForCausalLM` | Phi-4-multimodal (HF Transformers) | T + I<sup>+</sup> / T + A<sup>+</sup> / I<sup>+</sup> + A<sup>+</sup> | `microsoft/Phi-4-multimodal-instruct` (with revision `refs/pr/70`), etc. | ✅︎ | ✅︎ |
714 -| `PixtralForConditionalGeneration` | Mistral 3 (Mistral format), Pixtral (Mistral format) | T + I<sup>+</sup> | `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, `mistralai/Pixtral-12B-2409`, etc. | | ✅︎ |
715 +| `PixtralForConditionalGeneration` | Ministral 3 (Mistral format), Mistral 3 (Mistral format), Mistral Large 3 (Mistral format), Pixtral (Mistral format) | T + I<sup>+</sup> | `mistralai/Ministral-3-3B-Instruct-2512`, `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, `mistralai/Mistral-Large-3-675B-Instruct-2512` `mistralai/Pixtral-12B-2409` etc. | | ✅︎ |
716 | `QwenVLForConditionalGeneration`<sup>^</sup> | Qwen-VL | T + I<sup>E+</sup> | `Qwen/Qwen-VL`, `Qwen/Qwen-VL-Chat`, etc. | ✅︎ | ✅︎ |
717 | `Qwen2AudioForConditionalGeneration` | Qwen2-Audio | T + A<sup>+</sup> | `Qwen/Qwen2-Audio-7B-Instruct` | | ✅︎ |
718 | `Qwen2VLForConditionalGeneration` | QVQ, Qwen2-VL | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/QVQ-72B-Preview`, `Qwen/Qwen2-VL-7B-Instruct`, `Qwen/Qwen2-VL-72B-Instruct`, etc. | ✅︎ | ✅︎ |
tests/models/registry.py
+14
@@ -358,6 +358,9 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
358 trust_remote_code=True,
359 ),
360 "MistralForCausalLM": _HfExamplesInfo("mistralai/Mistral-7B-Instruct-v0.1"),
361 + "MistralLarge3ForCausalLM": _HfExamplesInfo(
362 + "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4", is_available_online=False
363 + ),
364 "MixtralForCausalLM": _HfExamplesInfo(
365 "mistralai/Mixtral-8x7B-Instruct-v0.1",
366 {"tiny": "TitanML/tiny-mixtral"},
@@ -770,7 +773,13 @@ _MULTIMODAL_EXAMPLE_MODELS = {
773 ),
774 "PixtralForConditionalGeneration": _HfExamplesInfo(
775 "mistralai/Pixtral-12B-2409",
776 + extras={
777 + "mistral-large-3": "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4",
778 + "ministral-3": "mistralai/Ministral-3-3B-Instruct-2512",
779 + },
780 tokenizer_mode="mistral",
781 + # TODO: revert once Mistral-Large-3 and Ministral-3 are publicly available.
782 + is_available_online=False,
783 ),
784 "QwenVLForConditionalGeneration": _HfExamplesInfo(
785 "Qwen/Qwen-VL",
@@ -870,6 +879,11 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
879 use_original_num_layers=True,
880 max_model_len=10240,
881 ),
882 + "EagleMistralLarge3ForCausalLM": _HfExamplesInfo(
883 + "mistralai/Mistral-Large-3-675B-Instruct-2512",
884 + speculative_model="mistralai/Mistral-Large-3-675B-Instruct-2512-Eagle",
885 + is_available_online=False,
886 + ),
887 "LlamaForCausalLMEagle3": _HfExamplesInfo(
888 "Qwen/Qwen3-8B",
889 trust_remote_code=True,
tests/tokenizers_/test_mistral.py
+151 -7
@@ -91,6 +91,118 @@ from vllm.tokenizers.mistral import (
91 ],
92 ),
93 ),
94 + (
95 + {
96 + "messages": [
97 + {
98 + "role": "user",
99 + "content": "What is the current local date and time?",
100 + }
101 + ],
102 + "tools": [
103 + {
104 + "type": "function",
105 + "function": {
106 + "description": "Fetch the current local date and time.",
107 + "unsupported_field": False,
108 + "name": "get_current_time",
109 + "parameters": {},
110 + },
111 + },
112 + {
113 + "type": "function",
114 + "function": {
115 + "description": "Fetch the current local date and time.",
116 + "unsupported_field2": False,
117 + "name": "get_current_time",
118 + "parameters": {},
119 + },
120 + },
121 + ],
122 + },
123 + (
124 + [
125 + {
126 + "role": "user",
127 + "content": "What is the current local date and time?",
128 + }
129 + ],
130 + [
131 + {
132 + "type": "function",
133 + "function": {
134 + "description": "Fetch the current local date and time.",
135 + "name": "get_current_time",
136 + "parameters": {},
137 + },
138 + },
139 + {
140 + "type": "function",
141 + "function": {
142 + "description": "Fetch the current local date and time.",
143 + "name": "get_current_time",
144 + "parameters": {},
145 + },
146 + },
147 + ],
148 + ),
149 + ),
150 + (
151 + {
152 + "messages": [
153 + {
154 + "role": "user",
155 + "content": "What is the current local date and time?",
156 + }
157 + ],
158 + "tools": [
159 + {
160 + "type": "function",
161 + "unsupported_field": False,
162 + "function": {
163 + "description": "Fetch the current local date and time.",
164 + "name": "get_current_time",
165 + "parameters": {},
166 + },
167 + },
168 + {
169 + "type": "function",
170 + "unsupported_field2": False,
171 + "function": {
172 + "description": "Fetch the current local date and time 2.",
173 + "name": "get_current_time2",
174 + "parameters": {"a": "1"},
175 + },
176 + },
177 + ],
178 + },
179 + (
180 + [
181 + {
182 + "role": "user",
183 + "content": "What is the current local date and time?",
184 + }
185 + ],
186 + [
187 + {
188 + "type": "function",
189 + "function": {
190 + "description": "Fetch the current local date and time.",
191 + "name": "get_current_time",
192 + "parameters": {},
193 + },
194 + },
195 + {
196 + "type": "function",
197 + "function": {
198 + "description": "Fetch the current local date and time 2.",
199 + "name": "get_current_time2",
200 + "parameters": {"a": "1"},
201 + },
202 + },
203 + ],
204 + ),
205 + ),
206 ],
207 )
208 def test_prepare_apply_chat_template_tools_and_messages(
@@ -1108,13 +1220,6 @@ class TestMistralTokenizer:
1220 )
1221 == expected_tokens[mistral_tokenizer.is_tekken]
1222 )
1111 - assert (
1112 - mistral_tokenizer.decode(
1113 - ids[mistral_tokenizer.is_tekken],
1114 - skip_special_tokens=skip_special_tokens,
1115 - )
1116 - == expected_tokens[mistral_tokenizer.is_tekken]
1117 - )
1223
1224 def test_decode_empty(
1225 self,
@@ -1140,6 +1245,45 @@ class TestMistralTokenizer:
1245 == "<s>"
1246 )
1247
1248 + @pytest.mark.parametrize(
1249 + "skip_special_tokens,expected_tokens",
1250 + (
1251 + (
1252 + False,
1253 + (
1254 + ["<s>[INST]▁Hello▁world▁![/INST]▁Hello</s>"],
1255 + ["<s>[INST]Hello world ![/INST]Hello</s>"],
1256 + ),
1257 + ),
1258 + (True, (["Hello world ! Hello"], ["Hello world !Hello"])),
1259 + ),
1260 + )
1261 + def test_batch_decode(
1262 + self,
1263 + mistral_tokenizer: MistralTokenizer,
1264 + skip_special_tokens: bool,
1265 + expected_tokens: tuple[str, str],
1266 + ):
1267 + ids = (
1268 + [[1, 3, 23325, 2294, 1686, 4, 23325, 2]],
1269 + [[1, 3, 22177, 4304, 2662, 4, 22177, 2]],
1270 + )
1271 + assert (
1272 + mistral_tokenizer.batch_decode(
1273 + ids[mistral_tokenizer.is_tekken],
1274 + skip_special_tokens=skip_special_tokens,
1275 + )
1276 + == expected_tokens[mistral_tokenizer.is_tekken]
1277 + )
1278 +
1279 + def test_batch_decode_empty(
1280 + self,
1281 + mistral_tokenizer: MistralTokenizer,
1282 + ):
1283 + assert mistral_tokenizer.batch_decode(
1284 + [[]],
1285 + ) == [""]
1286 +
1287 def test_convert_tokens_to_string(self, mistral_tokenizer: MistralTokenizer):
1288 tokens = (
1289 [
vllm/config/speculative.py
+4
@@ -167,6 +167,7 @@ class SpeculativeConfig:
167
168 @staticmethod
169 def hf_config_override(hf_config: PretrainedConfig) -> PretrainedConfig:
170 + initial_architecture = hf_config.architectures[0]
171 if hf_config.model_type in ("deepseek_v3", "deepseek_v32"):
172 hf_config.model_type = "deepseek_mtp"
173 if hf_config.model_type == "deepseek_mtp":
@@ -226,6 +227,9 @@ class SpeculativeConfig:
227 {"n_predict": n_predict, "architectures": ["LongCatFlashMTPModel"]}
228 )
229
230 + if initial_architecture == "MistralLarge3ForCausalLM":
231 + hf_config.update({"architectures": ["EagleMistralLarge3ForCausalLM"]})
232 +
233 return hf_config
234
235 def __post_init__(self):
vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py
+1 -1
@@ -80,7 +80,7 @@ class MistralToolParser(ToolParser):
80 self.tool_call_regex = re.compile(r"\[{.*}\]", re.DOTALL)
81 if _is_fn_name_regex_support(self.model_tokenizer):
82 self.fn_name_regex = re.compile(
83 - r"([a-zA-Z0-9_-]+)(\{[\s\S]*?\})(?=\s*$|,|\s)", re.DOTALL
83 + r"([a-zA-Z0-9_-]+)(\{[\s\S]*?\})(?=\s*$|,|\s)?", re.DOTALL
84 )
85 else:
86 self.fn_name_regex = None
vllm/model_executor/layers/fused_moe/configs/E=128,N=512,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 256,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 256,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 8,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 256,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 2
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 256,
77 + "BLOCK_SIZE_K": 256,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 8,
80 + "num_stages": 2
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 16,
84 + "BLOCK_SIZE_N": 256,
85 + "BLOCK_SIZE_K": 256,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 8,
88 + "num_stages": 2
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 16,
92 + "BLOCK_SIZE_N": 256,
93 + "BLOCK_SIZE_K": 256,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 8,
96 + "num_stages": 2
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 32,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 8,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 8,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
vllm/model_executor/layers/mla.py
+4
@@ -111,6 +111,7 @@ class MultiHeadLatentAttentionWrapper(CustomOp):
111 self,
112 positions: torch.Tensor,
113 hidden_states: torch.Tensor,
114 + llama_4_scaling: torch.Tensor | None = None,
115 ) -> torch.Tensor:
116 q_c = None
117 kv_lora = None
@@ -159,6 +160,9 @@ class MultiHeadLatentAttentionWrapper(CustomOp):
160 hidden_states, q_c, positions, self.indexer_rope_emb
161 )
162
163 + if llama_4_scaling is not None:
164 + q *= llama_4_scaling
165 +
166 attn_out = self.mla_attn(
167 q,
168 kv_c_normed,
vllm/model_executor/layers/rotary_embedding/__init__.py
+1 -1
@@ -238,7 +238,7 @@ def get_rope(
238 dtype,
239 **extra_kwargs,
240 )
241 - elif scaling_type == "deepseek_yarn":
241 + elif scaling_type in ["deepseek_yarn", "deepseek_llama_scaling"]:
242 scaling_factor = rope_parameters["factor"]
243 original_max_position = rope_parameters["original_max_position_embeddings"]
244 # assert max_position == original_max_position * scaling_factor
vllm/model_executor/models/deepseek_v2.py
+59 -7
@@ -395,6 +395,16 @@ def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float:
395 return 0.1 * mscale * math.log(scale) + 1.0
396
397
398 +def _get_llama_4_scaling(
399 + original_max_position_embeddings: int, scaling_beta: float, positions: torch.Tensor
400 +) -> torch.Tensor:
401 + scaling = 1 + scaling_beta * torch.log(
402 + 1 + torch.floor(positions / original_max_position_embeddings)
403 + )
404 + # Broadcast over num_heads and head_dim
405 + return scaling[..., None, None]
406 +
407 +
408 class DeepseekV2Attention(nn.Module):
409 def __init__(
410 self,
@@ -481,7 +491,11 @@ class DeepseekV2Attention(nn.Module):
491 prefix=f"{prefix}.o_proj",
492 )
493 if config.rope_parameters["rope_type"] != "default":
484 - config.rope_parameters["rope_type"] = "deepseek_yarn"
494 + config.rope_parameters["rope_type"] = (
495 + "deepseek_yarn"
496 + if config.rope_parameters.get("apply_yarn_scaling", True)
497 + else "deepseek_llama_scaling"
498 + )
499
500 self.rotary_emb = get_rope(
501 qk_rope_head_dim,
@@ -491,7 +505,10 @@ class DeepseekV2Attention(nn.Module):
505 is_neox_style=False,
506 )
507
494 - if config.rope_parameters["rope_type"] != "default":
508 + if (
509 + config.rope_parameters["rope_type"] != "default"
510 + and config.rope_parameters["rope_type"] == "deepseek_yarn"
511 + ):
512 mscale_all_dim = config.rope_parameters.get("mscale_all_dim", False)
513 scaling_factor = config.rope_parameters["factor"]
514 mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
@@ -511,6 +528,7 @@ class DeepseekV2Attention(nn.Module):
528 self,
529 positions: torch.Tensor,
530 hidden_states: torch.Tensor,
531 + llama_4_scaling: torch.Tensor | None,
532 ) -> torch.Tensor:
533 if self.q_lora_rank is not None:
534 q = self.q_a_proj(hidden_states)[0]
@@ -536,6 +554,11 @@ class DeepseekV2Attention(nn.Module):
554 k = torch.empty_like(q)
555 k[..., : self.qk_nope_head_dim] = k_nope
556 k[..., self.qk_nope_head_dim :] = k_pe
557 +
558 + # Apply llama 4 scaling if provided
559 + if llama_4_scaling is not None:
560 + q *= llama_4_scaling
561 +
562 # padding value to qk_head_dim for alignment
563 v = torch.nn.functional.pad(
564 v, [0, self.qk_head_dim - self.v_head_dim], value=0
@@ -987,7 +1010,12 @@ class DeepseekV2MLAAttention(nn.Module):
1010 )
1011
1012 if config.rope_parameters["rope_type"] != "default":
990 - config.rope_parameters["rope_type"] = "deepseek_yarn"
1013 + config.rope_parameters["rope_type"] = (
1014 + "deepseek_yarn"
1015 + if config.rope_parameters.get("apply_yarn_scaling", True)
1016 + else "deepseek_llama_scaling"
1017 + )
1018 +
1019 self.rotary_emb = get_rope(
1020 qk_rope_head_dim,
1021 rotary_dim=qk_rope_head_dim,
@@ -995,7 +1023,11 @@ class DeepseekV2MLAAttention(nn.Module):
1023 rope_parameters=config.rope_parameters,
1024 is_neox_style=False,
1025 )
998 - if config.rope_parameters["rope_type"] != "default":
1026 +
1027 + if (
1028 + config.rope_parameters["rope_type"] != "default"
1029 + and config.rope_parameters["rope_type"] == "deepseek_yarn"
1030 + ):
1031 mscale_all_dim = config.rope_parameters.get("mscale_all_dim", False)
1032 scaling_factor = config.rope_parameters["factor"]
1033 mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
@@ -1064,8 +1096,9 @@ class DeepseekV2MLAAttention(nn.Module):
1096 self,
1097 positions: torch.Tensor,
1098 hidden_states: torch.Tensor,
1099 + llama_4_scaling: torch.Tensor | None,
1100 ) -> torch.Tensor:
1068 - return self.mla_attn(positions, hidden_states)
1101 + return self.mla_attn(positions, hidden_states, llama_4_scaling)
1102
1103
1104 class DeepseekV2DecoderLayer(nn.Module):
@@ -1155,6 +1188,7 @@ class DeepseekV2DecoderLayer(nn.Module):
1188 positions: torch.Tensor,
1189 hidden_states: torch.Tensor,
1190 residual: torch.Tensor | None,
1191 + llama_4_scaling: torch.Tensor | None = None,
1192 ) -> torch.Tensor:
1193 # Self Attention
1194 if residual is None:
@@ -1165,6 +1199,7 @@ class DeepseekV2DecoderLayer(nn.Module):
1199 hidden_states = self.self_attn(
1200 positions=positions,
1201 hidden_states=hidden_states,
1202 + llama_4_scaling=llama_4_scaling,
1203 )
1204
1205 if (
@@ -1266,8 +1301,24 @@ class DeepseekV2Model(nn.Module):
1301 hidden_states = intermediate_tensors["hidden_states"]
1302 residual = intermediate_tensors["residual"]
1303
1304 + # Compute llama 4 scaling once per forward pass if enabled
1305 + llama_4_scaling_config = getattr(self.config, "llama_4_scaling", None)
1306 + llama_4_scaling: torch.Tensor | None
1307 + if llama_4_scaling_config is not None:
1308 + llama_4_scaling = _get_llama_4_scaling(
1309 + original_max_position_embeddings=llama_4_scaling_config[
1310 + "original_max_position_embeddings"
1311 + ],
1312 + scaling_beta=llama_4_scaling_config["beta"],
1313 + positions=positions,
1314 + )
1315 + else:
1316 + llama_4_scaling = None
1317 +
1318 for layer in islice(self.layers, self.start_layer, self.end_layer):
1270 - hidden_states, residual = layer(positions, hidden_states, residual)
1319 + hidden_states, residual = layer(
1320 + positions, hidden_states, residual, llama_4_scaling
1321 + )
1322
1323 if not get_pp_group().is_last_rank:
1324 return IntermediateTensors(
@@ -1325,6 +1376,7 @@ class DeepseekV2ForCausalLM(
1376 packed_modules_mapping = {
1377 "gate_up_proj": ["gate_proj", "up_proj"],
1378 }
1379 + model_cls = DeepseekV2Model
1380
1381 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1382 super().__init__()
@@ -1355,7 +1407,7 @@ class DeepseekV2ForCausalLM(
1407 "kv_a_proj_with_mqa",
1408 ]
1409
1358 - self.model = DeepseekV2Model(
1410 + self.model = self.model_cls(
1411 vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
1412 )
1413 if get_pp_group().is_last_rank:
vllm/model_executor/models/mistral_large_3.py new
+63
@@ -0,0 +1,63 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +from collections.abc import Iterable
4 +
5 +import regex as re
6 +import torch
7 +
8 +from vllm.model_executor.models.deepseek_v2 import DeepseekV3ForCausalLM
9 +
10 +
11 +class MistralLarge3ForCausalLM(DeepseekV3ForCausalLM):
12 + # fmt: off
13 + remapping = {
14 + r"layers\.(\d+)\.attention_norm\.weight": r"model.layers.\1.input_layernorm.weight", # noqa: E501
15 + r"layers\.(\d+)\.attention\.wq_a\.(\w+)": r"model.layers.\1.self_attn.q_a_proj.\2", # noqa: E501
16 + r"layers\.(\d+)\.attention\.q_a_norm\.weight": r"model.layers.\1.self_attn.q_a_layernorm.weight", # noqa: E501
17 + r"layers\.(\d+)\.attention\.wq_b\.(\w+)": r"model.layers.\1.self_attn.q_b_proj.\2", # noqa: E501
18 + r"layers\.(\d+)\.attention\.wkv_a_with_mqa\.(\w+)": r"model.layers.\1.self_attn.kv_a_proj_with_mqa.\2", # noqa: E501
19 + r"layers\.(\d+)\.attention\.kv_a_norm\.weight": r"model.layers.\1.self_attn.kv_a_layernorm.weight", # noqa: E501
20 + r"layers\.(\d+)\.attention\.wkv_b\.(\w+)": r"model.layers.\1.self_attn.kv_b_proj.\2", # noqa: E501
21 + r"layers\.(\d+)\.attention\.wo\.(\w+)": r"model.layers.\1.self_attn.o_proj.\2", # noqa: E501
22 + r"layers\.(\d+)\.ffn_norm\.weight": r"model.layers.\1.post_attention_layernorm.weight", # noqa: E501
23 + r"layers\.(\d+)\.feed_forward\.w1\.(\w+)": r"model.layers.\1.mlp.gate_proj.\2", # noqa: E501
24 + r"layers\.(\d+)\.feed_forward\.w2\.(\w+)": r"model.layers.\1.mlp.down_proj.\2", # noqa: E501
25 + r"layers\.(\d+)\.feed_forward\.w3\.(\w+)": r"model.layers.\1.mlp.up_proj.\2", # noqa: E501
26 + r"layers\.(\d+)\.gate\.weight": r"model.layers.\1.mlp.gate.weight", # noqa: E501
27 + r"layers\.(\d+)\.shared_experts\.w1\.(\w+)": r"model.layers.\1.mlp.shared_experts.gate_proj.\2", # noqa: E501
28 + r"layers\.(\d+)\.shared_experts\.w2\.(\w+)": r"model.layers.\1.mlp.shared_experts.down_proj.\2", # noqa: E501
29 + r"layers\.(\d+)\.shared_experts\.w3\.(\w+)": r"model.layers.\1.mlp.shared_experts.up_proj.\2", # noqa: E501
30 + r"layers\.(\d+)\.experts\.(\d+)\.w1\.(\w+)": r"model.layers.\1.mlp.experts.\2.gate_proj.\3", # noqa: E501
31 + r"layers\.(\d+)\.experts\.(\d+)\.w2\.(\w+)": r"model.layers.\1.mlp.experts.\2.down_proj.\3", # noqa: E501
32 + r"layers\.(\d+)\.experts\.(\d+)\.w3\.(\w+)": r"model.layers.\1.mlp.experts.\2.up_proj.\3", # noqa: E501
33 + r"norm\.weight": "model.norm.weight", # noqa: E501
34 + r"tok_embeddings\.weight": "model.embed_tokens.weight", # noqa: E501
35 + r"output\.weight": "lm_head.weight", # noqa: E501
36 + }
37 + # fmt: on
38 +
39 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
40 + return super().load_weights(map(self._remap_mistral_to_ds, weights))
41 +
42 + def _remap_mistral_to_ds(
43 + self, weight: tuple[str, torch.Tensor]
44 + ) -> tuple[str, torch.Tensor]:
45 + """Remap Mistral parameters to DeepseekV2 parameters."""
46 + name, loaded_weight = weight
47 +
48 + for k, v in self.remapping.items():
49 + match = re.fullmatch(k, name)
50 + if match:
51 + name = re.sub(k, v, name)
52 + break
53 + else:
54 + raise ValueError(f"Cannot remap {name}")
55 +
56 + # Remapping scale names. We could do this in the regex above but it
57 + # would triple the number of lines for most layers.
58 + if name.endswith(".qscale_act"):
59 + name = re.sub(r"\.qscale_act$", ".input_scale", name)
60 + elif name.endswith(".qscale_weight"):
61 + name = re.sub(r"\.qscale_weight$", ".weight_scale", name)
62 +
63 + return name, loaded_weight
vllm/model_executor/models/mistral_large_3_eagle.py new
+165
@@ -0,0 +1,165 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from collections.abc import Iterable
5 +from functools import partial
6 +
7 +import torch
8 +import torch.nn as nn
9 +
10 +from vllm.compilation.decorators import support_torch_compile
11 +from vllm.config import VllmConfig
12 +from vllm.distributed.parallel_state import get_pp_group
13 +from vllm.logger import init_logger
14 +from vllm.model_executor.layers.layernorm import RMSNorm
15 +from vllm.model_executor.layers.linear import RowParallelLinear
16 +from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
17 +from vllm.model_executor.models.deepseek_v2 import (
18 + DeepseekV2DecoderLayer,
19 + DeepseekV2Model,
20 +)
21 +from vllm.model_executor.models.interfaces import MultiModalEmbeddings
22 +from vllm.model_executor.models.mistral_large_3 import MistralLarge3ForCausalLM
23 +from vllm.multimodal.inputs import NestedTensors
24 +
25 +from .utils import (
26 + _merge_multimodal_embeddings,
27 + make_empty_intermediate_tensors_factory,
28 + maybe_prefix,
29 +)
30 +
31 +logger = init_logger(__name__)
32 +
33 +
34 +@support_torch_compile
35 +class EagleMistralLarge3Model(DeepseekV2Model):
36 + def __init__(
37 + self, *, vllm_config: VllmConfig, prefix: str = "", start_layer_id: int = 0
38 + ):
39 + nn.Module.__init__(self)
40 +
41 + config = vllm_config.model_config.hf_config
42 + quant_config = vllm_config.quant_config
43 + self.config = config
44 + self.vllm_config = vllm_config
45 +
46 + self.vocab_size = config.vocab_size
47 +
48 + assert get_pp_group().world_size == 1
49 + self.embed_tokens = VocabParallelEmbedding(
50 + config.vocab_size,
51 + config.hidden_size,
52 + quant_config=quant_config,
53 + prefix=f"{prefix}.embed_tokens",
54 + )
55 +
56 + self.layers = nn.ModuleList(
57 + [
58 + DeepseekV2DecoderLayer(
59 + vllm_config=vllm_config,
60 + prefix=maybe_prefix(prefix, f"layers.{i + start_layer_id}"),
61 + )
62 + for i in range(self.config.num_hidden_layers)
63 + ]
64 + )
65 + self.start_layer = 0
66 + self.end_layer = self.config.num_hidden_layers
67 +
68 + self.fc = RowParallelLinear(
69 + self.config.hidden_size * 2,
70 + self.config.hidden_size,
71 + bias=False,
72 + input_is_parallel=False,
73 + quant_config=quant_config,
74 + return_bias=False,
75 + )
76 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
77 + self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
78 + ["hidden_states", "residual"], config.hidden_size
79 + )
80 +
81 + def forward(
82 + self,
83 + input_ids: torch.Tensor,
84 + positions: torch.Tensor,
85 + hidden_states: torch.Tensor,
86 + inputs_embeds: torch.Tensor | None = None,
87 + ) -> torch.Tensor:
88 + if inputs_embeds is None:
89 + inputs_embeds = self.embed_input_ids(input_ids)
90 + inputs_embeds = self.fc(torch.cat((inputs_embeds, hidden_states), dim=-1))
91 + output = super().forward(
92 + input_ids, positions, intermediate_tensors=None, inputs_embeds=inputs_embeds
93 + )
94 + assert isinstance(output, torch.Tensor)
95 + return output
96 +
97 +
98 +class EagleMistralLarge3ForCausalLM(MistralLarge3ForCausalLM):
99 + remapping = MistralLarge3ForCausalLM.remapping | {
100 + r"eagle_linear\.weight": r"model.fc.weight",
101 + r"eagle_linear\.qscale_act": r"model.fc.input_scale",
102 + r"eagle_linear\.qscale_weight": r"model.fc.weight_scale",
103 + }
104 +
105 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
106 + target_layer_num = vllm_config.model_config.get_num_layers(
107 + vllm_config.parallel_config
108 + )
109 + vllm_config.model_config = vllm_config.speculative_config.draft_model_config
110 + # draft model quantization config may differ from target model
111 + self.quant_config = VllmConfig.get_quantization_config(
112 + vllm_config.speculative_config.draft_model_config, vllm_config.load_config
113 + )
114 + vllm_config.quant_config = self.quant_config
115 + self.model_cls = partial(
116 + EagleMistralLarge3Model, start_layer_id=target_layer_num
117 + )
118 + super().__init__(vllm_config=vllm_config, prefix=prefix)
119 +
120 + def get_input_embeddings(
121 + self,
122 + input_ids: torch.Tensor,
123 + multimodal_embeddings: MultiModalEmbeddings | None = None,
124 + *,
125 + is_multimodal: torch.Tensor | None = None,
126 + handle_oov_mm_token: bool = False,
127 + ) -> torch.Tensor:
128 + inputs_embeds = super().embed_input_ids(input_ids)
129 +
130 + if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
131 + return inputs_embeds
132 +
133 + assert is_multimodal is not None
134 +
135 + return _merge_multimodal_embeddings(
136 + inputs_embeds=inputs_embeds,
137 + multimodal_embeddings=multimodal_embeddings,
138 + is_multimodal=is_multimodal,
139 + )
140 +
141 + def forward(
142 + self,
143 + input_ids: torch.Tensor,
144 + positions: torch.Tensor,
145 + hidden_states: torch.Tensor,
146 + inputs_embeds: torch.Tensor | None = None,
147 + ) -> tuple[torch.Tensor, torch.Tensor]:
148 + hidden_states = self.model(input_ids, positions, hidden_states, inputs_embeds)
149 + return hidden_states, hidden_states
150 +
151 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
152 + # Pretend we've loaded the embedding and lm_head weights
153 + # (later copied from target model)
154 + return super().load_weights(weights) | {
155 + "model.embed_tokens.weight",
156 + "lm_head.weight",
157 + }
158 +
159 + def embed_input_ids(
160 + self,
161 + input_ids: torch.Tensor,
162 + multimodal_embeddings: NestedTensors | None = None,
163 + is_multimodal: torch.Tensor | None = None,
164 + ) -> torch.Tensor:
165 + return self.model.embed_input_ids(input_ids)
vllm/model_executor/models/registry.py
+5
@@ -145,6 +145,7 @@ _TEXT_GENERATION_MODELS = {
145 "MiniMaxM1ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
146 "MiniMaxM2ForCausalLM": ("minimax_m2", "MiniMaxM2ForCausalLM"),
147 "MistralForCausalLM": ("llama", "LlamaForCausalLM"),
148 + "MistralLarge3ForCausalLM": ("mistral_large_3", "MistralLarge3ForCausalLM"),
149 "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
150 # transformers's mpt class has lower case
151 "MptForCausalLM": ("mpt", "MPTForCausalLM"),
@@ -424,6 +425,10 @@ _SPECULATIVE_DECODING_MODELS = {
425 "LlamaForCausalLMEagle3": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
426 "Eagle3Qwen2_5vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
427 "Eagle3Qwen3vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
428 + "EagleMistralLarge3ForCausalLM": (
429 + "mistral_large_3_eagle",
430 + "EagleMistralLarge3ForCausalLM",
431 + ),
432 "EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
433 "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
434 "ErnieMTPModel": ("ernie_mtp", "ErnieMTP"),
vllm/tokenizers/mistral.py
+36
@@ -97,6 +97,8 @@ def _prepare_apply_chat_template_tools_and_messages(
97 continue_final_message: bool = False,
98 add_generation_prompt: bool = False,
99 ) -> tuple[list["ChatCompletionMessageParam"], list[dict[str, Any]] | None]:
100 + from mistral_common.protocol.instruct.tool_calls import Function, Tool
101 +
102 if add_generation_prompt and continue_final_message:
103 raise ValueError(
104 "Cannot set both `add_generation_prompt` and "
@@ -139,6 +141,33 @@ def _prepare_apply_chat_template_tools_and_messages(
141 if function.get("description") is None:
142 function["description"] = ""
143
144 + # We filter not supported arguments to avoid throwing an error.
145 + # TODO(juliendenize): remove this once OpenAI API is better supported by
146 + # `mistral-common`.
147 + tools_fields = set(Tool.model_fields.keys())
148 + function_fields = set(Function.model_fields.keys())
149 + for tool in tools:
150 + tool_keys = list(tool.keys())
151 + for tool_key in tool_keys:
152 + if tool_key not in tools_fields:
153 + tool.pop(tool_key)
154 + logger.warning_once(
155 + f"'{tool_key}' is not supported by mistral-common for tools. "
156 + "It has been poped from the tool definition."
157 + )
158 + if tool["type"] == "function":
159 + function_keys = list(tool["function"].keys())
160 + for function_key in function_keys:
161 + if function_key not in function_fields:
162 + tool["function"].pop(function_key)
163 + logger.warning_once(
164 + f"'{function_key}' is not supported by mistral-common "
165 + "for function tools. It has been poped from the "
166 + "function definition."
167 + )
168 + else:
169 + raise ValueError("mistral-common only supports function tools.")
170 +
171 return messages, tools
172
173
@@ -410,6 +439,13 @@ class MistralTokenizer(TokenizerLike):
439 ids, skip_special_tokens=skip_special_tokens
440 )
441
442 + def batch_decode(
443 + self, ids: list[list[int]] | list[int], skip_special_tokens: bool = False
444 + ) -> str:
445 + return self.transformers_tokenizer.batch_decode(
446 + ids, skip_special_tokens=skip_special_tokens
447 + )
448 +
449 def convert_tokens_to_string(self, tokens: list[str]) -> str:
450 from mistral_common.tokens.tokenizers.base import (
451 SpecialTokenPolicy,
vllm/transformers_utils/configs/eagle.py
+6
@@ -82,3 +82,9 @@ class EAGLEConfig(PretrainedConfig):
82 pretrained_model_name_or_path, **kwargs
83 )
84 return cls.from_dict(config_dict, **kwargs)
85 +
86 + def to_json_string(self, use_diff: bool = True) -> str:
87 + # we override use_diff to False as initializing
88 + # EAGLEConfig with default arguments is not supported
89 + del use_diff
90 + return super().to_json_string(use_diff=False)
vllm/transformers_utils/configs/mistral.py
+62 -12
@@ -18,9 +18,31 @@ def adapt_config_dict(
18 if bool(config_dict.get("quantization")):
19 config_dict = _remap_mistral_quantization_args(config_dict)
20
21 + is_moe = bool(config_dict.get("moe"))
22 + is_mistral_large_3 = (
23 + is_moe and (config_dict["moe"].get("num_shared_experts") or 0) > 0
24 + )
25 if config_dict.get("model_type") == "mamba":
26 config_dict["architectures"] = ["Mamba2ForCausalLM"]
23 - elif bool(config_dict.get("moe")):
27 + elif is_moe and is_mistral_large_3:
28 + config_dict = _remap_moe_args(config_dict)
29 + config_dict["model_type"] = "deepseek_v3"
30 + config_dict["architectures"] = ["MistralLarge3ForCausalLM"]
31 +
32 + assert "llama_4_scaling" in config_dict, (
33 + "MistralLarge3 expect llama4 scaling config."
34 + )
35 + llama_4_scaling_config_keys = ["original_max_position_embeddings", "beta"]
36 + assert all(
37 + [
38 + key in config_dict["llama_4_scaling"]
39 + for key in llama_4_scaling_config_keys
40 + ]
41 + ), (
42 + "llama_4_scaling config should define the keys: "
43 + f"{','.join(llama_4_scaling_config_keys)}"
44 + )
45 + elif is_moe:
46 config_dict["architectures"] = ["MixtralForCausalLM"]
47 else:
48 config_dict["architectures"] = ["MistralForCausalLM"]
@@ -140,17 +162,20 @@ def _remap_general_mistral_args(config: dict) -> dict:
162
163
164 def _remap_mistral_quantization_args(config: dict) -> dict:
143 - quantization = config.get("quantization", {})
144 - if quantization.get("qformat_weight") == "fp8_e4m3":
145 - # This maps to the FP8 static per-tensor quantization scheme
146 - quantization_config = {"quant_method": "fp8", "activation_scheme": "static"}
147 - elif quantization.get("quant_method") == "compressed-tensors":
148 - # Pass through the quantization config to compressed-tensors
149 - quantization_config = quantization
150 - else:
151 - raise ValueError(f"Found unknown quantization='{quantization}' in config")
152 -
153 - config["quantization_config"] = quantization_config
165 + if config.get("quantization"):
166 + quantization = config.pop("quantization", {})
167 + if quantization.get("qformat_weight") == "fp8_e4m3":
168 + qscheme_act = quantization.get("qscheme_act")
169 + assert qscheme_act in ("NO_SCALES", "TENSOR", None), (
170 + "Only NO_SCALES and TENSOR (default) are supported for qscheme_act"
171 + )
172 + is_dynamic = qscheme_act == "NO_SCALES"
173 + config["quantization_config"] = {
174 + "quant_method": "fp8",
175 + "activation_scheme": "dynamic" if is_dynamic else "static",
176 + }
177 + else:
178 + raise ValueError(f"Found unknown quantization='{quantization}' in config")
179
180 return config
181
@@ -183,3 +208,28 @@ def _remap_mistral_audio_args(config: dict) -> dict:
208 if quant_config:
209 config["quantization_config"] = quant_config
210 return config
211 +
212 +
213 +def _remap_moe_args(config: dict) -> dict:
214 + moe_config_map = {
215 + "route_every_n": "moe_layer_freq",
216 + "first_k_dense_replace": "first_k_dense_replace",
217 + "num_experts_per_tok": "num_experts_per_tok",
218 + "num_experts": "n_routed_experts",
219 + "expert_hidden_dim": "moe_intermediate_size",
220 + "routed_scale": "routed_scaling_factor",
221 + "num_shared_experts": "n_shared_experts",
222 + "num_expert_groups": "n_group",
223 + "num_expert_groups_per_tok": "topk_group",
224 + }
225 + moe_config = config.get("moe", {})
226 + for old_name, new_name in moe_config_map.items():
227 + if old_name in moe_config:
228 + value = moe_config.pop(old_name)
229 + config[new_name] = value
230 +
231 + config["topk_method"] = None
232 + config["norm_topk_prob"] = True
233 + config["scoring_func"] = "softmax"
234 +
235 + return config
vllm/v1/spec_decode/eagle.py
+4
@@ -1016,6 +1016,10 @@ class EagleProposer:
1016 "Qwen3VLForConditionalGeneration",
1017 ]:
1018 self.model.config.image_token_index = target_model.config.image_token_id
1019 + elif self.get_model_name(target_model) == "PixtralForConditionalGeneration":
1020 + self.model.config.image_token_index = (
1021 + target_model.config.vision_config.image_token_id
1022 + )
1023 else:
1024 self.model.config.image_token_index = (
1025 target_model.config.image_token_index