A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 21:45 UTC
Add support to Mistral large 3 eagle with dense layers (#36163)
Signed-off-by: juliendenize <julien.denize@mistral.ai> Signed-off-by: Julien Denize <40604584+juliendenize@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Julien Denize committed
Mar 11, 2026 at 16:42 UTC
afebeffbfbf2dd61bad940ce13942af8a8931524
2 files changed
+28
-1
vllm/model_executor/models/mistral_large_3_eagle.py
+5
-1
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+import copy
5
from collections.abc import Iterable
6
from functools import partial
7
@@ -33,7 +34,9 @@ class EagleMistralLarge3Model(DeepseekV2Model):
34
):
35
nn.Module.__init__(self)
36
36
- config = vllm_config.model_config.hf_config
37
+ config = copy.deepcopy(vllm_config.model_config.hf_config)
38
+ config.first_k_dense_replace += start_layer_id
39
+
40
quant_config = vllm_config.quant_config
41
self.config = config
42
self.vllm_config = vllm_config
@@ -53,6 +56,7 @@ class EagleMistralLarge3Model(DeepseekV2Model):
56
DeepseekV2DecoderLayer(
57
vllm_config=vllm_config,
58
prefix=maybe_prefix(prefix, f"layers.{i + start_layer_id}"),
59
+ config=config,
60
)
61
for i in range(self.config.num_hidden_layers)
62
]
vllm/transformers_utils/configs/mistral.py
+23
@@ -19,6 +19,10 @@ def adapt_config_dict(
19
if bool(config_dict.get("quantization")):
20
config_dict = _remap_mistral_quantization_args(config_dict)
21
22
+ is_mla = bool(config_dict.get("qk_nope_head_dim"))
23
+ if is_mla:
24
+ config_dict = _remap_mistral_mla_args(config_dict)
25
+
26
is_moe = bool(config_dict.get("moe"))
27
is_mistral_large_3 = (
28
is_moe and (config_dict["moe"].get("num_shared_experts") or 0) > 0
@@ -291,3 +295,22 @@ def _remap_moe_args(config: dict) -> dict:
295
config["scoring_func"] = "softmax"
296
297
return config
298
+
299
+
300
+def _remap_mistral_mla_args(config: dict) -> dict:
301
+ if not config.get("moe"):
302
+ moe = {
303
+ "num_experts": 1,
304
+ "first_k_dense_replace": config.get("num_hidden_layers"),
305
+ "route_every_n": 1,
306
+ "num_shared_experts": 1,
307
+ "expert_hidden_dim": config.get("intermediate_size"),
308
+ "num_experts_per_tok": 1,
309
+ "routed_scale": 1.0,
310
+ "renorm_strategy": "WEIGHTS",
311
+ "use_load_balancing_bias": False,
312
+ "num_expert_groups": 1,
313
+ "num_expert_groups_per_tok": 1,
314
+ }
315
+ config["moe"] = moe
316
+ return config