A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 16:30 UTC
Enable Eagle3 speculative decoding for Mistral3ForConditionalGeneration to support eagle3 (#33939)
Signed-off-by: Akintunde Oladipo <akintunde.oladipo@servicenow.com> Signed-off-by: TundeAtSN <akintunde.oladipo@servicenow.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
TundeAtSN committed
Feb 7, 2026 at 08:24 UTC
4df44c16ba8c4e44aeb7bf0dd622933c693d7613
1 file changed
+9
-1
vllm/model_executor/models/mistral3.py
+9
-1
@@ -44,6 +44,7 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape
44
45
from .interfaces import (
46
MultiModalEmbeddings,
47
+ SupportsEagle3,
48
SupportsLoRA,
49
SupportsMultiModal,
50
SupportsPP,
@@ -408,7 +409,7 @@ def init_vision_tower_for_llava(
409
dummy_inputs=Mistral3DummyInputsBuilder,
410
)
411
class Mistral3ForConditionalGeneration(
411
- nn.Module, SupportsLoRA, SupportsMultiModal, SupportsPP
412
+ nn.Module, SupportsLoRA, SupportsMultiModal, SupportsPP, SupportsEagle3
413
):
414
packed_modules_mapping = {
415
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
@@ -432,6 +433,13 @@ class Mistral3ForConditionalGeneration(
433
434
raise ValueError("Only image modality is supported")
435
436
+ def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
437
+ self.get_language_model().model.aux_hidden_state_layers = layers
438
+
439
+ def get_eagle3_aux_hidden_state_layers(self) -> tuple[int, ...]:
440
+ num_layers = len(self.get_language_model().model.layers)
441
+ return (2, num_layers // 2, num_layers - 3)
442
+
443
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
444
super().__init__()
445