A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 08:15 UTC
[Pixtral] Enable Pixtral language model support Eagle3 (#37182)
Signed-off-by: remi <remi@mistral.ai>
Rémi Delacourt committed
Mar 20, 2026 at 16:50 UTC
aa84e43ccb540dfbbd723f5b315ef7eefd732641
1 file changed
+18
-1
vllm/model_executor/models/pixtral.py
+18
-1
@@ -66,9 +66,11 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape
66
67
from .interfaces import (
68
MultiModalEmbeddings,
69
+ SupportsEagle3,
70
SupportsLoRA,
71
SupportsMultiModal,
72
SupportsPP,
73
+ supports_eagle3,
74
)
75
from .module_mapping import MultiModelKeys
76
from .utils import StageMissingLayer, init_vllm_registered_model, maybe_prefix
@@ -262,7 +264,7 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
264
dummy_inputs=PixtralDummyInputsBuilder,
265
)
266
class PixtralForConditionalGeneration(
265
- nn.Module, SupportsLoRA, SupportsMultiModal, SupportsPP
267
+ nn.Module, SupportsLoRA, SupportsEagle3, SupportsMultiModal, SupportsPP
268
):
269
@classmethod
270
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
@@ -390,6 +392,21 @@ class PixtralForConditionalGeneration(
392
) -> torch.Tensor | None:
393
return self.language_model.compute_logits(hidden_states)
394
395
+ def _require_language_model_eagle3(self) -> None:
396
+ if not supports_eagle3(self.language_model):
397
+ raise RuntimeError(
398
+ f"EAGLE-3 speculative decoding requires the language model to "
399
+ f"support EAGLE-3, but {type(self.language_model).__name__} does not."
400
+ )
401
+
402
+ def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
403
+ self._require_language_model_eagle3()
404
+ self.language_model.set_aux_hidden_state_layers(layers)
405
+
406
+ def get_eagle3_aux_hidden_state_layers(self) -> tuple[int, ...]:
407
+ self._require_language_model_eagle3()
408
+ return self.language_model.get_eagle3_aux_hidden_state_layers()
409
+
410
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
411
def is_vision_encoder_weights(weight: tuple[str, torch.Tensor]):
412
return weight[0].startswith(("vision_encoder", "vision_tower"))