A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 21:48 UTC
Enable Eagle3 speculative decoding for Pixtral (LlavaForConditionalGeneration) (#32542)
Signed-off-by: gopalsarda <gopal.sarda@servicenow.com>
gopalsarda committed
Jan 20, 2026 at 19:18 UTC
0900cedb3f89e475bea256c4cf5a13b5f02635bc
1 file changed
+9
-1
vllm/model_executor/models/llava.py
+9
-1
@@ -53,6 +53,7 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape
53
from .clip import CLIPVisionModel
54
from .interfaces import (
55
MultiModalEmbeddings,
56
+ SupportsEagle3,
57
SupportsLoRA,
58
SupportsMultiModal,
59
SupportsPP,
@@ -503,7 +504,7 @@ def init_vision_tower_for_llava(
504
dummy_inputs=LlavaDummyInputsBuilder,
505
)
506
class LlavaForConditionalGeneration(
506
- nn.Module, SupportsLoRA, SupportsMultiModal, SupportsPP
507
+ nn.Module, SupportsLoRA, SupportsMultiModal, SupportsPP, SupportsEagle3
508
):
509
packed_modules_mapping = {
510
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
@@ -527,6 +528,13 @@ class LlavaForConditionalGeneration(
528
529
raise ValueError("Only image modality is supported")
530
531
+ def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
532
+ self.get_language_model().model.aux_hidden_state_layers = layers
533
+
534
+ def get_eagle3_aux_hidden_state_layers(self) -> tuple[int, ...]:
535
+ num_layers = len(self.get_language_model().model.layers)
536
+ return (2, num_layers // 2, num_layers - 3)
537
+
538
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
539
super().__init__()
540