A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Speculators][Speculative Decoding] Fix Qwen 2 Eagle3 Support (#23337)
PapaGoose committed
Aug 22, 2025 at 19:39 UTC
88491c1b6bcac8fa6adfa22489c92419c5e89055
1 file changed
+9
-2
vllm/model_executor/models/qwen2.py
+9
-2
@@ -52,7 +52,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
52
from vllm.sequence import IntermediateTensors
53
from vllm.transformers_utils.config import is_interleaved
54
55
-from .interfaces import SupportsLoRA, SupportsPP
55
+from .interfaces import SupportsEagle3, SupportsLoRA, SupportsPP
56
from .utils import (AutoWeightsLoader, PPMissingLayer, extract_layer_index,
57
is_pp_missing_parameter,
58
make_empty_intermediate_tensors_factory, make_layers,
@@ -442,7 +442,7 @@ class Qwen2Model(nn.Module):
442
return loaded_params
443
444
445
-class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
445
+class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3):
446
packed_modules_mapping = {
447
"qkv_proj": [
448
"q_proj",
@@ -488,6 +488,13 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
488
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
489
return self.model.get_input_embeddings(input_ids)
490
491
+ def set_aux_hidden_state_layers(self, layers: tuple[int]) -> None:
492
+ self.model.aux_hidden_state_layers = layers
493
+
494
+ def get_eagle3_aux_hidden_state_layers(self) -> tuple[int]:
495
+ num_layers = len(self.model.layers)
496
+ return (2, num_layers // 2, num_layers - 3)
497
+
498
def forward(
499
self,
500
input_ids: torch.Tensor,