@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bd92089d33

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 9, 12:30 UTC

feature: support eagle3 for HunyuanVL & Hunyuan (#33035)

Signed-off-by: irisliu10 <601012173@qq.com> Signed-off-by: Iris <38269816+irisliu10@users.noreply.github.com>

Iris committed Jan 28, 2026 at 01:55 UTC bd92089d338ed5187dc5a466a8cc319c225515e7

4 files changed +49 -3

vllm/config/speculative.py

+8 -1

@@ -675,7 +675,14 @@ class SpeculativeConfig:

675 f"{self.disable_by_batch_size=}"

676 )

677

678 - eagle3_target_supported = ["llama", "qwen", "minicpm", "gpt_oss"]

678 + eagle3_target_supported = [

679 + "llama",

680 + "qwen",

681 + "minicpm",

682 + "gpt_oss",

683 + "hunyuan_vl",

684 + "hunyuan_v1_dense",

685 + ]

686 if (

687 self.method == "eagle3"

688 and self.target_model_config

vllm/model_executor/models/hunyuan_v1.py

+17 -2

@@ -66,7 +66,7 @@ from vllm.model_executor.model_loader.weight_utils import (

66 from vllm.sequence import IntermediateTensors

67 from vllm.v1.attention.backend import AttentionType

69 -from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP

69 +from .interfaces import MixtureOfExperts, SupportsEagle3, SupportsLoRA, SupportsPP

70 from .utils import (

71 AutoWeightsLoader,

72 PPMissingLayer,

@@ -630,6 +630,7 @@ class HunYuanModel(nn.Module):

630 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)

631 else:

632 self.norm = PPMissingLayer()

633 + self.aux_hidden_state_layers = tuple[int, ...]()

634

635 def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:

636 return self.embed_tokens(input_ids)

@@ -654,9 +655,13 @@ class HunYuanModel(nn.Module):

655

656 cla_factor = _get_cla_factor(self.config)

657 prev_kv_states = None

658 + aux_hidden_states = []

659 for i, layer in enumerate(

660 islice(self.layers, self.start_layer, self.end_layer)

661 ):

662 + if i in self.aux_hidden_state_layers:

663 + aux_hidden_states.append(hidden_states + residual)

664 +

665 hidden_states, residual, kv_states = layer(

666 positions,

667 hidden_states,

@@ -675,6 +680,9 @@ class HunYuanModel(nn.Module):

680 )

681

682 hidden_states, _ = self.norm(hidden_states, residual)

683 +

684 + if len(aux_hidden_states) > 0:

685 + return hidden_states, aux_hidden_states

686 return hidden_states

687

688 def _split_qkv_weight(self, qkv: torch.Tensor):

@@ -897,7 +905,7 @@ class HunYuanModel(nn.Module):

905 return loaded_params

906

907

900 -class HunyuanV1ModelBase(nn.Module, SupportsLoRA, SupportsPP):

908 +class HunyuanV1ModelBase(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3):

909 packed_modules_mapping = {

910 "qkv_proj": [

911 "q_proj",

@@ -936,6 +944,13 @@ class HunyuanV1ModelBase(nn.Module, SupportsLoRA, SupportsPP):

944 else:

945 self.lm_head = PPMissingLayer()

946

947 + def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:

948 + self.model.aux_hidden_state_layers = layers

949 +

950 + def get_eagle3_aux_hidden_state_layers(self) -> tuple[int, ...]:

951 + num_layers = len(self.model.layers)

952 + return (2, num_layers // 2, num_layers - 3)

953 +

954 def forward(

955 self,

956 input_ids: torch.Tensor | None,

vllm/model_executor/models/hunyuan_vision.py

@@ -83,6 +83,7 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape

84 from .interfaces import (

85 MultiModalEmbeddings,

86 + SupportsEagle3,

87 SupportsLoRA,

88 SupportsMultiModal,

89 SupportsPP,

@@ -780,6 +781,7 @@ class HunYuanVLForConditionalGeneration(

781 SupportsPP,

782 SupportsQuant,

783 SupportsXDRoPE,

784 + SupportsEagle3,

785 ):

786 # To ensure correct weight loading and mapping.

787 hf_to_vllm_mapper = WeightsMapper(

@@ -966,6 +968,13 @@ class HunYuanVLForConditionalGeneration(

968 multimodal_embeddings += tuple(image_embeddings)

969 return multimodal_embeddings

970

971 + def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:

972 + self.language_model.model.aux_hidden_state_layers = layers

973 +

974 + def get_eagle3_aux_hidden_state_layers(self) -> tuple[int, ...]:

975 + num_layers = len(self.language_model.model.layers)

976 + return (2, num_layers // 2, num_layers - 3)

977 +

978 def forward(

979 self,

980 input_ids: torch.Tensor | None,

vllm/v1/spec_decode/eagle.py

+15

@@ -115,6 +115,8 @@ class SpecDecodeBaseProposer:

115 # Use draft model's M-RoPE setting, not target model's

116 # Draft models may be text-only even if target is multimodal

117 self.uses_mrope = self.draft_model_config.uses_mrope

118 + self.uses_xdrope_dim = self.vllm_config.model_config.uses_xdrope_dim

119 + self.draft_uses_xdrope_dim = self.draft_model_config.uses_xdrope_dim

120 if self.uses_mrope:

121 # NOTE: `mrope_positions` is implemented with one additional dummy

122 # position on purpose to make it non-contiguous so that it can work

@@ -129,6 +131,12 @@ class SpecDecodeBaseProposer:

131 self.mrope_positions = torch.zeros(

132 (3, self.max_num_tokens + 1), dtype=torch.int64, device=device

133 )

134 + elif self.uses_xdrope_dim > 0 and self.draft_uses_xdrope_dim > 0:

135 + self.xdrope_positions = torch.zeros(

136 + (self.uses_xdrope_dim, self.max_num_tokens + 1),

137 + dtype=torch.int64,

138 + device=device,

139 + )

140 else:

141 # RoPE need (max_num_tokens,)

142 self.positions = torch.zeros(

@@ -221,11 +229,15 @@ class SpecDecodeBaseProposer:

229 def _get_positions(self, num_tokens: int):

230 if self.uses_mrope:

231 return self.mrope_positions[:, :num_tokens]

232 + if self.uses_xdrope_dim > 0 and self.draft_uses_xdrope_dim > 0:

233 + return self.xdrope_positions[:, :num_tokens]

234 return self.positions[:num_tokens]

235

236 def _set_positions(self, num_tokens: int, positions: torch.Tensor):

237 if self.uses_mrope:

238 self.mrope_positions[:, :num_tokens] = positions

239 + elif self.uses_xdrope_dim > 0 and self.draft_uses_xdrope_dim > 0:

240 + self.xdrope_positions[:, :num_tokens] = positions

241 else:

242 # Convert M-RoPE positions if target model uses M-RoPE

243 # but draft doesn't, For text inputs, all M-RoPE

@@ -623,6 +635,8 @@ class SpecDecodeBaseProposer:

635 self.input_ids[last_token_indices] = next_token_ids

636

637 # copy inputs to buffer for cudagraph

638 + if self.uses_xdrope_dim > 0 and self.draft_uses_xdrope_dim == 0:

639 + target_positions = target_positions[0]

640 self._set_positions(num_tokens, target_positions)

641

642 return num_tokens, last_token_indices, cad

@@ -1126,6 +1140,7 @@ class SpecDecodeBaseProposer:

1140 "Qwen2_5_VLForConditionalGeneration",

1141 "Qwen3VLForConditionalGeneration",

1142 "Qwen3VLMoeForConditionalGeneration",

1143 + "HunYuanVLForConditionalGeneration",

1144 "GlmOcrForConditionalGeneration",

1145 ]:

1146 self.model.config.image_token_index = target_model.config.image_token_id