A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 09:45 UTC
[Multimodal][Speculative Decoding]Eagle3 mm support, enablement on qwen3vl (#29594)
Signed-off-by: Tsai, Louie <louie.tsai@intel.com> Signed-off-by: EanWang211123 <wangyiheng@sangfor.com.cn> Co-authored-by: Louie Tsai <louie.tsai@intel.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
EanWang211123 committed
Nov 28, 2025 at 14:05 UTC
37b15e97e8443a7fd76f5aa95a78d5593f7241a4
5 files changed
+45
-5
tests/models/registry.py
+4
@@ -913,6 +913,10 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
913
"Qwen/Qwen2.5-VL-7B-Instruct",
914
speculative_model="Rayzl/qwen2.5-vl-7b-eagle3-sgl",
915
),
916
+ "Eagle3Qwen3vlForCausalLM": _HfExamplesInfo(
917
+ "Qwen/Qwen3-VL-8B-Instruct",
918
+ speculative_model="taobao-mnn/Qwen3-VL-8B-Instruct-Eagle3",
919
+ ),
920
"Qwen3NextMTP": _HfExamplesInfo(
921
"Qwen/Qwen3-Next-80B-A3B-Instruct", min_transformers_version="4.56.3"
922
),
tests/v1/e2e/test_spec_decode.py
+14
@@ -283,6 +283,19 @@ def test_speculators_model_integration(
283
["model_setup", "mm_enabled", "enable_chunked_prefill"],
284
[
285
(("eagle3", "Qwen/Qwen3-8B", "AngelSlim/Qwen3-8B_eagle3", 1), False, False),
286
+ pytest.param(
287
+ (
288
+ "eagle3",
289
+ "Qwen/Qwen3-VL-8B-Instruct",
290
+ "taobao-mnn/Qwen3-VL-8B-Instruct-Eagle3",
291
+ 1,
292
+ ),
293
+ False,
294
+ False,
295
+ marks=pytest.mark.skip(
296
+ reason="architecture of its eagle3 is LlamaForCausalLMEagle3"
297
+ ),
298
+ ),
299
pytest.param(
300
(
301
"eagle3",
@@ -352,6 +365,7 @@ def test_speculators_model_integration(
365
],
366
ids=[
367
"qwen3_eagle3",
368
+ "qwen3_vl_eagle3",
369
"qwen2_5_vl_eagle3",
370
"llama3_eagle",
371
"llama3_eagle3",
vllm/model_executor/models/qwen3_vl.py
+22
-1
@@ -89,6 +89,7 @@ from vllm.utils.collection_utils import is_list_of
89
90
from .interfaces import (
91
MultiModalEmbeddings,
92
+ SupportsEagle3,
93
SupportsLoRA,
94
SupportsMRoPE,
95
SupportsMultiModal,
@@ -1122,9 +1123,14 @@ class Qwen3LLMModel(Qwen3Model):
1123
assert intermediate_tensors is not None
1124
hidden_states = intermediate_tensors["hidden_states"]
1125
residual = intermediate_tensors["residual"]
1126
+
1127
+ aux_hidden_states = []
1128
for layer_idx, layer in islice(
1129
enumerate(self.layers), self.start_layer, self.end_layer
1130
):
1131
+ if layer_idx in self.aux_hidden_state_layers:
1132
+ aux_hidden_states.append(hidden_states + residual)
1133
+
1134
hidden_states, residual = layer(
1135
positions,
1136
hidden_states,
@@ -1144,6 +1150,9 @@ class Qwen3LLMModel(Qwen3Model):
1150
{"hidden_states": hidden_states, "residual": residual}
1151
)
1152
hidden_states, _ = self.norm(hidden_states, residual)
1153
+
1154
+ if len(aux_hidden_states) > 0:
1155
+ return hidden_states, aux_hidden_states
1156
return hidden_states
1157
1158
@@ -1186,7 +1195,12 @@ class Qwen3LLMForCausalLM(Qwen3ForCausalLM):
1195
dummy_inputs=Qwen3VLDummyInputsBuilder,
1196
)
1197
class Qwen3VLForConditionalGeneration(
1189
- nn.Module, SupportsMultiModal, SupportsLoRA, SupportsPP, SupportsMRoPE
1198
+ nn.Module,
1199
+ SupportsMultiModal,
1200
+ SupportsLoRA,
1201
+ SupportsPP,
1202
+ SupportsMRoPE,
1203
+ SupportsEagle3,
1204
):
1205
merge_by_field_config = True
1206
multimodal_cpu_fields = {"image_grid_thw", "video_grid_thw"}
@@ -1279,6 +1293,13 @@ class Qwen3VLForConditionalGeneration(
1293
self.visual_dim = config.vision_config.out_hidden_size
1294
self.multiscale_dim = self.visual_dim * self.deepstack_num_level
1295
1296
+ def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
1297
+ self.language_model.model.aux_hidden_state_layers = layers
1298
+
1299
+ def get_eagle3_aux_hidden_state_layers(self) -> tuple[int, ...]:
1300
+ num_layers = len(self.language_model.model.layers)
1301
+ return (2, num_layers // 2, num_layers - 3)
1302
+
1303
def _get_deepstack_input_embeds(self, num_tokens: int) -> IntermediateTensors:
1304
# get deepstack_input_embeds from buffer, and clear the buffer
1305
return IntermediateTensors(
vllm/model_executor/models/registry.py
+1
@@ -414,6 +414,7 @@ _SPECULATIVE_DECODING_MODELS = {
414
"Eagle3LlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
415
"LlamaForCausalLMEagle3": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
416
"Eagle3Qwen2_5vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
417
+ "Eagle3Qwen3vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
418
"EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
419
"DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
420
"ErnieMTPModel": ("ernie_mtp", "ErnieMTP"),
vllm/v1/spec_decode/eagle.py
+4
-4
@@ -1017,10 +1017,10 @@ class EagleProposer:
1017
1018
if supports_multimodal(target_model):
1019
# handle multimodality
1020
- if (
1021
- self.get_model_name(target_model)
1022
- == "Qwen2_5_VLForConditionalGeneration"
1023
- ):
1020
+ if self.get_model_name(target_model) in [
1021
+ "Qwen2_5_VLForConditionalGeneration",
1022
+ "Qwen3VLForConditionalGeneration",
1023
+ ]:
1024
self.model.config.image_token_index = target_model.config.image_token_id
1025
else:
1026
self.model.config.image_token_index = (