A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 21:48 UTC
Added qwen3 vision language moe support for speculative decoding (#32048)
Signed-off-by: shanjiaz <zsjwpianpian@gmail.com> Signed-off-by: shanjiaz <43143795+shanjiaz@users.noreply.github.com>
shanjiaz committed
Jan 20, 2026 at 22:24 UTC
7ab80a8e37ac62f73e09fdc9ba7f69dd09cda2a8
2 files changed
+17
-1
vllm/model_executor/models/qwen3_vl_moe.py
+8
@@ -110,9 +110,14 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
110
assert intermediate_tensors is not None
111
hidden_states = intermediate_tensors["hidden_states"]
112
residual = intermediate_tensors["residual"]
113
+
114
+ aux_hidden_states = []
115
for layer_idx, layer in islice(
116
enumerate(self.layers), self.start_layer, self.end_layer
117
):
118
+ if layer_idx in self.aux_hidden_state_layers:
119
+ aux_hidden_states.append(hidden_states + residual)
120
+
121
hidden_states, residual = layer(
122
positions,
123
hidden_states,
@@ -132,6 +137,9 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
137
{"hidden_states": hidden_states, "residual": residual}
138
)
139
hidden_states, _ = self.norm(hidden_states, residual)
140
+
141
+ if len(aux_hidden_states) > 0:
142
+ return hidden_states, aux_hidden_states
143
return hidden_states
144
145
def load_fused_expert_weights(
vllm/v1/spec_decode/eagle.py
+9
-1
@@ -112,7 +112,9 @@ class SpecDecodeBaseProposer:
112
self.input_ids = torch.zeros(
113
self.max_num_tokens, dtype=torch.int32, device=device
114
)
115
- self.uses_mrope = self.vllm_config.model_config.uses_mrope
115
+ # Use draft model's M-RoPE setting, not target model's
116
+ # Draft models may be text-only even if target is multimodal
117
+ self.uses_mrope = self.draft_model_config.uses_mrope
118
if self.uses_mrope:
119
# NOTE: `mrope_positions` is implemented with one additional dummy
120
# position on purpose to make it non-contiguous so that it can work
@@ -221,6 +223,11 @@ class SpecDecodeBaseProposer:
223
if self.uses_mrope:
224
self.mrope_positions[:, :num_tokens] = positions
225
else:
226
+ # Convert M-RoPE positions if target model uses M-RoPE
227
+ # but draft doesn't, For text inputs, all M-RoPE
228
+ # dimensions are identical
229
+ if self.vllm_config.model_config.uses_mrope:
230
+ positions = positions[0]
231
self.positions[:num_tokens] = positions
232
233
def initialize_cudagraph_keys(self, cudagraph_mode: CUDAGraphMode) -> None:
@@ -1080,6 +1087,7 @@ class SpecDecodeBaseProposer:
1087
if self.get_model_name(target_model) in [
1088
"Qwen2_5_VLForConditionalGeneration",
1089
"Qwen3VLForConditionalGeneration",
1090
+ "Qwen3VLMoeForConditionalGeneration",
1091
]:
1092
self.model.config.image_token_index = target_model.config.image_token_id
1093
elif self.get_model_name(target_model) == "PixtralForConditionalGeneration":