A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Fix DeepSeek MTP (#22934)
Signed-off-by: Benjamin Chislett <benjamin.chislett@centml.ai>
Benjamin Chislett committed
Aug 15, 2025 at 21:25 UTC
fbd88728b3aa7add999529e2c3f1b6b0aa8e428d
3 files changed
+13
-14
vllm/model_executor/models/deepseek_mtp.py
+7
-6
@@ -158,14 +158,13 @@ class DeepSeekMTP(nn.Module, SupportsPP):
158
self,
159
input_ids: torch.Tensor,
160
positions: torch.Tensor,
161
- previous_hidden_states: torch.Tensor,
161
+ hidden_states: torch.Tensor,
162
intermediate_tensors: Optional[IntermediateTensors] = None,
163
inputs_embeds: Optional[torch.Tensor] = None,
164
spec_step_idx: int = 0,
165
) -> torch.Tensor:
166
- hidden_states = self.model(input_ids, positions,
167
- previous_hidden_states, inputs_embeds,
168
- spec_step_idx)
166
+ hidden_states = self.model(input_ids, positions, hidden_states,
167
+ inputs_embeds, spec_step_idx)
168
return hidden_states
169
170
def compute_logits(
@@ -213,13 +212,15 @@ class DeepSeekMTP(nn.Module, SupportsPP):
212
# for mlp.experts[0].gate_gate_up_proj, which breaks load.
213
if (("mlp.experts." in name) and name not in params_dict):
214
continue
216
- name = name.replace(weight_name, param_name)
215
+ name_mapped = name.replace(weight_name, param_name)
216
217
# QKV fusion is optional, fall back to normal
218
# weight loading if it's not enabled
219
if ((param_name == "fused_qkv_a_proj")
221
- and name not in params_dict):
220
+ and name_mapped not in params_dict):
221
continue
222
+ else:
223
+ name = name_mapped
224
225
# Skip loading extra bias for GPTQ models.
226
if name.endswith(".bias") and name not in params_dict:
vllm/model_executor/models/glm4_moe_mtp.py
+3
-4
@@ -180,14 +180,13 @@ class Glm4MoeMTP(nn.Module, SupportsPP):
180
self,
181
input_ids: torch.Tensor,
182
positions: torch.Tensor,
183
- previous_hidden_states: torch.Tensor,
183
+ hidden_states: torch.Tensor,
184
intermediate_tensors: Optional[IntermediateTensors] = None,
185
inputs_embeds: Optional[torch.Tensor] = None,
186
spec_step_idx: int = 0,
187
) -> torch.Tensor:
188
- hidden_states = self.model(input_ids, positions,
189
- previous_hidden_states, inputs_embeds,
190
- spec_step_idx)
188
+ hidden_states = self.model(input_ids, positions, hidden_states,
189
+ inputs_embeds, spec_step_idx)
190
return hidden_states
191
192
def compute_logits(
vllm/model_executor/models/mimo_mtp.py
+3
-4
@@ -164,15 +164,14 @@ class MiMoMTP(nn.Module):
164
self,
165
input_ids: torch.Tensor,
166
positions: torch.Tensor,
167
- previous_hidden_states: torch.Tensor,
167
+ hidden_states: torch.Tensor,
168
intermediate_tensors: Optional[IntermediateTensors] = None,
169
inputs_embeds: Optional[torch.Tensor] = None,
170
spec_step_idx: int = 0,
171
) -> torch.Tensor:
172
assert spec_step_idx == 0, "mimo_mtp only support predict one token now"
173
- hidden_states = self.model(input_ids, positions,
174
- previous_hidden_states, inputs_embeds,
175
- spec_step_idx)
173
+ hidden_states = self.model(input_ids, positions, hidden_states,
174
+ inputs_embeds, spec_step_idx)
175
return hidden_states
176
177
def compute_logits(