A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 20:45 UTC
[Bugfix] Apply fc_norm in Eagle3DeepseekV2 combine_hidden_states (#43482)
Signed-off-by: Yubo Wang <yubowang2019@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
Yubo Wang committed
May 26, 2026 at 00:46 UTC
97e4022c6ccb7b2cf1a1fc0a13a17a2a06d74f0d
1 file changed
+6
vllm/model_executor/models/deepseek_eagle3.py
+6
@@ -407,6 +407,12 @@ class Eagle3DeepseekV2ForCausalLM(DeepseekV2ForCausalLM):
407
hidden_states: torch.Tensor,
408
) -> torch.Tensor:
409
# Combine multiple auxiliary hidden states returned by Eagle3
410
+ if self.model.fc_norm is not None:
411
+ chunks = hidden_states.chunk(self.model.num_aux_hidden_states, dim=-1)
412
+ hidden_states = torch.cat(
413
+ [norm(chunk) for norm, chunk in zip(self.model.fc_norm, chunks)],
414
+ dim=-1,
415
+ )
416
return self.model.fc(hidden_states)
417
418
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):