A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 03:45 UTC
[Bug] Fix `error 'Dynamo failed to run FX node with fake tensors` for Deepseek V3.2 (#31046)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
Dec 19, 2025 at 18:31 UTC
4cf9429897c1a6c720a0f099a7a46e9d51af9342
1 file changed
+5
-2
vllm/model_executor/models/deepseek_v2.py
+5
-2
@@ -878,8 +878,11 @@ class Indexer(nn.Module):
878
)
879
880
q_pe, k_pe = rotary_emb(positions, q_pe, k_pe.unsqueeze(1))
881
- q = torch.cat([q_pe.squeeze(0), q_nope], dim=-1)
882
- k = torch.cat([k_pe.squeeze((0, 2)), k_nope], dim=-1)
881
+ # `rotary_emb` is shape-preserving; `q_pe` is already
882
+ # [num_tokens, n_head, rope_dim].
883
+ q = torch.cat([q_pe, q_nope], dim=-1)
884
+ # `k_pe` is [num_tokens, 1, rope_dim] (MQA).
885
+ k = torch.cat([k_pe.squeeze(1), k_nope], dim=-1)
886
887
# we only quant q here since k quant is fused with cache insertion
888
q = q.view(-1, self.head_dim)