A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 06:00 UTC
[Bugfix] Fix tensor shape mismatch in sparse attention with speculative decoding (#39542)
Signed-off-by: Santino Ramos <santinor@inferact.ai>
Santino Ramos committed
Apr 13, 2026 at 08:57 UTC
4d042ed85f3b8fe8c73b6a4a365e65cf1811823e
1 file changed
+2
-2
vllm/model_executor/layers/sparse_attn_indexer.py
+2
-2
@@ -213,7 +213,7 @@ def sparse_attn_indexer(
213
)
214
torch.ops._C.persistent_topk(
215
logits,
216
- decode_metadata.seq_lens,
216
+ seq_lens,
217
topk_indices,
218
topk_workspace,
219
topk_tokens,
@@ -250,7 +250,7 @@ def sparse_attn_indexer(
250
topk_indices.reshape(batch_size, -1, topk_indices.shape[-1]),
251
decode_lens,
252
)
253
- topk_indices_buffer[:num_decode_tokens, : topk_indices.shape[-1]] = (
253
+ topk_indices_buffer[: topk_indices.shape[0], : topk_indices.shape[-1]] = (
254
topk_indices
255
)
256