A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
[BugFix] Fix cascade attention - RuntimeError: scheduler_metadata must have shape (metadata_size) (#17283)
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Lucas Wilkinson committed
Apr 28, 2025 at 16:55 UTC
cc5befbced220b3bbb1a041a2d6aadd98472278d
1 file changed
+1
-1
vllm/v1/attention/backends/flash_attn.py
+1
-1
@@ -372,7 +372,7 @@ class FlashAttentionMetadataBuilder:
372
suffix_kv_lens = torch.from_numpy(suffix_kv_lens).to(
373
self.runner.device)
374
prefix_scheduler_metadata = schedule(
375
- batch_size=num_reqs,
375
+ batch_size=1,
376
cu_query_lens=cu_prefix_query_lens,
377
max_query_len=num_actual_tokens,
378
seqlens=prefix_kv_lens,