A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[BugFix] Fix Llama4 - Index Error When Single Request Near Max Context (#16209)
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Lucas Wilkinson committed
Apr 8, 2025 at 14:56 UTC
e1a2c699dda82199e88e433c144eae66f3b31878
1 file changed
+1
-1
vllm/v1/attention/backends/flash_attn.py
+1
-1
@@ -264,7 +264,7 @@ def make_local_attention_virtual_batches(
264
np.arange(pages_per_local_batch, dtype=np.int32),
265
(virtual_batches, pages_per_local_batch)) \
266
+ np.expand_dims(block_starts, axis=1)
267
- block_indices = block_indices.flatten()
267
+ block_indices = block_indices.flatten().clip(max=block_table.shape[1] - 1)
268
batch_indices = np.repeat(np.arange(actual_batch_size, dtype=np.int32),
269
local_blocks * pages_per_local_batch)
270
block_table_local = block_table[batch_indices, block_indices]\