A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Bugfix] lookahead block table with cuda graph max capture (#8340)
[Bugfix] Ensure multistep lookahead allocation is compatible with cuda graph max capture (#8340)
Alexander Matveev committed
Sep 10, 2024 at 19:00 UTC
22f3a4bc6c6801101728d97edd25ffcdd5a7fd8c
1 file changed
+11
-1
vllm/attention/backends/flash_attn.py
+11
-1
@@ -471,9 +471,19 @@ class FlashAttentionMetadataBuilder(
471
# The shape of graph_block_tables is
472
# [max batch size, max context len // block size].
473
input_block_tables = self.runner.graph_block_tables[:batch_size]
474
+ max_blocks = input_block_tables.shape[1]
475
for i, block_table in enumerate(self.block_tables):
476
if block_table:
476
- input_block_tables[i, :len(block_table)] = block_table
477
+ num_blocks = len(block_table)
478
+ if num_blocks <= max_blocks:
479
+ input_block_tables[i, :num_blocks] = block_table
480
+ else:
481
+ # It may be possible to have more blocks allocated due
482
+ # to lookahead slots of multi-step, however, they are
483
+ # not used anyway, so can be safely ignored.
484
+ input_block_tables[
485
+ i, :max_blocks] = block_table[:max_blocks]
486
+
487
block_tables = torch.from_numpy(input_block_tables).to(
488
device=device, non_blocking=True)
489
else: