@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 22f3a4bc6c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Bugfix] lookahead block table with cuda graph max capture (#8340)

[Bugfix] Ensure multistep lookahead allocation is compatible with cuda graph max capture (#8340)

Alexander Matveev committed Sep 10, 2024 at 19:00 UTC 22f3a4bc6c6801101728d97edd25ffcdd5a7fd8c
1 file changed +11 -1
vllm/attention/backends/flash_attn.py
+11 -1
@@ -471,9 +471,19 @@ class FlashAttentionMetadataBuilder(
471 # The shape of graph_block_tables is
472 # [max batch size, max context len // block size].
473 input_block_tables = self.runner.graph_block_tables[:batch_size]
474 + max_blocks = input_block_tables.shape[1]
475 for i, block_table in enumerate(self.block_tables):
476 if block_table:
476 - input_block_tables[i, :len(block_table)] = block_table
477 + num_blocks = len(block_table)
478 + if num_blocks <= max_blocks:
479 + input_block_tables[i, :num_blocks] = block_table
480 + else:
481 + # It may be possible to have more blocks allocated due
482 + # to lookahead slots of multi-step, however, they are
483 + # not used anyway, so can be safely ignored.
484 + input_block_tables[
485 + i, :max_blocks] = block_table[:max_blocks]
486 +
487 block_tables = torch.from_numpy(input_block_tables).to(
488 device=device, non_blocking=True)
489 else: