A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Bugfix] multi-step + flashinfer: ensure cuda graph compatible (#8427)
Alexander Matveev committed
Sep 12, 2024 at 17:01 UTC
019877253be473bf0c12daaf2c29022150402052
1 file changed
+11
-1
vllm/attention/backends/flashinfer.py
+11
-1
@@ -597,9 +597,19 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
597
# The shape of graph_block_tables is
598
# [max batch size, max context len // block size].
599
input_block_tables = self.runner.graph_block_tables[:batch_size]
600
+ max_blocks = input_block_tables.shape[1]
601
for i, block_table in enumerate(self.block_tables):
602
if block_table:
602
- input_block_tables[i, :len(block_table)] = block_table
603
+ num_blocks = len(block_table)
604
+ if num_blocks <= max_blocks:
605
+ input_block_tables[i, :num_blocks] = block_table
606
+ else:
607
+ # It may be possible to have more blocks allocated due
608
+ # to lookahead slots of multi-step, however, they are
609
+ # not used anyway, so can be safely ignored.
610
+ input_block_tables[
611
+ i, :max_blocks] = block_table[:max_blocks]
612
+
613
block_tables = torch.from_numpy(input_block_tables).to(
614
device, non_blocking=True)
615