@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 019877253b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Bugfix] multi-step + flashinfer: ensure cuda graph compatible (#8427)

Alexander Matveev committed Sep 12, 2024 at 17:01 UTC 019877253be473bf0c12daaf2c29022150402052
1 file changed +11 -1
vllm/attention/backends/flashinfer.py
+11 -1
@@ -597,9 +597,19 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
597 # The shape of graph_block_tables is
598 # [max batch size, max context len // block size].
599 input_block_tables = self.runner.graph_block_tables[:batch_size]
600 + max_blocks = input_block_tables.shape[1]
601 for i, block_table in enumerate(self.block_tables):
602 if block_table:
602 - input_block_tables[i, :len(block_table)] = block_table
603 + num_blocks = len(block_table)
604 + if num_blocks <= max_blocks:
605 + input_block_tables[i, :num_blocks] = block_table
606 + else:
607 + # It may be possible to have more blocks allocated due
608 + # to lookahead slots of multi-step, however, they are
609 + # not used anyway, so can be safely ignored.
610 + input_block_tables[
611 + i, :max_blocks] = block_table[:max_blocks]
612 +
613 block_tables = torch.from_numpy(input_block_tables).to(
614 device, non_blocking=True)
615