@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c8a7d51c49
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix] Update flashinfer.py with PagedAttention forwards - Fixes Gemma2 OpenAI Server Crash (#6501)

Noam Gat committed Jul 18, 2024 at 10:47 UTC c8a7d51c4982b7b425debe5473867d9983e728fd
1 file changed +3 -2
vllm/attention/backends/flashinfer.py
+3 -2
@@ -20,6 +20,7 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
20 from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
21 compute_slot_mapping_start_idx,
22 is_block_tables_empty)
23 +from vllm.attention.ops.paged_attn import PagedAttention
24 from vllm.sequence import SequenceGroupMetadata
25 from vllm.utils import get_kv_cache_torch_dtype, make_tensor_with_pad
26
@@ -61,14 +62,14 @@ class FlashInferBackend(AttentionBackend):
62 dst_kv_cache: torch.Tensor,
63 src_to_dst: torch.Tensor,
64 ) -> None:
64 - raise NotImplementedError
65 + PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
66
67 @staticmethod
68 def copy_blocks(
69 kv_caches: List[torch.Tensor],
70 src_to_dists: torch.Tensor,
71 ) -> None:
71 - raise NotImplementedError
72 + PagedAttention.copy_blocks(kv_caches, src_to_dists)
73
74 @staticmethod
75 def get_supported_head_sizes() -> List[int]: