A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bugfix] Update flashinfer.py with PagedAttention forwards - Fixes Gemma2 OpenAI Server Crash (#6501)
Noam Gat committed
Jul 18, 2024 at 10:47 UTC
c8a7d51c4982b7b425debe5473867d9983e728fd
1 file changed
+3
-2
vllm/attention/backends/flashinfer.py
+3
-2
@@ -20,6 +20,7 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
20
from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
21
compute_slot_mapping_start_idx,
22
is_block_tables_empty)
23
+from vllm.attention.ops.paged_attn import PagedAttention
24
from vllm.sequence import SequenceGroupMetadata
25
from vllm.utils import get_kv_cache_torch_dtype, make_tensor_with_pad
26
@@ -61,14 +62,14 @@ class FlashInferBackend(AttentionBackend):
62
dst_kv_cache: torch.Tensor,
63
src_to_dst: torch.Tensor,
64
) -> None:
64
- raise NotImplementedError
65
+ PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
66
67
@staticmethod
68
def copy_blocks(
69
kv_caches: List[torch.Tensor],
70
src_to_dists: torch.Tensor,
71
) -> None:
71
- raise NotImplementedError
72
+ PagedAttention.copy_blocks(kv_caches, src_to_dists)
73
74
@staticmethod
75
def get_supported_head_sizes() -> List[int]: