A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Bugfix][Kernel] Add head size check for attention backend selection (#4944)
Isotr0py committed
May 22, 2024 at 03:33 UTC
99eff67ba9155b5fec9a9abd939e3a29a1b42dce
2 files changed
+21
-7
vllm/attention/backends/flash_attn.py
+8
-4
@@ -9,11 +9,13 @@ from vllm._C import cache_ops
9
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10
AttentionMetadata)
11
12
-_SUPPORTED_HEAD_SIZES = [32, 64, 96, 128, 160, 192, 224, 256]
13
-
12
13
class FlashAttentionBackend(AttentionBackend):
14
15
+ @staticmethod
16
+ def get_supported_head_sizes() -> List[int]:
17
+ return [32, 64, 96, 128, 160, 192, 224, 256]
18
+
19
@staticmethod
20
def get_name() -> str:
21
return "flash-attn"
@@ -237,10 +239,12 @@ class FlashAttentionImpl(AttentionImpl):
239
# paged KV cache.
240
raise ValueError(
241
"Sliding window is not supported in FlashAttention.")
240
- if head_size not in _SUPPORTED_HEAD_SIZES:
242
+
243
+ support_head_sizes = FlashAttentionBackend.get_supported_head_sizes()
244
+ if head_size not in support_head_sizes:
245
raise ValueError(
246
f"Head size {head_size} is not supported by FlashAttention. "
243
- f"Supported head sizes are: {_SUPPORTED_HEAD_SIZES}.")
247
+ f"Supported head sizes are: {support_head_sizes}.")
248
249
def forward(
250
self,
vllm/attention/selector.py
+13
-3
@@ -34,11 +34,21 @@ def get_attn_backend(
34
sliding_window, dtype, kv_cache_dtype,
35
block_size)
36
if backend == _Backend.FLASH_ATTN:
37
- logger.info("Using FlashAttention-2 backend.")
37
from vllm.attention.backends.flash_attn import ( # noqa: F401
38
FlashAttentionBackend)
40
- return FlashAttentionBackend
41
- elif backend == _Backend.XFORMERS:
39
+
40
+ # We check it here not in _which_attn_to_use because we cannot know
41
+ # the head size until we import FlashAttentionBackend.
42
+ supported_head_sizes = FlashAttentionBackend.get_supported_head_sizes()
43
+ if head_size in supported_head_sizes:
44
+ logger.info("Using FlashAttention-2 backend.")
45
+ return FlashAttentionBackend
46
+ logger.info(
47
+ "Cannot use FlashAttention-2 backend for head size %d. "
48
+ "Using XFormers backend instead.", head_size)
49
+ backend = _Backend.XFORMERS
50
+
51
+ if backend == _Backend.XFORMERS:
52
logger.info("Using XFormers backend.")
53
from vllm.attention.backends.xformers import ( # noqa: F401
54
XFormersBackend)