@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 99eff67ba9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Bugfix][Kernel] Add head size check for attention backend selection (#4944)

Isotr0py committed May 22, 2024 at 03:33 UTC 99eff67ba9155b5fec9a9abd939e3a29a1b42dce
2 files changed +21 -7
vllm/attention/backends/flash_attn.py
+8 -4
@@ -9,11 +9,13 @@ from vllm._C import cache_ops
9 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10 AttentionMetadata)
11
12 -_SUPPORTED_HEAD_SIZES = [32, 64, 96, 128, 160, 192, 224, 256]
13 -
12
13 class FlashAttentionBackend(AttentionBackend):
14
15 + @staticmethod
16 + def get_supported_head_sizes() -> List[int]:
17 + return [32, 64, 96, 128, 160, 192, 224, 256]
18 +
19 @staticmethod
20 def get_name() -> str:
21 return "flash-attn"
@@ -237,10 +239,12 @@ class FlashAttentionImpl(AttentionImpl):
239 # paged KV cache.
240 raise ValueError(
241 "Sliding window is not supported in FlashAttention.")
240 - if head_size not in _SUPPORTED_HEAD_SIZES:
242 +
243 + support_head_sizes = FlashAttentionBackend.get_supported_head_sizes()
244 + if head_size not in support_head_sizes:
245 raise ValueError(
246 f"Head size {head_size} is not supported by FlashAttention. "
243 - f"Supported head sizes are: {_SUPPORTED_HEAD_SIZES}.")
247 + f"Supported head sizes are: {support_head_sizes}.")
248
249 def forward(
250 self,
vllm/attention/selector.py
+13 -3
@@ -34,11 +34,21 @@ def get_attn_backend(
34 sliding_window, dtype, kv_cache_dtype,
35 block_size)
36 if backend == _Backend.FLASH_ATTN:
37 - logger.info("Using FlashAttention-2 backend.")
37 from vllm.attention.backends.flash_attn import ( # noqa: F401
38 FlashAttentionBackend)
40 - return FlashAttentionBackend
41 - elif backend == _Backend.XFORMERS:
39 +
40 + # We check it here not in _which_attn_to_use because we cannot know
41 + # the head size until we import FlashAttentionBackend.
42 + supported_head_sizes = FlashAttentionBackend.get_supported_head_sizes()
43 + if head_size in supported_head_sizes:
44 + logger.info("Using FlashAttention-2 backend.")
45 + return FlashAttentionBackend
46 + logger.info(
47 + "Cannot use FlashAttention-2 backend for head size %d. "
48 + "Using XFormers backend instead.", head_size)
49 + backend = _Backend.XFORMERS
50 +
51 + if backend == _Backend.XFORMERS:
52 logger.info("Using XFormers backend.")
53 from vllm.attention.backends.xformers import ( # noqa: F401
54 XFormersBackend)