A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Misc] Fix docstring of get_attn_backend (#5271)
Woosuk Kwon committed
Jun 5, 2024 at 09:18 UTC
c65146e75e71f72bdb93542efdc40c87caccb42e
1 file changed
+2
-3
vllm/attention/selector.py
+2
-3
@@ -31,15 +31,14 @@ def get_attn_backend(
31
block_size: int,
32
is_blocksparse: bool = False,
33
) -> Type[AttentionBackend]:
34
+ """Selects which attention backend to use and lazily imports it."""
35
36
if is_blocksparse:
37
logger.info("Using BlocksparseFlashAttention backend.")
38
from vllm.attention.backends.blocksparse_attn import (
39
BlocksparseFlashAttentionBackend)
40
return BlocksparseFlashAttentionBackend
40
- """Determine which attention backend to use and only import
41
- the selected backend module.
42
- """
41
+
42
backend = which_attn_to_use(num_heads, head_size, num_kv_heads,
43
sliding_window, dtype, kv_cache_dtype,
44
block_size)