@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c65146e75e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Misc] Fix docstring of get_attn_backend (#5271)

Woosuk Kwon committed Jun 5, 2024 at 09:18 UTC c65146e75e71f72bdb93542efdc40c87caccb42e
1 file changed +2 -3
vllm/attention/selector.py
+2 -3
@@ -31,15 +31,14 @@ def get_attn_backend(
31 block_size: int,
32 is_blocksparse: bool = False,
33 ) -> Type[AttentionBackend]:
34 + """Selects which attention backend to use and lazily imports it."""
35
36 if is_blocksparse:
37 logger.info("Using BlocksparseFlashAttention backend.")
38 from vllm.attention.backends.blocksparse_attn import (
39 BlocksparseFlashAttentionBackend)
40 return BlocksparseFlashAttentionBackend
40 - """Determine which attention backend to use and only import
41 - the selected backend module.
42 - """
41 +
42 backend = which_attn_to_use(num_heads, head_size, num_kv_heads,
43 sliding_window, dtype, kv_cache_dtype,
44 block_size)