A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Misc] Minor type annotation fix (#3716)
Woosuk Kwon committed
Mar 28, 2024 at 21:12 UTC
395aa823ea456ef4f4677b7a43c37806307be2bc
1 file changed
+2
-1
vllm/attention/selector.py
+2
-1
@@ -1,4 +1,5 @@
1
from functools import lru_cache
2
+from typing import Type
3
4
import torch
5
@@ -10,7 +11,7 @@ logger = init_logger(__name__)
11
12
13
@lru_cache(maxsize=None)
13
-def get_attn_backend(dtype: torch.dtype) -> AttentionBackend:
14
+def get_attn_backend(dtype: torch.dtype) -> Type[AttentionBackend]:
15
if _can_use_flash_attn(dtype):
16
logger.info("Using FlashAttention backend.")
17
from vllm.attention.backends.flash_attn import ( # noqa: F401