@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 395aa823ea
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Misc] Minor type annotation fix (#3716)

Woosuk Kwon committed Mar 28, 2024 at 21:12 UTC 395aa823ea456ef4f4677b7a43c37806307be2bc
1 file changed +2 -1
vllm/attention/selector.py
+2 -1
@@ -1,4 +1,5 @@
1 from functools import lru_cache
2 +from typing import Type
3
4 import torch
5
@@ -10,7 +11,7 @@ logger = init_logger(__name__)
11
12
13 @lru_cache(maxsize=None)
13 -def get_attn_backend(dtype: torch.dtype) -> AttentionBackend:
14 +def get_attn_backend(dtype: torch.dtype) -> Type[AttentionBackend]:
15 if _can_use_flash_attn(dtype):
16 logger.info("Using FlashAttention backend.")
17 from vllm.attention.backends.flash_attn import ( # noqa: F401