A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Misc] Fix flash attention backend log (#4368)
Roy committed
Apr 26, 2024 at 03:43 UTC
b6dcb4d44281a9e85cafcfa6376c373d02286779
1 file changed
+5
-5
vllm/attention/selector.py
+5
-5
@@ -25,7 +25,7 @@ class _Backend(enum.Enum):
25
def get_attn_backend(dtype: torch.dtype) -> Type[AttentionBackend]:
26
backend = _which_attn_to_use(dtype)
27
if backend == _Backend.FLASH_ATTN:
28
- logger.info("Using FlashAttention backend.")
28
+ logger.info("Using FlashAttention-2 backend.")
29
from vllm.attention.backends.flash_attn import ( # noqa: F401
30
FlashAttentionBackend)
31
return FlashAttentionBackend
@@ -62,12 +62,12 @@ def _which_attn_to_use(dtype: torch.dtype) -> _Backend:
62
# NVIDIA GPUs.
63
if torch.cuda.get_device_capability()[0] < 8:
64
# Volta and Turing NVIDIA GPUs.
65
- logger.info("Cannot use FlashAttention backend for Volta and Turing "
65
+ logger.info("Cannot use FlashAttention-2 backend for Volta and Turing "
66
"GPUs.")
67
return _Backend.XFORMERS
68
69
if dtype not in (torch.float16, torch.bfloat16):
70
- logger.info("Cannot use FlashAttention backend for dtype other than "
70
+ logger.info("Cannot use FlashAttention-2 backend for dtype other than "
71
"torch.float16 or torch.bfloat16.")
72
return _Backend.XFORMERS
73
@@ -75,8 +75,8 @@ def _which_attn_to_use(dtype: torch.dtype) -> _Backend:
75
import flash_attn # noqa: F401
76
except ImportError:
77
logger.info(
78
- "Cannot use FlashAttention backend because the flash_attn package "
79
- "is not found. Please install it for better performance.")
78
+ "Cannot use FlashAttention-2 backend because the flash_attn "
79
+ "package is not found. Please install it for better performance.")
80
return _Backend.XFORMERS
81
82
backend_by_env_var = os.getenv(VLLM_ATTENTION_BACKEND)