@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b6dcb4d442
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Misc] Fix flash attention backend log (#4368)

Roy committed Apr 26, 2024 at 03:43 UTC b6dcb4d44281a9e85cafcfa6376c373d02286779
1 file changed +5 -5
vllm/attention/selector.py
+5 -5
@@ -25,7 +25,7 @@ class _Backend(enum.Enum):
25 def get_attn_backend(dtype: torch.dtype) -> Type[AttentionBackend]:
26 backend = _which_attn_to_use(dtype)
27 if backend == _Backend.FLASH_ATTN:
28 - logger.info("Using FlashAttention backend.")
28 + logger.info("Using FlashAttention-2 backend.")
29 from vllm.attention.backends.flash_attn import ( # noqa: F401
30 FlashAttentionBackend)
31 return FlashAttentionBackend
@@ -62,12 +62,12 @@ def _which_attn_to_use(dtype: torch.dtype) -> _Backend:
62 # NVIDIA GPUs.
63 if torch.cuda.get_device_capability()[0] < 8:
64 # Volta and Turing NVIDIA GPUs.
65 - logger.info("Cannot use FlashAttention backend for Volta and Turing "
65 + logger.info("Cannot use FlashAttention-2 backend for Volta and Turing "
66 "GPUs.")
67 return _Backend.XFORMERS
68
69 if dtype not in (torch.float16, torch.bfloat16):
70 - logger.info("Cannot use FlashAttention backend for dtype other than "
70 + logger.info("Cannot use FlashAttention-2 backend for dtype other than "
71 "torch.float16 or torch.bfloat16.")
72 return _Backend.XFORMERS
73
@@ -75,8 +75,8 @@ def _which_attn_to_use(dtype: torch.dtype) -> _Backend:
75 import flash_attn # noqa: F401
76 except ImportError:
77 logger.info(
78 - "Cannot use FlashAttention backend because the flash_attn package "
79 - "is not found. Please install it for better performance.")
78 + "Cannot use FlashAttention-2 backend because the flash_attn "
79 + "package is not found. Please install it for better performance.")
80 return _Backend.XFORMERS
81
82 backend_by_env_var = os.getenv(VLLM_ATTENTION_BACKEND)