A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Misc] Take user preference in attention selector (#4960)
Cody Yu committed
May 22, 2024 at 15:55 UTC
ee3eea0a1b2c690557455d97074d8829d5a98320
3 files changed
+169
-61
tests/kernels/test_attention_selector.py
new
+84
@@ -0,0 +1,84 @@
1
+import os
2
+from unittest.mock import patch
3
+
4
+import pytest
5
+import torch
6
+
7
+from vllm.attention.selector import which_attn_to_use
8
+
9
+
10
+@pytest.mark.parametrize(
11
+ "name", ["TORCH_SDPA", "ROCM_FLASH", "XFORMERS", "FLASHINFER"])
12
+@pytest.mark.parametrize("device", ["cpu", "hip", "cuda"])
13
+def test_env(name: str, device: str):
14
+ """Test that the attention selector can be set via environment variable.
15
+ Note that we do not test FlashAttn because it is the default backend.
16
+ """
17
+ name_backup = os.environ.get("VLLM_ATTENTION_BACKEND", None)
18
+ os.environ["VLLM_ATTENTION_BACKEND"] = name
19
+
20
+ if device == "cpu":
21
+ with patch("vllm.attention.selector.is_cpu", return_value=True):
22
+ backend = which_attn_to_use(8, 16, 8, None, torch.float16,
23
+ torch.float16, 16)
24
+ assert backend.name == "TORCH_SDPA"
25
+ elif device == "hip":
26
+ with patch("vllm.attention.selector.is_hip", return_value=True):
27
+ backend = which_attn_to_use(8, 16, 8, None, torch.float16,
28
+ torch.float16, 16)
29
+ assert backend.name == "ROCM_FLASH"
30
+ else:
31
+ backend = which_attn_to_use(8, 16, 8, None, torch.float16,
32
+ torch.float16, 16)
33
+ assert backend.name == name
34
+
35
+ if name_backup is not None:
36
+ os.environ["VLLM_ATTENTION_BACKEND"] = name_backup
37
+
38
+
39
+def test_flash_attn():
40
+ """Test FlashAttn validation."""
41
+ name_backup = os.environ.get("VLLM_ATTENTION_BACKEND", None)
42
+ os.environ["VLLM_ATTENTION_BACKEND"] = "FLASH_ATTN"
43
+
44
+ # Unsupported CUDA arch
45
+ with patch("torch.cuda.get_device_capability", return_value=[7, 5]):
46
+ backend = which_attn_to_use(8, 16, 8, None, torch.float16, None, 16)
47
+ assert backend.name != "FLASH_ATTN"
48
+
49
+ # Unsupported data type
50
+ backend = which_attn_to_use(8, 16, 8, None, torch.float8_e4m3fn, None, 16)
51
+ assert backend.name != "FLASH_ATTN"
52
+
53
+ # Unsupported kv cache data type
54
+ backend = which_attn_to_use(8, 16, 8, None, torch.float16, "fp8", 16)
55
+ assert backend.name != "FLASH_ATTN"
56
+
57
+ # Unsupported block size
58
+ backend = which_attn_to_use(8, 16, 8, None, torch.float16, None, 8)
59
+ assert backend.name != "FLASH_ATTN"
60
+
61
+ # Unsupported sliding window
62
+ backend = which_attn_to_use(8, 16, 8, 1, torch.float16, None, 16)
63
+ assert backend.name != "FLASH_ATTN"
64
+
65
+ # flash-attn is not installed
66
+ with patch.dict('sys.modules', {'vllm_flash_attn': None}):
67
+ backend = which_attn_to_use(8, 16, 8, None, torch.float16, None, 16)
68
+ assert backend.name != "FLASH_ATTN"
69
+
70
+ # Unsupported head size
71
+ backend = which_attn_to_use(8, 17, 8, None, torch.float16, None, 16)
72
+ assert backend.name != "FLASH_ATTN"
73
+
74
+ if name_backup is not None:
75
+ os.environ["VLLM_ATTENTION_BACKEND"] = name_backup
76
+
77
+
78
+def test_invalid_env():
79
+ """Throw an exception if the backend name is invalid."""
80
+ name_backup = os.environ.get("VLLM_ATTENTION_BACKEND", None)
81
+ os.environ["VLLM_ATTENTION_BACKEND"] = "INVALID"
82
+ with pytest.raises(ValueError):
83
+ which_attn_to_use(8, 16, 8, None, torch.float16, None, 16)
84
+ os.environ["VLLM_ATTENTION_BACKEND"] = name_backup
vllm/attention/backends/flashinfer.py
+1
@@ -218,6 +218,7 @@ class FlashInferImpl(AttentionImpl):
218
)
219
220
if prefill_meta := attn_metadata.prefill_metadata:
221
+ # Prompt run.
222
assert prefill_meta.block_tables is not None
223
if kv_cache is None or prefill_meta.block_tables.numel() == 0:
224
output = flash_attn_varlen_func(
vllm/attention/selector.py
+84
-61
@@ -30,24 +30,16 @@ def get_attn_backend(
30
kv_cache_dtype: Optional[str],
31
block_size: int,
32
) -> Type[AttentionBackend]:
33
- backend = _which_attn_to_use(num_heads, head_size, num_kv_heads,
34
- sliding_window, dtype, kv_cache_dtype,
35
- block_size)
33
+ """Determine which attention backend to use and only import
34
+ the selected backend module.
35
+ """
36
+ backend = which_attn_to_use(num_heads, head_size, num_kv_heads,
37
+ sliding_window, dtype, kv_cache_dtype,
38
+ block_size)
39
if backend == _Backend.FLASH_ATTN:
40
from vllm.attention.backends.flash_attn import ( # noqa: F401
41
FlashAttentionBackend)
39
-
40
- # We check it here not in _which_attn_to_use because we cannot know
41
- # the head size until we import FlashAttentionBackend.
42
- supported_head_sizes = FlashAttentionBackend.get_supported_head_sizes()
43
- if head_size in supported_head_sizes:
44
- logger.info("Using FlashAttention-2 backend.")
45
- return FlashAttentionBackend
46
- logger.info(
47
- "Cannot use FlashAttention-2 backend for head size %d. "
48
- "Using XFormers backend instead.", head_size)
49
- backend = _Backend.XFORMERS
50
-
42
+ return FlashAttentionBackend
43
if backend == _Backend.XFORMERS:
44
logger.info("Using XFormers backend.")
45
from vllm.attention.backends.xformers import ( # noqa: F401
@@ -64,14 +56,15 @@ def get_attn_backend(
56
return TorchSDPABackend
57
elif backend == _Backend.FLASHINFER:
58
logger.info("Using Flashinfer backend.")
67
- logger.warning("Eager mode is enforced for the Flashinfer backend.")
59
+ logger.warning("Eager mode is required for the Flashinfer backend. "
60
+ "Please make sure --enforce-eager is set.")
61
from vllm.attention.backends.flashinfer import FlashInferBackend
62
return FlashInferBackend
63
else:
64
raise ValueError("Invalid attention backend.")
65
66
74
-def _which_attn_to_use(
67
+def which_attn_to_use(
68
num_heads: int,
69
head_size: int,
70
num_kv_heads: int,
@@ -81,54 +74,84 @@ def _which_attn_to_use(
74
block_size: int,
75
) -> _Backend:
76
"""Returns which flash attention backend to use."""
77
+
78
+ # Default case.
79
+ selected_backend = _Backend.FLASH_ATTN
80
+
81
+ # Check the environment variable and override if specified
82
+ backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND
83
+ if backend_by_env_var is not None:
84
+ backend_members = _Backend.__members__
85
+ if backend_by_env_var not in backend_members:
86
+ raise ValueError(
87
+ f"Invalid attention backend '{backend_by_env_var}'. "
88
+ f"Available backends: {', '.join(backend_members)} "
89
+ "(case-sensitive).")
90
+ selected_backend = _Backend[backend_by_env_var]
91
+
92
if is_cpu():
93
+ if selected_backend != _Backend.TORCH_SDPA:
94
+ logger.info("Cannot use %s backend on CPU.", selected_backend)
95
return _Backend.TORCH_SDPA
96
97
if is_hip():
98
# AMD GPUs.
89
- if torch.cuda.get_device_capability()[0] != 9:
90
- # not Instinct series GPUs.
91
- logger.info("flash_atten is not supported on NAVI GPUs.")
99
+ selected_backend = (_Backend.ROCM_FLASH if selected_backend
100
+ == _Backend.FLASH_ATTN else selected_backend)
101
+ if selected_backend == _Backend.ROCM_FLASH:
102
+ if torch.cuda.get_device_capability()[0] != 9:
103
+ # not Instinct series GPUs.
104
+ logger.info("flash_attn is not supported on NAVI GPUs.")
105
+ else:
106
+ logger.info("%s is not supported in AMD GPUs.", selected_backend)
107
return _Backend.ROCM_FLASH
108
94
- # NVIDIA GPUs.
95
- if torch.cuda.get_device_capability()[0] < 8:
96
- # Volta and Turing NVIDIA GPUs.
97
- logger.info("Cannot use FlashAttention-2 backend for Volta and Turing "
98
- "GPUs.")
99
- return _Backend.XFORMERS
100
-
101
- if dtype not in (torch.float16, torch.bfloat16):
102
- logger.info("Cannot use FlashAttention-2 backend for dtype other than "
103
- "torch.float16 or torch.bfloat16.")
104
- return _Backend.XFORMERS
105
-
106
- if kv_cache_dtype is not None and kv_cache_dtype.startswith("fp8"):
107
- logger.info("Cannot use FlashAttention-2 backend for FP8 KV cache.")
108
- return _Backend.XFORMERS
109
-
110
- if block_size % 16 != 0:
111
- logger.info("Cannot use FlashAttention-2 backend for block size not "
112
- "divisible by 16.")
113
- return _Backend.XFORMERS
114
-
115
- if sliding_window is not None:
116
- logger.info(
117
- "Cannot use FlashAttention-2 backend due to sliding window.")
118
- return _Backend.XFORMERS
119
-
120
- try:
121
- import vllm_flash_attn # noqa: F401
122
- except ImportError:
123
- logger.info(
124
- "Cannot use FlashAttention-2 backend because the vllm_flash_attn "
125
- "package is not found. `pip install vllm-flash-attn` for better "
126
- "performance.")
127
- return _Backend.XFORMERS
128
-
129
- backend_by_env_var = envs.VLLM_ATTENTION_BACKEND
130
- if backend_by_env_var is not None:
131
- return _Backend[backend_by_env_var]
132
-
133
- # Default case.
134
- return _Backend.FLASH_ATTN
109
+ # FlashAttn in NVIDIA GPUs.
110
+ if selected_backend == _Backend.FLASH_ATTN:
111
+ if torch.cuda.get_device_capability()[0] < 8:
112
+ # Volta and Turing NVIDIA GPUs.
113
+ logger.info(
114
+ "Cannot use FlashAttention-2 backend for Volta and Turing "
115
+ "GPUs.")
116
+ selected_backend = _Backend.XFORMERS
117
+ elif dtype not in (torch.float16, torch.bfloat16):
118
+ logger.info(
119
+ "Cannot use FlashAttention-2 backend for dtype other than "
120
+ "torch.float16 or torch.bfloat16.")
121
+ selected_backend = _Backend.XFORMERS
122
+ elif kv_cache_dtype is not None and kv_cache_dtype.startswith("fp8"):
123
+ logger.info(
124
+ "Cannot use FlashAttention-2 backend for FP8 KV cache.")
125
+ selected_backend = _Backend.XFORMERS
126
+ elif block_size % 16 != 0:
127
+ logger.info(
128
+ "Cannot use FlashAttention-2 backend for block size not "
129
+ "divisible by 16.")
130
+ selected_backend = _Backend.XFORMERS
131
+ elif sliding_window is not None:
132
+ logger.info(
133
+ "Cannot use FlashAttention-2 backend due to sliding window.")
134
+ selected_backend = _Backend.XFORMERS
135
+
136
+ # FlashAttn is valid for the model, checking if the package is installed.
137
+ if selected_backend == _Backend.FLASH_ATTN:
138
+ try:
139
+ import vllm_flash_attn # noqa: F401
140
+
141
+ from vllm.attention.backends.flash_attn import ( # noqa: F401
142
+ FlashAttentionBackend)
143
+
144
+ supported_sizes = FlashAttentionBackend.get_supported_head_sizes()
145
+ if head_size not in supported_sizes:
146
+ logger.info(
147
+ "Cannot use FlashAttention-2 backend for head size %d.",
148
+ head_size)
149
+ selected_backend = _Backend.XFORMERS
150
+ except ImportError:
151
+ logger.info(
152
+ "Cannot use FlashAttention-2 backend because the "
153
+ "vllm_flash_attn package is not found. "
154
+ "`pip install vllm-flash-attn` for better performance.")
155
+ selected_backend = _Backend.XFORMERS
156
+
157
+ return selected_backend