A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Kernel] Add flash-attn back (#4907)
Woosuk Kwon committed
May 19, 2024 at 18:11 UTC
b57e6c59491ea7d60af413ad4a6455812b9c6c50
6 files changed
+304
-61
requirements-cuda.txt
+1
-1
@@ -7,4 +7,4 @@ nvidia-ml-py # for pynvml package
7
vllm-nccl-cu12>=2.18,<2.19 # for downloading nccl library
8
torch == 2.3.0
9
xformers == 0.0.26.post1 # Requires PyTorch 2.3.0
10
-vllm-flash-attn == 2.5.8.post1 # Requires PyTorch 2.3.0
10
+vllm-flash-attn == 2.5.8.post2 # Requires PyTorch 2.3.0
tests/kernels/test_flash_attn.py
new
+208
@@ -0,0 +1,208 @@
1
+from typing import List, Optional, Tuple
2
+
3
+import pytest
4
+import torch
5
+from vllm_flash_attn import flash_attn_varlen_func, flash_attn_with_kvcache
6
+
7
+NUM_HEADS = [(16, 16), (32, 8), (64, 8)]
8
+HEAD_SIZES = [128, 256]
9
+BLOCK_SIZES = [16, 32]
10
+DTYPES = [torch.float16, torch.bfloat16]
11
+NUM_BLOCKS = 32768 # Large enough to test overflow in index calculation.
12
+
13
+
14
+def ref_paged_attn(
15
+ query: torch.Tensor,
16
+ key_cache: torch.Tensor,
17
+ value_cache: torch.Tensor,
18
+ query_lens: List[int],
19
+ kv_lens: List[int],
20
+ block_tables: torch.Tensor,
21
+ scale: float,
22
+ sliding_window: Optional[int] = None,
23
+) -> torch.Tensor:
24
+ num_seqs = len(query_lens)
25
+ block_tables = block_tables.cpu().numpy()
26
+ _, block_size, num_kv_heads, head_size = key_cache.shape
27
+
28
+ outputs = []
29
+ start_idx = 0
30
+ for i in range(num_seqs):
31
+ query_len = query_lens[i]
32
+ kv_len = kv_lens[i]
33
+ q = query[start_idx:start_idx + query_len]
34
+ q *= scale
35
+
36
+ num_kv_blocks = (kv_len + block_size - 1) // block_size
37
+ block_indices = block_tables[i, :num_kv_blocks]
38
+
39
+ k = key_cache[block_indices].view(-1, num_kv_heads, head_size)
40
+ k = k[:kv_len]
41
+ v = value_cache[block_indices].view(-1, num_kv_heads, head_size)
42
+ v = v[:kv_len]
43
+
44
+ if q.shape[1] != k.shape[1]:
45
+ k = torch.repeat_interleave(k, q.shape[1] // k.shape[1], dim=1)
46
+ v = torch.repeat_interleave(v, q.shape[1] // v.shape[1], dim=1)
47
+ attn = torch.einsum("qhd,khd->hqk", q, k).float()
48
+ empty_mask = torch.ones(query_len, kv_len)
49
+ mask = torch.triu(empty_mask, diagonal=kv_len - query_len + 1).bool()
50
+ if sliding_window is not None:
51
+ sliding_window_mask = torch.triu(empty_mask,
52
+ diagonal=kv_len -
53
+ (query_len + sliding_window) +
54
+ 1).bool().logical_not()
55
+ mask |= sliding_window_mask
56
+ attn.masked_fill_(mask, float("-inf"))
57
+ attn = torch.softmax(attn, dim=-1).to(v.dtype)
58
+ out = torch.einsum("hqk,khd->qhd", attn, v)
59
+
60
+ outputs.append(out)
61
+ start_idx += query_len
62
+
63
+ return torch.cat(outputs, dim=0)
64
+
65
+
66
+@pytest.mark.parametrize("kv_lens", [[1328, 18, 463], [1, 54, 293, 70]])
67
+@pytest.mark.parametrize("num_heads", NUM_HEADS)
68
+@pytest.mark.parametrize("head_size", HEAD_SIZES)
69
+@pytest.mark.parametrize("block_size", BLOCK_SIZES)
70
+@pytest.mark.parametrize("dtype", DTYPES)
71
+@torch.inference_mode
72
+def test_flash_attn_with_paged_kv(
73
+ kv_lens: List[Tuple[int, int]],
74
+ num_heads: Tuple[int, int],
75
+ head_size: int,
76
+ dtype: torch.dtype,
77
+ block_size: int,
78
+) -> None:
79
+ torch.set_default_device("cuda")
80
+ torch.cuda.manual_seed_all(0)
81
+ num_seqs = len(kv_lens)
82
+ num_query_heads = num_heads[0]
83
+ num_kv_heads = num_heads[1]
84
+ assert num_query_heads % num_kv_heads == 0
85
+ max_kv_len = max(kv_lens)
86
+ scale = head_size**-0.5
87
+
88
+ query = torch.randn(num_seqs, num_query_heads, head_size, dtype=dtype)
89
+ key_cache = torch.randn(NUM_BLOCKS,
90
+ block_size,
91
+ num_kv_heads,
92
+ head_size,
93
+ dtype=dtype)
94
+ value_cache = torch.randn_like(key_cache)
95
+ kv_lens_tensor = torch.tensor(kv_lens, dtype=torch.int32)
96
+
97
+ max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
98
+ block_tables = torch.randint(0,
99
+ NUM_BLOCKS,
100
+ (num_seqs, max_num_blocks_per_seq),
101
+ dtype=torch.int32)
102
+
103
+ output = flash_attn_with_kvcache(
104
+ q=query.unsqueeze(1),
105
+ k_cache=key_cache,
106
+ v_cache=value_cache,
107
+ softmax_scale=scale,
108
+ causal=True,
109
+ block_table=block_tables,
110
+ cache_seqlens=kv_lens_tensor,
111
+ ).squeeze(1)
112
+
113
+ ref_output = ref_paged_attn(
114
+ query=query,
115
+ key_cache=key_cache,
116
+ value_cache=value_cache,
117
+ query_lens=[1] * num_seqs,
118
+ kv_lens=kv_lens,
119
+ block_tables=block_tables,
120
+ scale=scale,
121
+ )
122
+ assert torch.allclose(output, ref_output, atol=1e-2, rtol=1e-2), \
123
+ f"{torch.max(torch.abs(output - ref_output))}"
124
+
125
+
126
+@pytest.mark.parametrize("seq_lens", [[(1, 1328), (5, 18), (129, 463)]])
127
+@pytest.mark.parametrize("num_heads", NUM_HEADS)
128
+@pytest.mark.parametrize("head_size", HEAD_SIZES)
129
+@pytest.mark.parametrize("block_size", BLOCK_SIZES)
130
+@pytest.mark.parametrize("sliding_window", [None])
131
+@pytest.mark.parametrize("dtype", DTYPES)
132
+@torch.inference_mode
133
+def test_varlen_with_paged_kv(
134
+ seq_lens: List[Tuple[int, int]],
135
+ num_heads: Tuple[int, int],
136
+ head_size: int,
137
+ sliding_window: Optional[int],
138
+ dtype: torch.dtype,
139
+ block_size: int,
140
+) -> None:
141
+ torch.set_default_device("cuda")
142
+ torch.cuda.manual_seed_all(0)
143
+ num_seqs = len(seq_lens)
144
+ query_lens = [x[0] for x in seq_lens]
145
+ kv_lens = [x[1] for x in seq_lens]
146
+ num_query_heads = num_heads[0]
147
+ num_kv_heads = num_heads[1]
148
+ assert num_query_heads % num_kv_heads == 0
149
+ max_query_len = max(query_lens)
150
+ max_kv_len = max(kv_lens)
151
+ window_size = ((sliding_window,
152
+ sliding_window) if sliding_window is not None else
153
+ (-1, -1))
154
+ scale = head_size**-0.5
155
+
156
+ query = torch.randn(sum(query_lens),
157
+ num_query_heads,
158
+ head_size,
159
+ dtype=dtype)
160
+ key_cache = torch.randn(NUM_BLOCKS,
161
+ block_size,
162
+ num_kv_heads,
163
+ head_size,
164
+ dtype=dtype)
165
+ value_cache = torch.randn_like(key_cache)
166
+ # Normalize the scale of the key and value caches to mitigate
167
+ # numerical instability.
168
+ key_cache /= head_size**0.5
169
+ value_cache /= head_size**0.5
170
+ cu_query_lens = torch.tensor([0] + query_lens,
171
+ dtype=torch.int32).cumsum(dim=0,
172
+ dtype=torch.int32)
173
+ cu_kv_lens = torch.tensor([0] + kv_lens,
174
+ dtype=torch.int32).cumsum(dim=0,
175
+ dtype=torch.int32)
176
+
177
+ max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
178
+ block_tables = torch.randint(0,
179
+ NUM_BLOCKS,
180
+ (num_seqs, max_num_blocks_per_seq),
181
+ dtype=torch.int32)
182
+
183
+ output = flash_attn_varlen_func(
184
+ q=query,
185
+ k=key_cache,
186
+ v=value_cache,
187
+ cu_seqlens_q=cu_query_lens,
188
+ cu_seqlens_k=cu_kv_lens,
189
+ max_seqlen_q=max_query_len,
190
+ max_seqlen_k=max_kv_len,
191
+ softmax_scale=scale,
192
+ causal=True,
193
+ window_size=window_size,
194
+ block_table=block_tables,
195
+ )
196
+
197
+ ref_output = ref_paged_attn(
198
+ query=query,
199
+ key_cache=key_cache,
200
+ value_cache=value_cache,
201
+ query_lens=query_lens,
202
+ kv_lens=kv_lens,
203
+ block_tables=block_tables,
204
+ scale=scale,
205
+ sliding_window=sliding_window,
206
+ )
207
+ assert torch.allclose(output, ref_output, atol=1e-2, rtol=1e-2), \
208
+ f"{torch.max(torch.abs(output - ref_output))}"
tests/models/test_big_models.py
+1
-1
@@ -12,7 +12,7 @@ MODELS = [
12
# "Deci/DeciLM-7b", # Broken
13
# "tiiuae/falcon-7b", # Broken
14
"EleutherAI/gpt-j-6b",
15
- "mosaicml/mpt-7b",
15
+ # "mosaicml/mpt-7b", # Broken
16
# "Qwen/Qwen1.5-0.5B" # Broken,
17
]
18
tests/models/test_fp8.py
+5
-5
@@ -25,18 +25,18 @@ EXPECTED_STRS_MAP = {
25
'LLaMA is a high-throughput and memory-efficient inference and serving engine for Large Language Models (',
26
'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
27
'Artificial intelligence (AI) and human intelligence (HI) differ significantly in how they process information.',
28
- 'A neural network is a complex system modeled after the human brain, composed of interconnected nodes or "ne',
29
- 'Zeta-5, a highly advanced robot designed for menial labor, whirred and beep',
30
- 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. Here',
28
+ 'A neural network is a complex system modeled after the human brain, consisting of interconnected nodes or "ne',
29
+ 'Zeta-5, a highly advanced robot designed for menial labor, whirred to a',
30
+ 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. The',
31
'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
32
- 'Here are the translations:\n\n**Japanese:** (Haya tori, nemuri nemuri)\n\n**'
32
+ 'Here are the translations:\n\n**Japanese:** (Haya aki no tori, guri o',
33
],
34
"meta-llama/Meta-Llama-3-8B-Instruct": [
35
'LLM (Large Language Model) is a type of artificial intelligence (AI) model that is trained',
36
'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
37
'Artificial intelligence (AI) and human intelligence (HI) differ significantly in how they process information.',
38
'A neural network is a complex system modeled after the human brain, composed of interconnected nodes or "ne',
39
- 'In the year 2154, the robotics lab at NeuroSpark Industries was on the cusp of',
39
+ 'In the vast, sterile laboratory, Robot 3456-Alpha, or "Alpha" for short',
40
'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. The',
41
'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
42
'Here are the translations:\n\n**Japanese:** (Haya aki wa mushi o tsukamu'
vllm/attention/backends/flash_attn.py
+75
-54
@@ -1,19 +1,15 @@
1
-"""Attention layer with Flash and PagedAttention.
2
-
3
-NOTE(woosuk): At the moment, this file includes a lot of duplicated code from
4
-XFormers backend. The duplicated code will be removed once we use flash-attn or
5
-flashinfer for all the attention operations.
6
-"""
1
+"""Attention layer with FlashAttention."""
2
from dataclasses import dataclass
3
from typing import List, Optional, Tuple, Type
4
5
import torch
11
-from vllm_flash_attn import flash_attn_varlen_func
6
+from vllm_flash_attn import flash_attn_varlen_func, flash_attn_with_kvcache
7
8
+from vllm._C import cache_ops
9
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10
AttentionMetadata)
15
-from vllm.attention.ops.paged_attn import (PagedAttention,
16
- PagedAttentionMetadata)
11
+
12
+_SUPPORTED_HEAD_SIZES = [32, 64, 96, 128, 160, 192, 224, 256]
13
14
15
class FlashAttentionBackend(AttentionBackend):
@@ -37,8 +33,9 @@ class FlashAttentionBackend(AttentionBackend):
33
num_kv_heads: int,
34
head_size: int,
35
) -> Tuple[int, ...]:
40
- return PagedAttention.get_kv_cache_shape(num_blocks, block_size,
41
- num_kv_heads, head_size)
36
+ if block_size % 16 != 0:
37
+ raise ValueError("Block size must be a multiple of 16.")
38
+ return (2, num_blocks, block_size, num_kv_heads, head_size)
39
40
@staticmethod
41
def swap_blocks(
@@ -46,18 +43,26 @@ class FlashAttentionBackend(AttentionBackend):
43
dst_kv_cache: torch.Tensor,
44
src_to_dst: torch.Tensor,
45
) -> None:
49
- PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
46
+ src_key_cache = src_kv_cache[0]
47
+ dst_key_cache = dst_kv_cache[0]
48
+ cache_ops.swap_blocks(src_key_cache, dst_key_cache, src_to_dst)
49
+
50
+ src_value_cache = src_kv_cache[1]
51
+ dst_value_cache = dst_kv_cache[1]
52
+ cache_ops.swap_blocks(src_value_cache, dst_value_cache, src_to_dst)
53
54
@staticmethod
55
def copy_blocks(
56
kv_caches: List[torch.Tensor],
57
src_to_dists: torch.Tensor,
58
) -> None:
56
- PagedAttention.copy_blocks(kv_caches, src_to_dists)
59
+ key_caches = [kv_cache[0] for kv_cache in kv_caches]
60
+ value_caches = [kv_cache[1] for kv_cache in kv_caches]
61
+ cache_ops.copy_blocks(key_caches, value_caches, src_to_dists)
62
63
64
@dataclass
60
-class FlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
65
+class FlashAttentionMetadata(AttentionMetadata):
66
"""Metadata for FlashAttentionBackend.
67
68
NOTE: Any python object stored here is not updated when it is
@@ -99,6 +104,14 @@ class FlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
104
# so far).
105
context_lens_tensor: Optional[torch.Tensor]
106
107
+ # (batch_size, max_blocks_per_seq).
108
+ # Block addresses per sequence. (Seq id -> list of physical block)
109
+ # E.g., [0, 1, 2] means tokens are stored in 0th, 1st, and 2nd blocks
110
+ # in the kv cache. Each block can contain up to block_size tokens.
111
+ # 2nd dimensions are padded up to max_blocks_per_seq if it is cuda-graph
112
+ # captured.
113
+ block_tables: Optional[torch.Tensor]
114
+
115
# Whether or not if cuda graph is enabled.
116
# Cuda-graph is currently enabled for decoding only.
117
# TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
@@ -219,11 +232,15 @@ class FlashAttentionImpl(AttentionImpl):
232
assert self.num_heads % self.num_kv_heads == 0
233
self.num_queries_per_kv = self.num_heads // self.num_kv_heads
234
222
- suppored_head_sizes = PagedAttention.get_supported_head_sizes()
223
- if head_size not in suppored_head_sizes:
235
+ if sliding_window is not None:
236
+ # NOTE(woosuk): flash-attn's sliding window does not work with
237
+ # paged KV cache.
238
+ raise ValueError(
239
+ "Sliding window is not supported in FlashAttention.")
240
+ if head_size not in _SUPPORTED_HEAD_SIZES:
241
raise ValueError(
225
- f"Head size {head_size} is not supported by PagedAttention. "
226
- f"Supported head sizes are: {suppored_head_sizes}.")
242
+ f"Head size {head_size} is not supported by FlashAttention. "
243
+ f"Supported head sizes are: {_SUPPORTED_HEAD_SIZES}.")
244
245
def forward(
246
self,
@@ -234,17 +251,20 @@ class FlashAttentionImpl(AttentionImpl):
251
attn_metadata: FlashAttentionMetadata,
252
kv_scale: float = 1.0,
253
) -> torch.Tensor:
237
- """Forward pass with FlashAttention and PagedAttention.
254
+ """Forward pass with FlashAttention.
255
256
Args:
257
query: shape = [num_tokens, num_heads * head_size]
258
key: shape = [num_tokens, num_kv_heads * head_size]
259
value: shape = [num_tokens, num_kv_heads * head_size]
243
- kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
260
+ kv_cache = [2, num_blocks, block_size, num_kv_heads, head_size]
261
attn_metadata: Metadata for attention.
262
Returns:
263
shape = [num_tokens, num_heads * head_size]
264
"""
265
+ # NOTE(woosuk): FlashAttention does not support FP8 KV cache.
266
+ assert kv_scale == 1.0, "kv_scale is not supported in FlashAttention."
267
+
268
num_tokens, hidden_size = query.shape
269
# Reshape the query, key, and value tensors.
270
query = query.view(-1, self.num_heads, self.head_size)
@@ -252,16 +272,20 @@ class FlashAttentionImpl(AttentionImpl):
272
value = value.view(-1, self.num_kv_heads, self.head_size)
273
274
if kv_cache is not None:
255
- key_cache, value_cache = PagedAttention.split_kv_cache(
256
- kv_cache, self.num_kv_heads, self.head_size)
275
+ key_cache = kv_cache[0]
276
+ value_cache = kv_cache[1]
277
278
# Reshape the input keys and values and store them in the cache.
279
# If kv_cache is not provided, the new key and value tensors are
280
# not cached. This happens during the initial memory profiling run.
261
- PagedAttention.write_to_paged_cache(key, value, key_cache,
262
- value_cache,
263
- attn_metadata.slot_mapping,
264
- self.kv_cache_dtype, kv_scale)
281
+ cache_ops.reshape_and_cache_flash(
282
+ key,
283
+ value,
284
+ key_cache,
285
+ value_cache,
286
+ attn_metadata.slot_mapping.flatten(),
287
+ self.kv_cache_dtype,
288
+ )
289
290
num_prefill_tokens = attn_metadata.num_prefill_tokens
291
num_decode_tokens = attn_metadata.num_decode_tokens
@@ -281,7 +305,8 @@ class FlashAttentionImpl(AttentionImpl):
305
306
if prefill_meta := attn_metadata.prefill_metadata:
307
# Prompt run.
284
- if kv_cache is None or prefill_meta.block_tables.numel() == 0:
308
+ if (kv_cache is None or prefill_meta.block_tables is None
309
+ or prefill_meta.block_tables.numel() == 0):
310
# normal attention
311
# When block_tables are not filled, it means q and k are the
312
# prompt, and they have the same length.
@@ -302,38 +327,34 @@ class FlashAttentionImpl(AttentionImpl):
327
output[:num_prefill_tokens] = out
328
else:
329
# prefix-enabled attention
305
- # TODO(Hai) this triton kernel has regression issue (broke) to
306
- # deal with different data types between KV and FP8 KV cache,
307
- # to be addressed separately.
308
- output[:num_prefill_tokens] = PagedAttention.forward_prefix(
309
- query,
310
- key,
311
- value,
312
- key_cache,
313
- value_cache,
314
- prefill_meta.block_tables,
315
- prefill_meta.query_start_loc,
316
- prefill_meta.seq_lens_tensor,
317
- prefill_meta.context_lens_tensor,
318
- prefill_meta.max_query_len,
319
- self.alibi_slopes,
320
- self.sliding_window[0],
330
+ assert prefill_meta.seq_lens is not None
331
+ max_seq_len = max(prefill_meta.seq_lens)
332
+ output[:num_prefill_tokens] = flash_attn_varlen_func(
333
+ q=query,
334
+ k=key_cache,
335
+ v=value_cache,
336
+ cu_seqlens_q=prefill_meta.query_start_loc,
337
+ max_seqlen_q=prefill_meta.max_query_len,
338
+ cu_seqlens_k=prefill_meta.seq_start_loc,
339
+ max_seqlen_k=max_seq_len,
340
+ softmax_scale=self.scale,
341
+ causal=True,
342
+ alibi_slopes=self.alibi_slopes,
343
+ block_table=prefill_meta.block_tables,
344
)
345
+
346
if decode_meta := attn_metadata.decode_metadata:
347
# Decoding run.
324
- output[num_prefill_tokens:] = PagedAttention.forward_decode(
325
- decode_query,
348
+ output[num_prefill_tokens:] = flash_attn_with_kvcache(
349
+ decode_query.unsqueeze(1),
350
key_cache,
351
value_cache,
328
- decode_meta.block_tables,
329
- decode_meta.seq_lens_tensor,
330
- decode_meta.max_decode_seq_len,
331
- self.kv_cache_dtype,
332
- self.num_kv_heads,
333
- self.scale,
334
- self.alibi_slopes,
335
- kv_scale,
336
- )
352
+ block_table=decode_meta.block_tables,
353
+ cache_seqlens=decode_meta.seq_lens_tensor,
354
+ softmax_scale=self.scale,
355
+ causal=True,
356
+ alibi_slopes=self.alibi_slopes,
357
+ ).squeeze(1)
358
359
# Reshape the output tensor.
360
return output.view(num_tokens, hidden_size)
vllm/attention/selector.py
+14
@@ -93,6 +93,20 @@ def _which_attn_to_use(
93
"torch.float16 or torch.bfloat16.")
94
return _Backend.XFORMERS
95
96
+ if kv_cache_dtype is not None and kv_cache_dtype.startswith("fp8"):
97
+ logger.info("Cannot use FlashAttention-2 backend for FP8 KV cache.")
98
+ return _Backend.XFORMERS
99
+
100
+ if block_size % 16 != 0:
101
+ logger.info("Cannot use FlashAttention-2 backend for block size not "
102
+ "divisible by 16.")
103
+ return _Backend.XFORMERS
104
+
105
+ if sliding_window is not None:
106
+ logger.info(
107
+ "Cannot use FlashAttention-2 backend due to sliding window.")
108
+ return _Backend.XFORMERS
109
+
110
try:
111
import vllm_flash_attn # noqa: F401
112
except ImportError: