A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Use FlashAttention for `multi_query_kv_attention` (#4)
Woosuk Kwon committed
Mar 1, 2023 at 21:13 UTC
3e9f991d6acd7efd90f04f1f530b837a40c93442
4 files changed
+108
-34
README.md
+1
@@ -4,6 +4,7 @@
4
5
```bash
6
pip install cmake torch transformers
7
+pip install flash-attn # This may take up to 10 mins.
8
pip install -e .
9
```
10
cacheflow/models/attention.py
+37
-30
@@ -1,5 +1,6 @@
1
from typing import List, Optional
2
3
+from flash_attn.flash_attention import FlashAttention
4
import torch
5
import torch.nn as nn
6
@@ -14,20 +15,7 @@ class OPTCacheFlowAttention(nn.Module):
15
super().__init__()
16
self.scale = float(scale)
17
17
- def _masked_attention(
18
- self,
19
- query: torch.Tensor, # [num_queries, num_heads, head_size]
20
- key: torch.Tensor, # [num_keys, num_heads, head_size]
21
- value: torch.Tensor, # [num_keys, num_heads, head_size]
22
- attn_mask: Optional[torch.Tensor] = None, # [num_queries, num_keys]
23
- ) -> torch.Tensor: # [num_queries, num_heads, head_size]
24
- query = query * self.scale
25
- attn = torch.einsum('qhd,khd->hqk', query, key)
26
- if attn_mask is not None:
27
- attn = attn + attn_mask
28
- attn = torch.softmax(attn, dim=-1)
29
- out = torch.einsum('hqk,khd->qhd', attn, value)
30
- return out
18
+ self.flash_attn = FlashAttention(softmax_scale=self.scale)
19
20
def multi_query_kv_attention(
21
self,
@@ -37,21 +25,31 @@ class OPTCacheFlowAttention(nn.Module):
25
value: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
26
prompt_lens: List[int],
27
) -> None:
40
- # FIXME(woosuk): Replace the following with a custom op.
41
- start_idx = 0
28
+ if query.dtype == torch.float:
29
+ raise ValueError('The float data type is not supported by '
30
+ 'FlashAttention. Use the half data type instead.')
31
+ head_size = query.shape[2]
32
+ if head_size > 128:
33
+ raise ValueError('FlashAttention does not support head_size > 128.')
34
+
35
+ device = query.device
36
+ prefix_sum = [0]
37
for prompt_len in prompt_lens:
43
- out = output[start_idx:start_idx + prompt_len]
44
- q = query[start_idx:start_idx + prompt_len]
45
- k = key[start_idx:start_idx + prompt_len]
46
- v = value[start_idx:start_idx + prompt_len]
47
-
48
- attention_mask = torch.triu(
49
- torch.ones(q.shape[0], k.shape[0]), diagonal=1) * -1e5
50
- attention_mask = attention_mask.to(dtype=q.dtype, device=q.device)
51
- attention_out = self._masked_attention(q, k, v, attention_mask)
52
- out.copy_(attention_out, non_blocking=True)
53
-
54
- start_idx += prompt_len
38
+ prefix_sum.append(prefix_sum[-1] + prompt_len)
39
+ prefix_sum = torch.tensor(prefix_sum, dtype=torch.int, device=device)
40
+ max_prompt_len = max(prompt_lens)
41
+
42
+ # FIXME(woosuk): Unnecessary copy. Optimize this.
43
+ qkv = torch.stack([query, key, value], dim=1)
44
+ out = self.flash_attn(
45
+ qkv,
46
+ cu_seqlens=prefix_sum,
47
+ max_s=max_prompt_len,
48
+ causal=True,
49
+ )[0]
50
+ num_tokens = prefix_sum[-1]
51
+ # FIXME(woosuk): Unnecessary copy. Optimize this.
52
+ output[:num_tokens].copy_(out, non_blocking=True)
53
54
def single_query_cached_kv_attention(
55
self,
@@ -61,6 +59,14 @@ class OPTCacheFlowAttention(nn.Module):
59
value_cache: torch.Tensor, # [num_blocks, num_heads, head_size, block_size]
60
input_metadata: InputMetadata,
61
) -> None:
62
+ head_size = value_cache.shape[2]
63
+ supported_head_sizes = [32, 64, 80, 96, 128, 160, 192, 256]
64
+ if head_size not in supported_head_sizes:
65
+ raise ValueError(f'head_size ({head_size}) is not supported by '
66
+ 'the single_query_cached_kv_attention kernel. '
67
+ 'Use one of the following head sizes: '
68
+ f'{supported_head_sizes}.')
69
+
70
block_size = value_cache.shape[3]
71
attention_ops.single_query_cached_kv_attention(
72
output,
@@ -101,8 +107,9 @@ class OPTCacheFlowAttention(nn.Module):
107
output = output.view(-1, num_heads, head_size)
108
109
# Compute the attention op for prompts.
104
- self.multi_query_kv_attention(
105
- output, query, key, value, input_metadata.prompt_lens)
110
+ if input_metadata.num_prompts > 0:
111
+ self.multi_query_kv_attention(
112
+ output, query, key, value, input_metadata.prompt_lens)
113
114
# Wait until the cache op is done.
115
if cache_event is not None:
server.py
+5
-2
@@ -9,10 +9,12 @@ parser = argparse.ArgumentParser(description='CacheFlow server')
9
parser.add_argument('--model', type=str, default='facebook/opt-125m', help='model name')
10
parser.add_argument('--num-nodes', type=int, default=1, help='number of nodes')
11
parser.add_argument('--num-workers', type=int, default=1, help='number of workers per node')
12
-parser.add_argument('--block-size', type=int, default=8, help='token block size')
12
+parser.add_argument('--block-size', type=int, default=8, choices=[8, 16], help='token block size')
13
# TODO(woosuk): Add an analytical model to determine the maximum number of GPU/CPU blocks.
14
parser.add_argument('--num-gpu-blocks', type=int, default=1024, help='number of GPU blocks (per GPU)')
15
-parser.add_argument('--num-cpu-blocks', type=int, default=256, help='number of CPU blocks (per GPU)')
15
+parser.add_argument('--num-cpu-blocks', type=int, default=32, help='number of CPU blocks (per GPU)')
16
+# NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
17
+parser.add_argument('--dtype', type=str, default='half', choices=['half', 'float'], help='data type')
18
args = parser.parse_args()
19
20
@@ -27,6 +29,7 @@ def main():
29
block_size=args.block_size,
30
num_gpu_blocks=args.num_gpu_blocks,
31
num_cpu_blocks=args.num_cpu_blocks,
32
+ dtype=args.dtype,
33
)
34
controllers.append(controller)
35
tests/kernels/attention.py
+65
-2
@@ -1,10 +1,13 @@
1
import random
2
from typing import Optional
3
4
+from flash_attn.flash_attention import FlashAttention
5
import torch
6
7
from cacheflow import attention_ops
8
9
+MAX_SEQ_LEN = 4096
10
+
11
12
def ref_masked_attention(
13
query: torch.Tensor,
@@ -79,7 +82,7 @@ def test_single_query_cached_kv_attention(
82
value_cache = torch.randn(
83
size=(num_blocks, *value_block_shape), dtype=dtype, device='cuda')
84
82
- context_lens = [random.randint(1, 4096) for _ in range(num_tokens)]
85
+ context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_tokens)]
86
max_context_len = max(context_lens)
87
context_lens = torch.tensor(context_lens, dtype=torch.int, device='cuda')
88
@@ -123,11 +126,60 @@ def test_single_query_cached_kv_attention(
126
assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
127
128
129
+def test_multi_query_kv_attention(
130
+ num_seqs: int,
131
+ num_heads: int,
132
+ head_size: int,
133
+ dtype: torch.dtype,
134
+) -> None:
135
+ seq_lens = random.sample(range(1, MAX_SEQ_LEN), num_seqs)
136
+ max_seq_len = max(seq_lens)
137
+ num_tokens = sum(seq_lens)
138
+
139
+ cu_seq_lens = [0]
140
+ for seq_len in seq_lens:
141
+ cu_seq_lens.append(cu_seq_lens[-1] + seq_len)
142
+ cu_seq_lens = torch.tensor(cu_seq_lens, dtype=torch.int, device='cuda')
143
+
144
+ scale = float(1.0 / (head_size ** 0.5))
145
+ query = torch.randn(
146
+ num_tokens, num_heads, head_size, dtype=dtype, device='cuda')
147
+ key = torch.rand_like(query)
148
+ value = torch.rand_like(query)
149
+
150
+ qkv = torch.stack([query, key, value], dim=1)
151
+ flash_attn = FlashAttention(softmax_scale=scale)
152
+ output = flash_attn(
153
+ qkv,
154
+ cu_seqlens=cu_seq_lens,
155
+ max_s=max_seq_len,
156
+ causal=True,
157
+ )[0]
158
+
159
+ ref_outputs = []
160
+ for i, seq_len in enumerate(seq_lens):
161
+ attn_mask = torch.triu(torch.ones(seq_len, seq_len), diagonal=1) * -1e5
162
+ attn_mask = attn_mask.to(dtype=dtype, device='cuda')
163
+ start_idx = cu_seq_lens[i]
164
+ end_idx = cu_seq_lens[i + 1]
165
+ ref_output = ref_masked_attention(
166
+ query[start_idx:end_idx],
167
+ key[start_idx:end_idx],
168
+ value[start_idx:end_idx],
169
+ scale,
170
+ attn_mask=attn_mask,
171
+ )
172
+ ref_outputs.append(ref_output)
173
+ ref_output = torch.cat(ref_outputs, dim=0)
174
+
175
+ assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
176
+
177
+
178
@torch.inference_mode()
179
def test_attention() -> None:
180
for dtype in [torch.half, torch.float]:
181
for block_size in [8, 16]:
130
- for head_size in [64, 80, 96, 128, 256]:
182
+ for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
183
test_single_query_cached_kv_attention(
184
num_tokens=37,
185
num_heads=3,
@@ -137,6 +189,17 @@ def test_attention() -> None:
189
dtype=dtype,
190
)
191
192
+ # NOTE(woosuk): FlashAttention does not support FP32.
193
+ for dtype in [torch.half]:
194
+ # NOTE(woosuk): FlashAttention does not support head_size > 128.
195
+ for head_size in [64, 80, 96, 128]:
196
+ test_multi_query_kv_attention(
197
+ num_seqs=11,
198
+ num_heads=3,
199
+ head_size=head_size,
200
+ dtype=dtype,
201
+ )
202
+
203
204
if __name__ == '__main__':
205
test_attention()