@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3e9f991d6a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Use FlashAttention for `multi_query_kv_attention` (#4)

Woosuk Kwon committed Mar 1, 2023 at 21:13 UTC 3e9f991d6acd7efd90f04f1f530b837a40c93442
4 files changed +108 -34
README.md
+1
@@ -4,6 +4,7 @@
4
5 ```bash
6 pip install cmake torch transformers
7 +pip install flash-attn # This may take up to 10 mins.
8 pip install -e .
9 ```
10
cacheflow/models/attention.py
+37 -30
@@ -1,5 +1,6 @@
1 from typing import List, Optional
2
3 +from flash_attn.flash_attention import FlashAttention
4 import torch
5 import torch.nn as nn
6
@@ -14,20 +15,7 @@ class OPTCacheFlowAttention(nn.Module):
15 super().__init__()
16 self.scale = float(scale)
17
17 - def _masked_attention(
18 - self,
19 - query: torch.Tensor, # [num_queries, num_heads, head_size]
20 - key: torch.Tensor, # [num_keys, num_heads, head_size]
21 - value: torch.Tensor, # [num_keys, num_heads, head_size]
22 - attn_mask: Optional[torch.Tensor] = None, # [num_queries, num_keys]
23 - ) -> torch.Tensor: # [num_queries, num_heads, head_size]
24 - query = query * self.scale
25 - attn = torch.einsum('qhd,khd->hqk', query, key)
26 - if attn_mask is not None:
27 - attn = attn + attn_mask
28 - attn = torch.softmax(attn, dim=-1)
29 - out = torch.einsum('hqk,khd->qhd', attn, value)
30 - return out
18 + self.flash_attn = FlashAttention(softmax_scale=self.scale)
19
20 def multi_query_kv_attention(
21 self,
@@ -37,21 +25,31 @@ class OPTCacheFlowAttention(nn.Module):
25 value: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
26 prompt_lens: List[int],
27 ) -> None:
40 - # FIXME(woosuk): Replace the following with a custom op.
41 - start_idx = 0
28 + if query.dtype == torch.float:
29 + raise ValueError('The float data type is not supported by '
30 + 'FlashAttention. Use the half data type instead.')
31 + head_size = query.shape[2]
32 + if head_size > 128:
33 + raise ValueError('FlashAttention does not support head_size > 128.')
34 +
35 + device = query.device
36 + prefix_sum = [0]
37 for prompt_len in prompt_lens:
43 - out = output[start_idx:start_idx + prompt_len]
44 - q = query[start_idx:start_idx + prompt_len]
45 - k = key[start_idx:start_idx + prompt_len]
46 - v = value[start_idx:start_idx + prompt_len]
47 -
48 - attention_mask = torch.triu(
49 - torch.ones(q.shape[0], k.shape[0]), diagonal=1) * -1e5
50 - attention_mask = attention_mask.to(dtype=q.dtype, device=q.device)
51 - attention_out = self._masked_attention(q, k, v, attention_mask)
52 - out.copy_(attention_out, non_blocking=True)
53 -
54 - start_idx += prompt_len
38 + prefix_sum.append(prefix_sum[-1] + prompt_len)
39 + prefix_sum = torch.tensor(prefix_sum, dtype=torch.int, device=device)
40 + max_prompt_len = max(prompt_lens)
41 +
42 + # FIXME(woosuk): Unnecessary copy. Optimize this.
43 + qkv = torch.stack([query, key, value], dim=1)
44 + out = self.flash_attn(
45 + qkv,
46 + cu_seqlens=prefix_sum,
47 + max_s=max_prompt_len,
48 + causal=True,
49 + )[0]
50 + num_tokens = prefix_sum[-1]
51 + # FIXME(woosuk): Unnecessary copy. Optimize this.
52 + output[:num_tokens].copy_(out, non_blocking=True)
53
54 def single_query_cached_kv_attention(
55 self,
@@ -61,6 +59,14 @@ class OPTCacheFlowAttention(nn.Module):
59 value_cache: torch.Tensor, # [num_blocks, num_heads, head_size, block_size]
60 input_metadata: InputMetadata,
61 ) -> None:
62 + head_size = value_cache.shape[2]
63 + supported_head_sizes = [32, 64, 80, 96, 128, 160, 192, 256]
64 + if head_size not in supported_head_sizes:
65 + raise ValueError(f'head_size ({head_size}) is not supported by '
66 + 'the single_query_cached_kv_attention kernel. '
67 + 'Use one of the following head sizes: '
68 + f'{supported_head_sizes}.')
69 +
70 block_size = value_cache.shape[3]
71 attention_ops.single_query_cached_kv_attention(
72 output,
@@ -101,8 +107,9 @@ class OPTCacheFlowAttention(nn.Module):
107 output = output.view(-1, num_heads, head_size)
108
109 # Compute the attention op for prompts.
104 - self.multi_query_kv_attention(
105 - output, query, key, value, input_metadata.prompt_lens)
110 + if input_metadata.num_prompts > 0:
111 + self.multi_query_kv_attention(
112 + output, query, key, value, input_metadata.prompt_lens)
113
114 # Wait until the cache op is done.
115 if cache_event is not None:
server.py
+5 -2
@@ -9,10 +9,12 @@ parser = argparse.ArgumentParser(description='CacheFlow server')
9 parser.add_argument('--model', type=str, default='facebook/opt-125m', help='model name')
10 parser.add_argument('--num-nodes', type=int, default=1, help='number of nodes')
11 parser.add_argument('--num-workers', type=int, default=1, help='number of workers per node')
12 -parser.add_argument('--block-size', type=int, default=8, help='token block size')
12 +parser.add_argument('--block-size', type=int, default=8, choices=[8, 16], help='token block size')
13 # TODO(woosuk): Add an analytical model to determine the maximum number of GPU/CPU blocks.
14 parser.add_argument('--num-gpu-blocks', type=int, default=1024, help='number of GPU blocks (per GPU)')
15 -parser.add_argument('--num-cpu-blocks', type=int, default=256, help='number of CPU blocks (per GPU)')
15 +parser.add_argument('--num-cpu-blocks', type=int, default=32, help='number of CPU blocks (per GPU)')
16 +# NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
17 +parser.add_argument('--dtype', type=str, default='half', choices=['half', 'float'], help='data type')
18 args = parser.parse_args()
19
20
@@ -27,6 +29,7 @@ def main():
29 block_size=args.block_size,
30 num_gpu_blocks=args.num_gpu_blocks,
31 num_cpu_blocks=args.num_cpu_blocks,
32 + dtype=args.dtype,
33 )
34 controllers.append(controller)
35
tests/kernels/attention.py
+65 -2
@@ -1,10 +1,13 @@
1 import random
2 from typing import Optional
3
4 +from flash_attn.flash_attention import FlashAttention
5 import torch
6
7 from cacheflow import attention_ops
8
9 +MAX_SEQ_LEN = 4096
10 +
11
12 def ref_masked_attention(
13 query: torch.Tensor,
@@ -79,7 +82,7 @@ def test_single_query_cached_kv_attention(
82 value_cache = torch.randn(
83 size=(num_blocks, *value_block_shape), dtype=dtype, device='cuda')
84
82 - context_lens = [random.randint(1, 4096) for _ in range(num_tokens)]
85 + context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_tokens)]
86 max_context_len = max(context_lens)
87 context_lens = torch.tensor(context_lens, dtype=torch.int, device='cuda')
88
@@ -123,11 +126,60 @@ def test_single_query_cached_kv_attention(
126 assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
127
128
129 +def test_multi_query_kv_attention(
130 + num_seqs: int,
131 + num_heads: int,
132 + head_size: int,
133 + dtype: torch.dtype,
134 +) -> None:
135 + seq_lens = random.sample(range(1, MAX_SEQ_LEN), num_seqs)
136 + max_seq_len = max(seq_lens)
137 + num_tokens = sum(seq_lens)
138 +
139 + cu_seq_lens = [0]
140 + for seq_len in seq_lens:
141 + cu_seq_lens.append(cu_seq_lens[-1] + seq_len)
142 + cu_seq_lens = torch.tensor(cu_seq_lens, dtype=torch.int, device='cuda')
143 +
144 + scale = float(1.0 / (head_size ** 0.5))
145 + query = torch.randn(
146 + num_tokens, num_heads, head_size, dtype=dtype, device='cuda')
147 + key = torch.rand_like(query)
148 + value = torch.rand_like(query)
149 +
150 + qkv = torch.stack([query, key, value], dim=1)
151 + flash_attn = FlashAttention(softmax_scale=scale)
152 + output = flash_attn(
153 + qkv,
154 + cu_seqlens=cu_seq_lens,
155 + max_s=max_seq_len,
156 + causal=True,
157 + )[0]
158 +
159 + ref_outputs = []
160 + for i, seq_len in enumerate(seq_lens):
161 + attn_mask = torch.triu(torch.ones(seq_len, seq_len), diagonal=1) * -1e5
162 + attn_mask = attn_mask.to(dtype=dtype, device='cuda')
163 + start_idx = cu_seq_lens[i]
164 + end_idx = cu_seq_lens[i + 1]
165 + ref_output = ref_masked_attention(
166 + query[start_idx:end_idx],
167 + key[start_idx:end_idx],
168 + value[start_idx:end_idx],
169 + scale,
170 + attn_mask=attn_mask,
171 + )
172 + ref_outputs.append(ref_output)
173 + ref_output = torch.cat(ref_outputs, dim=0)
174 +
175 + assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
176 +
177 +
178 @torch.inference_mode()
179 def test_attention() -> None:
180 for dtype in [torch.half, torch.float]:
181 for block_size in [8, 16]:
130 - for head_size in [64, 80, 96, 128, 256]:
182 + for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
183 test_single_query_cached_kv_attention(
184 num_tokens=37,
185 num_heads=3,
@@ -137,6 +189,17 @@ def test_attention() -> None:
189 dtype=dtype,
190 )
191
192 + # NOTE(woosuk): FlashAttention does not support FP32.
193 + for dtype in [torch.half]:
194 + # NOTE(woosuk): FlashAttention does not support head_size > 128.
195 + for head_size in [64, 80, 96, 128]:
196 + test_multi_query_kv_attention(
197 + num_seqs=11,
198 + num_heads=3,
199 + head_size=head_size,
200 + dtype=dtype,
201 + )
202 +
203
204 if __name__ == '__main__':
205 test_attention()