A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Replace FlashAttention with xformers (#70)
Woosuk Kwon committed
May 5, 2023 at 02:01 UTC
c9d5b6d4a8b3f51ff6c9eee7eb52bb5149d89b6a
13 files changed
+87
-131
README.md
+1
-5
@@ -3,11 +3,7 @@
3
## Installation
4
5
```bash
6
-pip install psutil numpy ray torch
7
-pip install git+https://github.com/huggingface/transformers # Required for LLaMA.
8
-pip install sentencepiece # Required for LlamaTokenizer.
9
-pip install ninja # To parallelize the compilation of flash-attn.
10
-pip install flash-attn # This may take up to 10 mins.
6
+pip install ninja psutil numpy sentencepiece ray torch transformers xformers
7
pip install -e .
8
```
9
cacheflow/master/server.py
+1
-1
@@ -213,7 +213,7 @@ def add_server_arguments(parser: argparse.ArgumentParser):
213
parser.add_argument('--use-np-cache', action='store_true',
214
help='save a numpy copy of model weights for faster loading')
215
parser.add_argument('--use-dummy-weights', action='store_true', help='use dummy values for model weights')
216
- # NOTE(woosuk): FlashAttention does not support float32.
216
+ # TODO(woosuk): Support FP32 for debugging.
217
parser.add_argument('--dtype', type=str, default='default', choices=['default', 'half', 'bfloat16'],
218
help=('data type for model weights and activations. '
219
'The "default" option will use FP16 precision '
cacheflow/models/attention.py
+16
-38
@@ -1,8 +1,8 @@
1
from typing import Optional
2
3
-from flash_attn.flash_attn_interface import _flash_attn_forward
3
import torch
4
import torch.nn as nn
5
+from xformers import ops as xops
6
7
from cacheflow import attention_ops
8
from cacheflow import cache_ops
@@ -15,6 +15,7 @@ class GPTCacheFlowAttention(nn.Module):
15
def __init__(self, scale: float) -> None:
16
super().__init__()
17
self.scale = float(scale)
18
+ self.attn_op = xops.fmha.cutlass.FwOp()
19
20
def multi_query_kv_attention(
21
self,
@@ -22,32 +23,21 @@ class GPTCacheFlowAttention(nn.Module):
23
query: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
24
key: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
25
value: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
25
- cumulative_prompt_lens: torch.Tensor, # [num_prompts + 1]
26
- max_prompt_len: int,
26
+ attn_bias: xops.AttentionBias,
27
) -> None:
28
- if query.dtype == torch.float:
29
- raise ValueError('The float data type is not supported by '
30
- 'FlashAttention. Use the half data type instead.')
31
- head_size = query.shape[-1]
32
- if head_size > 128:
33
- raise ValueError('FlashAttention does not support head_size > 128.')
34
-
35
- # Directly call FlashAttention's internal function to avoid allocating
36
- # a new tensor for the output.
37
- _flash_attn_forward(
38
- query,
39
- key,
40
- value,
41
- output,
42
- cumulative_prompt_lens,
43
- cumulative_prompt_lens,
44
- max_prompt_len,
45
- max_prompt_len,
46
- dropout_p=0.0,
47
- softmax_scale=self.scale,
48
- causal=True,
49
- return_softmax=False,
28
+ # TODO(woosuk): The unsqueeze op may incur some CPU overhead. Optimize.
29
+ out = xops.memory_efficient_attention_forward(
30
+ query.unsqueeze(0),
31
+ key.unsqueeze(0),
32
+ value.unsqueeze(0),
33
+ attn_bias=attn_bias,
34
+ p=0.0,
35
+ scale=self.scale,
36
+ op=self.attn_op,
37
)
38
+ # TODO(woosuk): Unnecessary copy. Optimize.
39
+ output.copy_(out.squeeze(0))
40
+ return output
41
42
def single_query_cached_kv_attention(
43
self,
@@ -109,8 +99,7 @@ class GPTCacheFlowAttention(nn.Module):
99
query[:num_prompt_tokens],
100
key[:num_prompt_tokens],
101
value[:num_prompt_tokens],
112
- input_metadata.cumulative_prompt_lens,
113
- input_metadata.max_prompt_len,
102
+ input_metadata.attn_bias,
103
)
104
105
# Wait until the cache op is done.
@@ -143,13 +132,6 @@ class GPTCacheFlowAttention(nn.Module):
132
return output.view(-1, num_heads * head_size)
133
134
146
-class OPTCacheFlowAttention(GPTCacheFlowAttention):
147
- """OPT uses the same attention mechanism as GPT."""
148
-
149
- def __init__(self, scale: float) -> None:
150
- super().__init__(scale)
151
-
152
-
135
class GPTNeoXCacheFlowAttention(GPTCacheFlowAttention):
136
"""Attention with GPT-NeoX style rotary embedding."""
137
@@ -207,7 +189,3 @@ class GPTNeoXCacheFlowAttention(GPTCacheFlowAttention):
189
input_metadata,
190
cache_event,
191
)
210
-
211
-
212
-class LlamaCacheFlowAttention(GPTNeoXCacheFlowAttention):
213
- """LLaMA uses the GPT-NeoX style rotary embedding."""
cacheflow/models/input_metadata.py
+9
-12
@@ -1,6 +1,7 @@
1
from typing import List, Dict, Tuple
2
3
import torch
4
+from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
5
6
from cacheflow.sampling_params import SamplingParams
7
@@ -12,7 +13,6 @@ class InputMetadata:
13
seq_groups: List[Tuple[List[int], SamplingParams]],
14
seq_logprobs: Dict[int, float], # Seq id -> cumulative logprobs.
15
prompt_lens: List[int],
15
- cumulative_prompt_lens: torch.Tensor,
16
slot_mapping: torch.Tensor,
17
context_lens: torch.Tensor,
18
max_context_len: int,
@@ -21,15 +21,14 @@ class InputMetadata:
21
self.seq_groups = seq_groups
22
self.seq_logprobs = seq_logprobs
23
self.prompt_lens = prompt_lens
24
- self.cumulative_prompt_lens = cumulative_prompt_lens
24
self.slot_mapping = slot_mapping
25
self.context_lens = context_lens
26
self.max_context_len = max_context_len
27
self.block_tables = block_tables
28
29
+ self.attn_bias = BlockDiagonalCausalMask.from_seqlens(prompt_lens)
30
self.num_prompts = len(prompt_lens)
31
self.num_prompt_tokens = sum(prompt_lens)
32
- self.max_prompt_len = max(prompt_lens) if prompt_lens else 0
32
self.num_generation_tokens = context_lens.shape[0]
33
self.num_valid_tokens = slot_mapping.shape[0]
34
if block_tables.numel() > 0:
@@ -41,15 +40,13 @@ class InputMetadata:
40
41
def __repr__(self) -> str:
42
return (f'InputMetadata('
44
- f'num_prompts={self.num_prompts}, '
45
- f'num_prompt_tokens={self.num_prompt_tokens}, '
46
- f'max_prompt_len={self.max_prompt_len}, '
47
- f'num_generation_tokens={self.num_generation_tokens}, '
43
f'num_valid_tokens={self.num_valid_tokens}, '
49
- f'max_num_blocks_per_seq={self.max_num_blocks_per_seq}, '
50
- f'max_context_len={self.max_context_len}), '
44
+ f'num_prompt_tokens={self.num_prompt_tokens}, '
45
+ f'num_prompts={self.num_prompts}, '
46
f'prompt_lens={self.prompt_lens}, '
52
- f'cumulative_prompt_lens={self.cumulative_prompt_lens}, '
53
- f'slot_mapping={self.slot_mapping}, '
47
+ f'num_generation_tokens={self.num_generation_tokens}, '
48
f'context_lens={self.context_lens}, '
55
- f'block_tables={self.block_tables})')
49
+ f'max_context_len={self.max_context_len}), '
50
+ f'max_num_blocks_per_seq={self.max_num_blocks_per_seq}, '
51
+ f'block_tables={self.block_tables}), '
52
+ f'slot_mapping={self.slot_mapping}')
cacheflow/models/llama.py
+2
-2
@@ -7,7 +7,7 @@ from transformers import LlamaConfig
7
8
from cacheflow.models import InputMetadata
9
from cacheflow.models.activation import SiluAndMul
10
-from cacheflow.models.attention import LlamaCacheFlowAttention
10
+from cacheflow.models.attention import GPTNeoXCacheFlowAttention
11
from cacheflow.models.layernorm import RMSNorm
12
from cacheflow.models.sample import Sampler
13
from cacheflow.models.utils import (hf_model_weights_iterator,
@@ -79,7 +79,7 @@ class LlamaAttention(nn.Module):
79
input_is_parallel=True,
80
perform_initialization=False,
81
)
82
- self.attn = LlamaCacheFlowAttention(self.scaling, self.head_dim)
82
+ self.attn = GPTNeoXCacheFlowAttention(self.scaling, self.head_dim)
83
84
def forward(
85
self,
cacheflow/models/memory_analyzer.py
+6
-6
@@ -202,8 +202,8 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
202
# estimating
203
# 1) the maximum activation tensor size during inference
204
# 2) the residual tensor size during inference
205
- # Here, we assume that FlashAttention is used and
206
- # thus the attention maps are never materialized in GPU DRAM.
205
+ # Here, we assume that we use memory-efficient attention which
206
+ # does not materialize the attention maps in GPU DRAM.
207
residual = max_num_batched_tokens * self.hidden_size
208
qkv = 3 * (max_num_batched_tokens * self.hidden_size) // self.tensor_parallel_size
209
ffn = max_num_batched_tokens * self.ffn_size // self.tensor_parallel_size
@@ -277,8 +277,8 @@ class LlamaMemoryAnalyzer(CacheFlowMemoryAnalyzer):
277
# estimating
278
# 1) the maximum activation tensor size during inference
279
# 2) the residual tensor size during inference
280
- # Here, we assume that FlashAttention is used and
281
- # thus the attention maps are never materialized in GPU DRAM.
280
+ # Here, we assume that we use memory-efficient attention which
281
+ # does not materialize the attention maps in GPU DRAM.
282
residual = max_num_batched_tokens * self.hidden_size
283
qkv = 3 * (max_num_batched_tokens * self.hidden_size) // self.tensor_parallel_size
284
ffn = 2 * (max_num_batched_tokens * self.ffn_size) // self.tensor_parallel_size
@@ -353,8 +353,8 @@ class GPTNeoXMemoryAnalyzer(CacheFlowMemoryAnalyzer):
353
# estimating
354
# 1) the maximum activation tensor size during inference
355
# 2) the residual tensor size during inference
356
- # Here, we assume that FlashAttention is used and
357
- # thus the attention maps are never materialized in GPU DRAM.
356
+ # Here, we assume that we use memory-efficient attention which
357
+ # does not materialize the attention maps in GPU DRAM.
358
residual = max_num_batched_tokens * self.hidden_size
359
qkv = 3 * (max_num_batched_tokens * self.hidden_size) // self.tensor_parallel_size
360
ffn = 2 * (max_num_batched_tokens * self.ffn_size) // self.tensor_parallel_size
cacheflow/models/opt.py
+2
-2
@@ -6,7 +6,7 @@ from torch import nn
6
from transformers import OPTConfig
7
8
from cacheflow.models import InputMetadata
9
-from cacheflow.models.attention import OPTCacheFlowAttention
9
+from cacheflow.models.attention import GPTCacheFlowAttention
10
from cacheflow.models.sample import Sampler
11
from cacheflow.models.utils import (hf_model_weights_iterator,
12
load_tensor_parallel_weights)
@@ -55,7 +55,7 @@ class OPTAttention(nn.Module):
55
self.out_proj = RowParallelLinear(embed_dim, embed_dim, bias=bias,
56
input_is_parallel=True,
57
perform_initialization=False)
58
- self.attn = OPTCacheFlowAttention(scale=self.scaling)
58
+ self.attn = GPTCacheFlowAttention(scale=self.scaling)
59
60
def forward(
61
self,
cacheflow/worker/worker.py
-8
@@ -136,11 +136,6 @@ class Worker:
136
slot = block_number * self.block_size + block_offset
137
slot_mapping.append(slot)
138
139
- cumulative_prompt_lens: List[int] = [0]
140
- for prompt_len in prompt_lens:
141
- cumulative_prompt_lens.append(
142
- cumulative_prompt_lens[-1] + prompt_len)
143
-
139
# Add generation tokens.
140
max_context_len = 0
141
max_num_blocks_per_seq = 0
@@ -196,14 +191,11 @@ class Worker:
191
for block_table in generation_block_tables]
192
block_tables_tensor = torch.tensor(
193
padded_block_tables, dtype=torch.int, device='cuda')
199
- cumulative_prompt_lens_tensor = torch.tensor(
200
- cumulative_prompt_lens, dtype=torch.int, device='cuda')
194
195
input_metadata = InputMetadata(
196
seq_groups=seq_groups,
197
seq_logprobs=seq_logprobs,
198
prompt_lens=prompt_lens,
206
- cumulative_prompt_lens=cumulative_prompt_lens_tensor,
199
slot_mapping=slot_mapping_tensor,
200
context_lens=context_lens_tensor,
201
max_context_len=max_context_len,
tests/kernels/activation.py
+1
-1
@@ -23,7 +23,7 @@ def test_silu_and_mul(
23
24
25
if __name__ == '__main__':
26
- for dtype in [torch.half, torch.float]:
26
+ for dtype in [torch.half, torch.bfloat16, torch.float]:
27
for num_tokens in [7, 83, 2048]:
28
for d in [512, 4096, 13824]:
29
print(f'Testing dtype={dtype}, num_tokens={num_tokens}, d={d}')
tests/kernels/attention.py
+35
-44
@@ -1,8 +1,9 @@
1
import random
2
from typing import List, Optional
3
4
-from flash_attn.flash_attn_interface import _flash_attn_forward
4
import torch
5
+from xformers import ops as xops
6
+from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
7
8
from cacheflow import attention_ops
9
@@ -81,8 +82,10 @@ def ref_multi_query_kv_attention(
82
end_idx = cu_seq_lens[i + 1]
83
seq_len = end_idx - start_idx
84
84
- # Create attention mask
85
- attn_mask = torch.triu(torch.ones(seq_len, seq_len), diagonal=1) * -1e5
85
+ # Create attention mask.
86
+ attn_mask = torch.triu(
87
+ torch.ones(seq_len, seq_len, dtype=dtype), diagonal=1)
88
+ attn_mask = attn_mask * torch.finfo(dtype).min
89
attn_mask = attn_mask.to(dtype=dtype, device='cuda')
90
91
ref_output = ref_masked_attention(
@@ -160,21 +163,20 @@ def test_single_query_cached_kv_attention(
163
num_blocks: int,
164
dtype: torch.dtype,
165
) -> None:
163
- qkv = torch.randn(
166
+ qkv = torch.empty(
167
num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
168
+ qkv.uniform_(-1e-3, 1e-3)
169
query, _, _ = qkv.unbind(dim=1)
170
+
171
x = 16 // torch.tensor([], dtype=dtype).element_size()
172
key_block_shape = (num_heads, head_size // x, block_size, x)
168
- key_cache = torch.randn(
173
+ key_cache = torch.empty(
174
size=(num_blocks, *key_block_shape), dtype=dtype, device='cuda')
175
+ key_cache.uniform_(-1e-3, 1e-3)
176
value_block_shape = (num_heads, head_size, block_size)
171
- value_cache = torch.randn(
177
+ value_cache = torch.empty(
178
size=(num_blocks, *value_block_shape), dtype=dtype, device='cuda')
173
-
174
- # Adjust the range of the values to reduce precision errors.
175
- query = query / (head_size ** 0.5)
176
- key_cache = key_cache / (head_size ** 0.5)
177
- value_cache = value_cache / (head_size ** 0.5)
179
+ value_cache.uniform_(-1e-3, 1e-3)
180
181
context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_tokens)]
182
max_context_len = max(context_lens)
@@ -228,39 +230,30 @@ def test_multi_query_kv_attention(
230
dtype: torch.dtype,
231
) -> None:
232
seq_lens = random.sample(range(1, MAX_SEQ_LEN), num_seqs)
231
- max_seq_len = max(seq_lens)
233
num_tokens = sum(seq_lens)
234
234
- cu_seq_lens = [0]
235
- for seq_len in seq_lens:
236
- cu_seq_lens.append(cu_seq_lens[-1] + seq_len)
237
- cu_seq_lens = torch.tensor(cu_seq_lens, dtype=torch.int, device='cuda')
238
-
235
scale = float(1.0 / (head_size ** 0.5))
240
- qkv = torch.randn(
236
+ qkv = torch.empty(
237
num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
242
- # Adjust the range of the values to reduce precision errors.
243
- qkv = qkv / (head_size ** 0.5)
244
-
238
+ qkv.uniform_(-1e-3, 1e-3)
239
query, key, value = qkv.unbind(dim=1)
246
- output = torch.empty(
247
- num_tokens, num_heads, head_size, dtype=dtype, device='cuda')
248
- _flash_attn_forward(
249
- query,
250
- key,
251
- value,
252
- output,
253
- cu_seq_lens,
254
- cu_seq_lens,
255
- max_seq_len,
256
- max_seq_len,
257
- dropout_p=0.0,
258
- softmax_scale=scale,
259
- causal=True,
260
- return_softmax=False,
240
+
241
+ attn_op = xops.fmha.cutlass.FwOp()
242
+ attn_bias = BlockDiagonalCausalMask.from_seqlens(seq_lens)
243
+ output = xops.memory_efficient_attention_forward(
244
+ query.unsqueeze(0),
245
+ key.unsqueeze(0),
246
+ value.unsqueeze(0),
247
+ attn_bias=attn_bias,
248
+ p=0.0,
249
+ scale=scale,
250
+ op=attn_op,
251
)
252
+ output = output.squeeze(0)
253
263
- cu_seq_lens = cu_seq_lens.cpu().tolist()
254
+ cu_seq_lens = [0]
255
+ for seq_len in seq_lens:
256
+ cu_seq_lens.append(cu_seq_lens[-1] + seq_len)
257
ref_output = ref_multi_query_kv_attention(
258
cu_seq_lens,
259
query,
@@ -277,8 +270,8 @@ def test_attention(seed: int) -> None:
270
# the test fails due to the precision issue. Re-run the test if it fails.
271
torch.random.manual_seed(seed)
272
torch.cuda.manual_seed(seed)
280
- for dtype in [torch.half, torch.float]:
281
- for block_size in [8, 16, 32]:
273
+ for dtype in [torch.half, torch.bfloat16]:
274
+ for block_size in [8, 16, 32, 64]:
275
for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
276
print(f'Testing single_query_cached_kv_attention with '
277
f'dtype={dtype}, block_size={block_size}, '
@@ -292,14 +285,12 @@ def test_attention(seed: int) -> None:
285
dtype=dtype,
286
)
287
295
- # NOTE(woosuk): FlashAttention does not support FP32.
296
- for dtype in [torch.half]:
297
- # NOTE(woosuk): FlashAttention does not support head_size > 128.
298
- for head_size in [64, 80, 96, 128]:
288
+ for dtype in [torch.half, torch.bfloat16]:
289
+ for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
290
print(f'Testing multi_query_kv_attention with dtype={dtype}, '
291
f'head_size={head_size}')
292
test_multi_query_kv_attention(
302
- num_seqs=11,
293
+ num_seqs=5,
294
num_heads=3,
295
head_size=head_size,
296
dtype=dtype,
tests/kernels/cache.py
+10
-9
@@ -142,15 +142,16 @@ def test_gather_cached_kv(
142
143
@torch.inference_mode()
144
def test_cache() -> None:
145
- test_copy_blocks(
146
- num_mappings=23, num_layers=7, num_heads=17, head_size=16,
147
- block_size=8, num_blocks=1024, dtype=torch.half)
148
- test_reshape_and_cache(
149
- num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
150
- dtype=torch.half)
151
- test_gather_cached_kv(
152
- num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
153
- dtype=torch.half)
145
+ for dtype in [torch.half, torch.bfloat16, torch.float]:
146
+ test_copy_blocks(
147
+ num_mappings=23, num_layers=7, num_heads=17, head_size=16,
148
+ block_size=8, num_blocks=1024, dtype=dtype)
149
+ test_reshape_and_cache(
150
+ num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
151
+ dtype=dtype)
152
+ test_gather_cached_kv(
153
+ num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
154
+ dtype=dtype)
155
156
157
if __name__ == '__main__':
tests/kernels/layernorm.py
+3
-2
@@ -8,7 +8,8 @@ class RefRMSNorm(nn.Module):
8
9
def __init__(self, hidden_size, eps=1e-6):
10
super().__init__()
11
- weight = torch.randn(hidden_size) / (hidden_size ** 0.5)
11
+ weight = torch.empty(hidden_size)
12
+ weight.uniform_(-1e-3, 1e-3)
13
self.weight = nn.Parameter(weight)
14
self.variance_epsilon = eps
15
@@ -41,7 +42,7 @@ def test_rms_norm(
42
43
44
if __name__ == '__main__':
44
- for dtype in [torch.half, torch.float]:
45
+ for dtype in [torch.half, torch.bfloat16, torch.float]:
46
for num_tokens in [7, 128, 2048]:
47
for hidden_size in [13, 64, 1024, 5120]:
48
print(f'Testing RMS kernel with dtype={dtype}, num_tokens='
tests/kernels/pos_encoding.py
+1
-1
@@ -129,7 +129,7 @@ def test_rotary_embedding_neox(
129
130
131
if __name__ == '__main__':
132
- for dtype in [torch.half, torch.float]:
132
+ for dtype in [torch.half, torch.bfloat16, torch.float]:
133
for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
134
print(f'Running tests for head_size={head_size} and dtype={dtype}')
135
test_rotary_embedding_neox(