@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c9d5b6d4a8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Replace FlashAttention with xformers (#70)

Woosuk Kwon committed May 5, 2023 at 02:01 UTC c9d5b6d4a8b3f51ff6c9eee7eb52bb5149d89b6a
13 files changed +87 -131
README.md
+1 -5
@@ -3,11 +3,7 @@
3 ## Installation
4
5 ```bash
6 -pip install psutil numpy ray torch
7 -pip install git+https://github.com/huggingface/transformers # Required for LLaMA.
8 -pip install sentencepiece # Required for LlamaTokenizer.
9 -pip install ninja # To parallelize the compilation of flash-attn.
10 -pip install flash-attn # This may take up to 10 mins.
6 +pip install ninja psutil numpy sentencepiece ray torch transformers xformers
7 pip install -e .
8 ```
9
cacheflow/master/server.py
+1 -1
@@ -213,7 +213,7 @@ def add_server_arguments(parser: argparse.ArgumentParser):
213 parser.add_argument('--use-np-cache', action='store_true',
214 help='save a numpy copy of model weights for faster loading')
215 parser.add_argument('--use-dummy-weights', action='store_true', help='use dummy values for model weights')
216 - # NOTE(woosuk): FlashAttention does not support float32.
216 + # TODO(woosuk): Support FP32 for debugging.
217 parser.add_argument('--dtype', type=str, default='default', choices=['default', 'half', 'bfloat16'],
218 help=('data type for model weights and activations. '
219 'The "default" option will use FP16 precision '
cacheflow/models/attention.py
+16 -38
@@ -1,8 +1,8 @@
1 from typing import Optional
2
3 -from flash_attn.flash_attn_interface import _flash_attn_forward
3 import torch
4 import torch.nn as nn
5 +from xformers import ops as xops
6
7 from cacheflow import attention_ops
8 from cacheflow import cache_ops
@@ -15,6 +15,7 @@ class GPTCacheFlowAttention(nn.Module):
15 def __init__(self, scale: float) -> None:
16 super().__init__()
17 self.scale = float(scale)
18 + self.attn_op = xops.fmha.cutlass.FwOp()
19
20 def multi_query_kv_attention(
21 self,
@@ -22,32 +23,21 @@ class GPTCacheFlowAttention(nn.Module):
23 query: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
24 key: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
25 value: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
25 - cumulative_prompt_lens: torch.Tensor, # [num_prompts + 1]
26 - max_prompt_len: int,
26 + attn_bias: xops.AttentionBias,
27 ) -> None:
28 - if query.dtype == torch.float:
29 - raise ValueError('The float data type is not supported by '
30 - 'FlashAttention. Use the half data type instead.')
31 - head_size = query.shape[-1]
32 - if head_size > 128:
33 - raise ValueError('FlashAttention does not support head_size > 128.')
34 -
35 - # Directly call FlashAttention's internal function to avoid allocating
36 - # a new tensor for the output.
37 - _flash_attn_forward(
38 - query,
39 - key,
40 - value,
41 - output,
42 - cumulative_prompt_lens,
43 - cumulative_prompt_lens,
44 - max_prompt_len,
45 - max_prompt_len,
46 - dropout_p=0.0,
47 - softmax_scale=self.scale,
48 - causal=True,
49 - return_softmax=False,
28 + # TODO(woosuk): The unsqueeze op may incur some CPU overhead. Optimize.
29 + out = xops.memory_efficient_attention_forward(
30 + query.unsqueeze(0),
31 + key.unsqueeze(0),
32 + value.unsqueeze(0),
33 + attn_bias=attn_bias,
34 + p=0.0,
35 + scale=self.scale,
36 + op=self.attn_op,
37 )
38 + # TODO(woosuk): Unnecessary copy. Optimize.
39 + output.copy_(out.squeeze(0))
40 + return output
41
42 def single_query_cached_kv_attention(
43 self,
@@ -109,8 +99,7 @@ class GPTCacheFlowAttention(nn.Module):
99 query[:num_prompt_tokens],
100 key[:num_prompt_tokens],
101 value[:num_prompt_tokens],
112 - input_metadata.cumulative_prompt_lens,
113 - input_metadata.max_prompt_len,
102 + input_metadata.attn_bias,
103 )
104
105 # Wait until the cache op is done.
@@ -143,13 +132,6 @@ class GPTCacheFlowAttention(nn.Module):
132 return output.view(-1, num_heads * head_size)
133
134
146 -class OPTCacheFlowAttention(GPTCacheFlowAttention):
147 - """OPT uses the same attention mechanism as GPT."""
148 -
149 - def __init__(self, scale: float) -> None:
150 - super().__init__(scale)
151 -
152 -
135 class GPTNeoXCacheFlowAttention(GPTCacheFlowAttention):
136 """Attention with GPT-NeoX style rotary embedding."""
137
@@ -207,7 +189,3 @@ class GPTNeoXCacheFlowAttention(GPTCacheFlowAttention):
189 input_metadata,
190 cache_event,
191 )
210 -
211 -
212 -class LlamaCacheFlowAttention(GPTNeoXCacheFlowAttention):
213 - """LLaMA uses the GPT-NeoX style rotary embedding."""
cacheflow/models/input_metadata.py
+9 -12
@@ -1,6 +1,7 @@
1 from typing import List, Dict, Tuple
2
3 import torch
4 +from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
5
6 from cacheflow.sampling_params import SamplingParams
7
@@ -12,7 +13,6 @@ class InputMetadata:
13 seq_groups: List[Tuple[List[int], SamplingParams]],
14 seq_logprobs: Dict[int, float], # Seq id -> cumulative logprobs.
15 prompt_lens: List[int],
15 - cumulative_prompt_lens: torch.Tensor,
16 slot_mapping: torch.Tensor,
17 context_lens: torch.Tensor,
18 max_context_len: int,
@@ -21,15 +21,14 @@ class InputMetadata:
21 self.seq_groups = seq_groups
22 self.seq_logprobs = seq_logprobs
23 self.prompt_lens = prompt_lens
24 - self.cumulative_prompt_lens = cumulative_prompt_lens
24 self.slot_mapping = slot_mapping
25 self.context_lens = context_lens
26 self.max_context_len = max_context_len
27 self.block_tables = block_tables
28
29 + self.attn_bias = BlockDiagonalCausalMask.from_seqlens(prompt_lens)
30 self.num_prompts = len(prompt_lens)
31 self.num_prompt_tokens = sum(prompt_lens)
32 - self.max_prompt_len = max(prompt_lens) if prompt_lens else 0
32 self.num_generation_tokens = context_lens.shape[0]
33 self.num_valid_tokens = slot_mapping.shape[0]
34 if block_tables.numel() > 0:
@@ -41,15 +40,13 @@ class InputMetadata:
40
41 def __repr__(self) -> str:
42 return (f'InputMetadata('
44 - f'num_prompts={self.num_prompts}, '
45 - f'num_prompt_tokens={self.num_prompt_tokens}, '
46 - f'max_prompt_len={self.max_prompt_len}, '
47 - f'num_generation_tokens={self.num_generation_tokens}, '
43 f'num_valid_tokens={self.num_valid_tokens}, '
49 - f'max_num_blocks_per_seq={self.max_num_blocks_per_seq}, '
50 - f'max_context_len={self.max_context_len}), '
44 + f'num_prompt_tokens={self.num_prompt_tokens}, '
45 + f'num_prompts={self.num_prompts}, '
46 f'prompt_lens={self.prompt_lens}, '
52 - f'cumulative_prompt_lens={self.cumulative_prompt_lens}, '
53 - f'slot_mapping={self.slot_mapping}, '
47 + f'num_generation_tokens={self.num_generation_tokens}, '
48 f'context_lens={self.context_lens}, '
55 - f'block_tables={self.block_tables})')
49 + f'max_context_len={self.max_context_len}), '
50 + f'max_num_blocks_per_seq={self.max_num_blocks_per_seq}, '
51 + f'block_tables={self.block_tables}), '
52 + f'slot_mapping={self.slot_mapping}')
cacheflow/models/llama.py
+2 -2
@@ -7,7 +7,7 @@ from transformers import LlamaConfig
7
8 from cacheflow.models import InputMetadata
9 from cacheflow.models.activation import SiluAndMul
10 -from cacheflow.models.attention import LlamaCacheFlowAttention
10 +from cacheflow.models.attention import GPTNeoXCacheFlowAttention
11 from cacheflow.models.layernorm import RMSNorm
12 from cacheflow.models.sample import Sampler
13 from cacheflow.models.utils import (hf_model_weights_iterator,
@@ -79,7 +79,7 @@ class LlamaAttention(nn.Module):
79 input_is_parallel=True,
80 perform_initialization=False,
81 )
82 - self.attn = LlamaCacheFlowAttention(self.scaling, self.head_dim)
82 + self.attn = GPTNeoXCacheFlowAttention(self.scaling, self.head_dim)
83
84 def forward(
85 self,
cacheflow/models/memory_analyzer.py
+6 -6
@@ -202,8 +202,8 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
202 # estimating
203 # 1) the maximum activation tensor size during inference
204 # 2) the residual tensor size during inference
205 - # Here, we assume that FlashAttention is used and
206 - # thus the attention maps are never materialized in GPU DRAM.
205 + # Here, we assume that we use memory-efficient attention which
206 + # does not materialize the attention maps in GPU DRAM.
207 residual = max_num_batched_tokens * self.hidden_size
208 qkv = 3 * (max_num_batched_tokens * self.hidden_size) // self.tensor_parallel_size
209 ffn = max_num_batched_tokens * self.ffn_size // self.tensor_parallel_size
@@ -277,8 +277,8 @@ class LlamaMemoryAnalyzer(CacheFlowMemoryAnalyzer):
277 # estimating
278 # 1) the maximum activation tensor size during inference
279 # 2) the residual tensor size during inference
280 - # Here, we assume that FlashAttention is used and
281 - # thus the attention maps are never materialized in GPU DRAM.
280 + # Here, we assume that we use memory-efficient attention which
281 + # does not materialize the attention maps in GPU DRAM.
282 residual = max_num_batched_tokens * self.hidden_size
283 qkv = 3 * (max_num_batched_tokens * self.hidden_size) // self.tensor_parallel_size
284 ffn = 2 * (max_num_batched_tokens * self.ffn_size) // self.tensor_parallel_size
@@ -353,8 +353,8 @@ class GPTNeoXMemoryAnalyzer(CacheFlowMemoryAnalyzer):
353 # estimating
354 # 1) the maximum activation tensor size during inference
355 # 2) the residual tensor size during inference
356 - # Here, we assume that FlashAttention is used and
357 - # thus the attention maps are never materialized in GPU DRAM.
356 + # Here, we assume that we use memory-efficient attention which
357 + # does not materialize the attention maps in GPU DRAM.
358 residual = max_num_batched_tokens * self.hidden_size
359 qkv = 3 * (max_num_batched_tokens * self.hidden_size) // self.tensor_parallel_size
360 ffn = 2 * (max_num_batched_tokens * self.ffn_size) // self.tensor_parallel_size
cacheflow/models/opt.py
+2 -2
@@ -6,7 +6,7 @@ from torch import nn
6 from transformers import OPTConfig
7
8 from cacheflow.models import InputMetadata
9 -from cacheflow.models.attention import OPTCacheFlowAttention
9 +from cacheflow.models.attention import GPTCacheFlowAttention
10 from cacheflow.models.sample import Sampler
11 from cacheflow.models.utils import (hf_model_weights_iterator,
12 load_tensor_parallel_weights)
@@ -55,7 +55,7 @@ class OPTAttention(nn.Module):
55 self.out_proj = RowParallelLinear(embed_dim, embed_dim, bias=bias,
56 input_is_parallel=True,
57 perform_initialization=False)
58 - self.attn = OPTCacheFlowAttention(scale=self.scaling)
58 + self.attn = GPTCacheFlowAttention(scale=self.scaling)
59
60 def forward(
61 self,
cacheflow/worker/worker.py
-8
@@ -136,11 +136,6 @@ class Worker:
136 slot = block_number * self.block_size + block_offset
137 slot_mapping.append(slot)
138
139 - cumulative_prompt_lens: List[int] = [0]
140 - for prompt_len in prompt_lens:
141 - cumulative_prompt_lens.append(
142 - cumulative_prompt_lens[-1] + prompt_len)
143 -
139 # Add generation tokens.
140 max_context_len = 0
141 max_num_blocks_per_seq = 0
@@ -196,14 +191,11 @@ class Worker:
191 for block_table in generation_block_tables]
192 block_tables_tensor = torch.tensor(
193 padded_block_tables, dtype=torch.int, device='cuda')
199 - cumulative_prompt_lens_tensor = torch.tensor(
200 - cumulative_prompt_lens, dtype=torch.int, device='cuda')
194
195 input_metadata = InputMetadata(
196 seq_groups=seq_groups,
197 seq_logprobs=seq_logprobs,
198 prompt_lens=prompt_lens,
206 - cumulative_prompt_lens=cumulative_prompt_lens_tensor,
199 slot_mapping=slot_mapping_tensor,
200 context_lens=context_lens_tensor,
201 max_context_len=max_context_len,
tests/kernels/activation.py
+1 -1
@@ -23,7 +23,7 @@ def test_silu_and_mul(
23
24
25 if __name__ == '__main__':
26 - for dtype in [torch.half, torch.float]:
26 + for dtype in [torch.half, torch.bfloat16, torch.float]:
27 for num_tokens in [7, 83, 2048]:
28 for d in [512, 4096, 13824]:
29 print(f'Testing dtype={dtype}, num_tokens={num_tokens}, d={d}')
tests/kernels/attention.py
+35 -44
@@ -1,8 +1,9 @@
1 import random
2 from typing import List, Optional
3
4 -from flash_attn.flash_attn_interface import _flash_attn_forward
4 import torch
5 +from xformers import ops as xops
6 +from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
7
8 from cacheflow import attention_ops
9
@@ -81,8 +82,10 @@ def ref_multi_query_kv_attention(
82 end_idx = cu_seq_lens[i + 1]
83 seq_len = end_idx - start_idx
84
84 - # Create attention mask
85 - attn_mask = torch.triu(torch.ones(seq_len, seq_len), diagonal=1) * -1e5
85 + # Create attention mask.
86 + attn_mask = torch.triu(
87 + torch.ones(seq_len, seq_len, dtype=dtype), diagonal=1)
88 + attn_mask = attn_mask * torch.finfo(dtype).min
89 attn_mask = attn_mask.to(dtype=dtype, device='cuda')
90
91 ref_output = ref_masked_attention(
@@ -160,21 +163,20 @@ def test_single_query_cached_kv_attention(
163 num_blocks: int,
164 dtype: torch.dtype,
165 ) -> None:
163 - qkv = torch.randn(
166 + qkv = torch.empty(
167 num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
168 + qkv.uniform_(-1e-3, 1e-3)
169 query, _, _ = qkv.unbind(dim=1)
170 +
171 x = 16 // torch.tensor([], dtype=dtype).element_size()
172 key_block_shape = (num_heads, head_size // x, block_size, x)
168 - key_cache = torch.randn(
173 + key_cache = torch.empty(
174 size=(num_blocks, *key_block_shape), dtype=dtype, device='cuda')
175 + key_cache.uniform_(-1e-3, 1e-3)
176 value_block_shape = (num_heads, head_size, block_size)
171 - value_cache = torch.randn(
177 + value_cache = torch.empty(
178 size=(num_blocks, *value_block_shape), dtype=dtype, device='cuda')
173 -
174 - # Adjust the range of the values to reduce precision errors.
175 - query = query / (head_size ** 0.5)
176 - key_cache = key_cache / (head_size ** 0.5)
177 - value_cache = value_cache / (head_size ** 0.5)
179 + value_cache.uniform_(-1e-3, 1e-3)
180
181 context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_tokens)]
182 max_context_len = max(context_lens)
@@ -228,39 +230,30 @@ def test_multi_query_kv_attention(
230 dtype: torch.dtype,
231 ) -> None:
232 seq_lens = random.sample(range(1, MAX_SEQ_LEN), num_seqs)
231 - max_seq_len = max(seq_lens)
233 num_tokens = sum(seq_lens)
234
234 - cu_seq_lens = [0]
235 - for seq_len in seq_lens:
236 - cu_seq_lens.append(cu_seq_lens[-1] + seq_len)
237 - cu_seq_lens = torch.tensor(cu_seq_lens, dtype=torch.int, device='cuda')
238 -
235 scale = float(1.0 / (head_size ** 0.5))
240 - qkv = torch.randn(
236 + qkv = torch.empty(
237 num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
242 - # Adjust the range of the values to reduce precision errors.
243 - qkv = qkv / (head_size ** 0.5)
244 -
238 + qkv.uniform_(-1e-3, 1e-3)
239 query, key, value = qkv.unbind(dim=1)
246 - output = torch.empty(
247 - num_tokens, num_heads, head_size, dtype=dtype, device='cuda')
248 - _flash_attn_forward(
249 - query,
250 - key,
251 - value,
252 - output,
253 - cu_seq_lens,
254 - cu_seq_lens,
255 - max_seq_len,
256 - max_seq_len,
257 - dropout_p=0.0,
258 - softmax_scale=scale,
259 - causal=True,
260 - return_softmax=False,
240 +
241 + attn_op = xops.fmha.cutlass.FwOp()
242 + attn_bias = BlockDiagonalCausalMask.from_seqlens(seq_lens)
243 + output = xops.memory_efficient_attention_forward(
244 + query.unsqueeze(0),
245 + key.unsqueeze(0),
246 + value.unsqueeze(0),
247 + attn_bias=attn_bias,
248 + p=0.0,
249 + scale=scale,
250 + op=attn_op,
251 )
252 + output = output.squeeze(0)
253
263 - cu_seq_lens = cu_seq_lens.cpu().tolist()
254 + cu_seq_lens = [0]
255 + for seq_len in seq_lens:
256 + cu_seq_lens.append(cu_seq_lens[-1] + seq_len)
257 ref_output = ref_multi_query_kv_attention(
258 cu_seq_lens,
259 query,
@@ -277,8 +270,8 @@ def test_attention(seed: int) -> None:
270 # the test fails due to the precision issue. Re-run the test if it fails.
271 torch.random.manual_seed(seed)
272 torch.cuda.manual_seed(seed)
280 - for dtype in [torch.half, torch.float]:
281 - for block_size in [8, 16, 32]:
273 + for dtype in [torch.half, torch.bfloat16]:
274 + for block_size in [8, 16, 32, 64]:
275 for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
276 print(f'Testing single_query_cached_kv_attention with '
277 f'dtype={dtype}, block_size={block_size}, '
@@ -292,14 +285,12 @@ def test_attention(seed: int) -> None:
285 dtype=dtype,
286 )
287
295 - # NOTE(woosuk): FlashAttention does not support FP32.
296 - for dtype in [torch.half]:
297 - # NOTE(woosuk): FlashAttention does not support head_size > 128.
298 - for head_size in [64, 80, 96, 128]:
288 + for dtype in [torch.half, torch.bfloat16]:
289 + for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
290 print(f'Testing multi_query_kv_attention with dtype={dtype}, '
291 f'head_size={head_size}')
292 test_multi_query_kv_attention(
302 - num_seqs=11,
293 + num_seqs=5,
294 num_heads=3,
295 head_size=head_size,
296 dtype=dtype,
tests/kernels/cache.py
+10 -9
@@ -142,15 +142,16 @@ def test_gather_cached_kv(
142
143 @torch.inference_mode()
144 def test_cache() -> None:
145 - test_copy_blocks(
146 - num_mappings=23, num_layers=7, num_heads=17, head_size=16,
147 - block_size=8, num_blocks=1024, dtype=torch.half)
148 - test_reshape_and_cache(
149 - num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
150 - dtype=torch.half)
151 - test_gather_cached_kv(
152 - num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
153 - dtype=torch.half)
145 + for dtype in [torch.half, torch.bfloat16, torch.float]:
146 + test_copy_blocks(
147 + num_mappings=23, num_layers=7, num_heads=17, head_size=16,
148 + block_size=8, num_blocks=1024, dtype=dtype)
149 + test_reshape_and_cache(
150 + num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
151 + dtype=dtype)
152 + test_gather_cached_kv(
153 + num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
154 + dtype=dtype)
155
156
157 if __name__ == '__main__':
tests/kernels/layernorm.py
+3 -2
@@ -8,7 +8,8 @@ class RefRMSNorm(nn.Module):
8
9 def __init__(self, hidden_size, eps=1e-6):
10 super().__init__()
11 - weight = torch.randn(hidden_size) / (hidden_size ** 0.5)
11 + weight = torch.empty(hidden_size)
12 + weight.uniform_(-1e-3, 1e-3)
13 self.weight = nn.Parameter(weight)
14 self.variance_epsilon = eps
15
@@ -41,7 +42,7 @@ def test_rms_norm(
42
43
44 if __name__ == '__main__':
44 - for dtype in [torch.half, torch.float]:
45 + for dtype in [torch.half, torch.bfloat16, torch.float]:
46 for num_tokens in [7, 128, 2048]:
47 for hidden_size in [13, 64, 1024, 5120]:
48 print(f'Testing RMS kernel with dtype={dtype}, num_tokens='
tests/kernels/pos_encoding.py
+1 -1
@@ -129,7 +129,7 @@ def test_rotary_embedding_neox(
129
130
131 if __name__ == '__main__':
132 - for dtype in [torch.half, torch.float]:
132 + for dtype in [torch.half, torch.bfloat16, torch.float]:
133 for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
134 print(f'Running tests for head_size={head_size} and dtype={dtype}')
135 test_rotary_embedding_neox(