A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add unoptimized OPT Attention
Woosuk Kwon committed
Feb 23, 2023 at 09:31 UTC
d4bc1a4d248a5d23e1f731ecb53511a9a54f5dfc
2 files changed
+177
-14
cacheflow/models/attention.py
new
+118
@@ -0,0 +1,118 @@
1
+from typing import Optional, Tuple
2
+
3
+import torch
4
+import torch.nn as nn
5
+import xformers.ops as xops
6
+
7
+from cacheflow import ops
8
+from cacheflow.models import InputMetadata
9
+
10
+
11
+class OPTCacheFlowAttention(nn.Module):
12
+
13
+ def __init__(self, scale: float) -> None:
14
+ super().__init__()
15
+ self.scale = scale
16
+
17
+ # Shape-agnostic attention mask.
18
+ self.attention_mask = xops.LowerTriangularMask()
19
+
20
+ def multi_query_kv_attention(
21
+ self,
22
+ output: torch.Tensor,
23
+ query: torch.Tensor,
24
+ key: torch.Tensor,
25
+ value: torch.Tensor,
26
+ ) -> None:
27
+ out = xops.memory_efficient_attention(
28
+ query, key, value, attn_bias=self.attention_mask, scale=self.scale)
29
+ # FIXME(woosuk): Directly write the attention output.
30
+ output.copy_(out, non_blocking=True)
31
+
32
+ def single_query_cached_kv_attention(
33
+ self,
34
+ output: torch.Tensor,
35
+ query: torch.Tensor,
36
+ key_cache: torch.Tensor,
37
+ value_cache: torch.Tensor,
38
+ input_metadata: InputMetadata,
39
+ ) -> None:
40
+ num_heads = value_cache.shape[1]
41
+ head_size = value_cache.shape[3]
42
+ block_size = value_cache.shape[2]
43
+ block_tables = input_metadata.block_tables
44
+
45
+ # FIXME(woosuk): Replace the following with a custom op.
46
+ for i in range(input_metadata.num_generation_tokens):
47
+ q = query[i]
48
+ block_table = block_tables[i]
49
+ context_len = int(input_metadata.context_lens[i])
50
+ keys = []
51
+ for j in range(context_len):
52
+ block_number = block_table[j // block_size]
53
+ block_offset = j % block_size
54
+ k = key_cache[block_number, :, :, block_offset, :]
55
+ k = k.view(num_heads, head_size)
56
+ keys.append(k)
57
+ keys = torch.stack(keys, dim=-1)
58
+ logits = q @ keys
59
+ attention_weights = torch.softmax(logits, dim=-1)
60
+
61
+ values = []
62
+ for j in range(context_len):
63
+ block_number = block_table[j // block_size]
64
+ block_offset = j % block_size
65
+ v = value_cache[block_number, :, block_offset, :]
66
+ values.append(v)
67
+ values = torch.stack(values, dim=-1)
68
+ out = attention_weights @ values
69
+ output[i].copy_(out, non_blocking=True)
70
+
71
+ def forward(
72
+ self,
73
+ query: torch.Tensor,
74
+ key: torch.Tensor,
75
+ value: torch.Tensor,
76
+ key_cache: torch.Tensor,
77
+ value_cache: torch.Tensor,
78
+ input_metadata: InputMetadata,
79
+ cache_event: Optional[torch.cuda.Event],
80
+ ) -> torch.Tensor:
81
+ # Reshape the input tensors.
82
+ num_heads = value_cache.shape[1]
83
+ head_size = value_cache.shape[3]
84
+ query = query.view(-1, num_heads, head_size)
85
+ key = key.view(-1, num_heads, head_size)
86
+ value = value.view(-1, num_heads, head_size)
87
+
88
+ # Compute the attention op for prompts.
89
+ output = torch.empty_like(query)
90
+ start_idx = 0
91
+ for i in range(input_metadata.num_prompts):
92
+ prompt_len = input_metadata.prompt_lens[i]
93
+ out = output[start_idx:start_idx + prompt_len]
94
+ q = query[start_idx:start_idx + prompt_len]
95
+ k = key[start_idx:start_idx + prompt_len]
96
+ v = value[start_idx:start_idx + prompt_len]
97
+ self.multi_query_kv_attention(out, q, k, v)
98
+ start_idx += prompt_len
99
+
100
+ # Wait until the cache op is done.
101
+ if cache_event is not None:
102
+ cache_event.wait()
103
+
104
+ # Reshape the keys and values and store them in the cache.
105
+ ops.reshape_and_cache(
106
+ key, value, key_cache, value_cache, input_metadata.slot_mapping)
107
+
108
+ if input_metadata.num_generation_tokens > 0:
109
+ # Compute the attention op for generation tokens.
110
+ self.single_query_cached_kv_attention(
111
+ output[start_idx:],
112
+ query[start_idx:],
113
+ key_cache,
114
+ value_cache,
115
+ input_metadata)
116
+
117
+ # Reshape the output tensor.
118
+ return output.view(-1, num_heads * head_size)
cacheflow/models/opt.py
+59
-14
@@ -1,9 +1,17 @@
1
"""1D OPT model compatible with HuggingFace weights."""
2
+from typing import Dict, List, Optional, Tuple
3
+
4
import torch
5
from torch import nn
6
from transformers import OPTConfig
7
from transformers import PreTrainedModel
8
9
+from cacheflow.models import InputMetadata
10
+from cacheflow.models.attention import OPTCacheFlowAttention
11
+from cacheflow.models.sample import Sampler
12
+
13
+KVCache = Tuple[torch.Tensor, torch.Tensor]
14
+
15
16
class OPTLearnedPositionalEmbedding(nn.Embedding):
17
@@ -31,17 +39,27 @@ class OPTAttention(nn.Module):
39
self.head_dim = embed_dim // num_heads
40
self.scaling = self.head_dim**-0.5
41
42
+ # TODO(woosuk): Fuse the three linear layers into one QKV linear layer.
43
self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
44
self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
45
self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
46
self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
47
39
- def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
40
- q = self.q_proj(hidden_states) * self.scaling
48
+ self.attn = OPTCacheFlowAttention(scale=self.scaling)
49
+
50
+ def forward(
51
+ self,
52
+ hidden_states: torch.Tensor,
53
+ kv_cache: KVCache,
54
+ input_metadata: InputMetadata,
55
+ cache_event: Optional[torch.cuda.Event],
56
+ ) -> torch.Tensor:
57
+ q = self.q_proj(hidden_states)
58
k = self.k_proj(hidden_states)
59
v = self.v_proj(hidden_states)
43
- # TODO
44
- attn_output = None
60
+ key_cache, value_cache = kv_cache
61
+ attn_output = self.attn(
62
+ q, k, v, key_cache, value_cache, input_metadata, cache_event)
63
output = self.out_proj(attn_output)
64
return output
65
@@ -66,13 +84,23 @@ class OPTDecoderLayer(nn.Module):
84
self.fc2 = nn.Linear(config.ffn_dim, self.embed_dim, bias=config.enable_bias)
85
self.final_layer_norm = nn.LayerNorm(self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine)
86
69
- def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
87
+ def forward(
88
+ self,
89
+ hidden_states: torch.Tensor,
90
+ kv_cache: KVCache,
91
+ input_metadata: InputMetadata,
92
+ cache_event: Optional[torch.cuda.Event],
93
+ ) -> torch.Tensor:
94
# Self Attention
95
residual = hidden_states
96
# 125m, 1.7B, ..., 175B applies layer norm BEFORE attention
97
if self.do_layer_norm_before:
98
hidden_states = self.self_attn_layer_norm(hidden_states)
75
- hidden_states = self.self_attn(hidden_states=hidden_states)
99
+ hidden_states = self.self_attn(
100
+ hidden_states=hidden_states,
101
+ kv_cache=kv_cache,
102
+ input_metadata=input_metadata,
103
+ cache_event=cache_event)
104
hidden_states = residual + hidden_states
105
# 350m applies layer norm AFTER attention
106
if not self.do_layer_norm_before:
@@ -145,6 +173,9 @@ class OPTDecoder(OPTPreTrainedModel):
173
self,
174
input_ids: torch.LongTensor,
175
positions: torch.LongTensor,
176
+ kv_caches: List[KVCache],
177
+ input_metadata: InputMetadata,
178
+ cache_events: Optional[List[torch.cuda.Event]],
179
) -> torch.Tensor:
180
inputs_embeds = self.embed_tokens(input_ids)
181
pos_embeds = self.embed_positions(positions)
@@ -153,8 +184,14 @@ class OPTDecoder(OPTPreTrainedModel):
184
inputs_embeds = self.project_in(inputs_embeds)
185
hidden_states = inputs_embeds + pos_embeds
186
156
- for layer in self.layers:
157
- hidden_states = layer(hidden_states)
187
+ for i in range(len(self.layers)):
188
+ if cache_events is None:
189
+ cache_event = None
190
+ else:
191
+ cache_event = cache_events[i]
192
+ layer = self.layers[i]
193
+ hidden_states = layer(
194
+ hidden_states, kv_caches[i], input_metadata, cache_event)
195
196
if self.final_layer_norm is not None:
197
hidden_states = self.final_layer_norm(hidden_states)
@@ -175,8 +212,12 @@ class OPTModel(OPTPreTrainedModel):
212
self,
213
input_ids: torch.LongTensor,
214
positions: torch.LongTensor,
215
+ kv_caches: List[KVCache],
216
+ input_metadata: InputMetadata,
217
+ cache_events: Optional[List[torch.cuda.Event]],
218
) -> torch.Tensor:
179
- return self.decoder(input_ids, positions)
219
+ return self.decoder(
220
+ input_ids, positions, kv_caches, input_metadata, cache_events)
221
222
223
class OPTForCausalLM(OPTPreTrainedModel):
@@ -185,9 +226,9 @@ class OPTForCausalLM(OPTPreTrainedModel):
226
def __init__(self, config):
227
super().__init__(config)
228
self.model = OPTModel(config)
188
-
229
# the lm_head weight is automatically tied to the embed tokens weight
230
self.lm_head = nn.Linear(config.word_embed_proj_dim, config.vocab_size, bias=False)
231
+ self.sampler = Sampler(embedding=self.lm_head.weight)
232
233
# Initialize weights and apply final processing
234
self.post_init()
@@ -196,7 +237,11 @@ class OPTForCausalLM(OPTPreTrainedModel):
237
self,
238
input_ids: torch.LongTensor,
239
positions: torch.LongTensor,
199
- ) -> torch.Tensor:
200
- hidden_states = self.model.decoder(input_ids, positions)
201
- logits = self.lm_head(hidden_states).contiguous()
202
- return logits
240
+ kv_caches: List[KVCache],
241
+ input_metadata: InputMetadata,
242
+ cache_events: Optional[List[torch.cuda.Event]],
243
+ ) -> Dict[int, Tuple[int, int]]:
244
+ hidden_states = self.model(
245
+ input_ids, positions, kv_caches, input_metadata, cache_events)
246
+ next_tokens = self.sampler(hidden_states, input_metadata)
247
+ return next_tokens