@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d4bc1a4d24
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add unoptimized OPT Attention

Woosuk Kwon committed Feb 23, 2023 at 09:31 UTC d4bc1a4d248a5d23e1f731ecb53511a9a54f5dfc
2 files changed +177 -14
cacheflow/models/attention.py new
+118
@@ -0,0 +1,118 @@
1 +from typing import Optional, Tuple
2 +
3 +import torch
4 +import torch.nn as nn
5 +import xformers.ops as xops
6 +
7 +from cacheflow import ops
8 +from cacheflow.models import InputMetadata
9 +
10 +
11 +class OPTCacheFlowAttention(nn.Module):
12 +
13 + def __init__(self, scale: float) -> None:
14 + super().__init__()
15 + self.scale = scale
16 +
17 + # Shape-agnostic attention mask.
18 + self.attention_mask = xops.LowerTriangularMask()
19 +
20 + def multi_query_kv_attention(
21 + self,
22 + output: torch.Tensor,
23 + query: torch.Tensor,
24 + key: torch.Tensor,
25 + value: torch.Tensor,
26 + ) -> None:
27 + out = xops.memory_efficient_attention(
28 + query, key, value, attn_bias=self.attention_mask, scale=self.scale)
29 + # FIXME(woosuk): Directly write the attention output.
30 + output.copy_(out, non_blocking=True)
31 +
32 + def single_query_cached_kv_attention(
33 + self,
34 + output: torch.Tensor,
35 + query: torch.Tensor,
36 + key_cache: torch.Tensor,
37 + value_cache: torch.Tensor,
38 + input_metadata: InputMetadata,
39 + ) -> None:
40 + num_heads = value_cache.shape[1]
41 + head_size = value_cache.shape[3]
42 + block_size = value_cache.shape[2]
43 + block_tables = input_metadata.block_tables
44 +
45 + # FIXME(woosuk): Replace the following with a custom op.
46 + for i in range(input_metadata.num_generation_tokens):
47 + q = query[i]
48 + block_table = block_tables[i]
49 + context_len = int(input_metadata.context_lens[i])
50 + keys = []
51 + for j in range(context_len):
52 + block_number = block_table[j // block_size]
53 + block_offset = j % block_size
54 + k = key_cache[block_number, :, :, block_offset, :]
55 + k = k.view(num_heads, head_size)
56 + keys.append(k)
57 + keys = torch.stack(keys, dim=-1)
58 + logits = q @ keys
59 + attention_weights = torch.softmax(logits, dim=-1)
60 +
61 + values = []
62 + for j in range(context_len):
63 + block_number = block_table[j // block_size]
64 + block_offset = j % block_size
65 + v = value_cache[block_number, :, block_offset, :]
66 + values.append(v)
67 + values = torch.stack(values, dim=-1)
68 + out = attention_weights @ values
69 + output[i].copy_(out, non_blocking=True)
70 +
71 + def forward(
72 + self,
73 + query: torch.Tensor,
74 + key: torch.Tensor,
75 + value: torch.Tensor,
76 + key_cache: torch.Tensor,
77 + value_cache: torch.Tensor,
78 + input_metadata: InputMetadata,
79 + cache_event: Optional[torch.cuda.Event],
80 + ) -> torch.Tensor:
81 + # Reshape the input tensors.
82 + num_heads = value_cache.shape[1]
83 + head_size = value_cache.shape[3]
84 + query = query.view(-1, num_heads, head_size)
85 + key = key.view(-1, num_heads, head_size)
86 + value = value.view(-1, num_heads, head_size)
87 +
88 + # Compute the attention op for prompts.
89 + output = torch.empty_like(query)
90 + start_idx = 0
91 + for i in range(input_metadata.num_prompts):
92 + prompt_len = input_metadata.prompt_lens[i]
93 + out = output[start_idx:start_idx + prompt_len]
94 + q = query[start_idx:start_idx + prompt_len]
95 + k = key[start_idx:start_idx + prompt_len]
96 + v = value[start_idx:start_idx + prompt_len]
97 + self.multi_query_kv_attention(out, q, k, v)
98 + start_idx += prompt_len
99 +
100 + # Wait until the cache op is done.
101 + if cache_event is not None:
102 + cache_event.wait()
103 +
104 + # Reshape the keys and values and store them in the cache.
105 + ops.reshape_and_cache(
106 + key, value, key_cache, value_cache, input_metadata.slot_mapping)
107 +
108 + if input_metadata.num_generation_tokens > 0:
109 + # Compute the attention op for generation tokens.
110 + self.single_query_cached_kv_attention(
111 + output[start_idx:],
112 + query[start_idx:],
113 + key_cache,
114 + value_cache,
115 + input_metadata)
116 +
117 + # Reshape the output tensor.
118 + return output.view(-1, num_heads * head_size)
cacheflow/models/opt.py
+59 -14
@@ -1,9 +1,17 @@
1 """1D OPT model compatible with HuggingFace weights."""
2 +from typing import Dict, List, Optional, Tuple
3 +
4 import torch
5 from torch import nn
6 from transformers import OPTConfig
7 from transformers import PreTrainedModel
8
9 +from cacheflow.models import InputMetadata
10 +from cacheflow.models.attention import OPTCacheFlowAttention
11 +from cacheflow.models.sample import Sampler
12 +
13 +KVCache = Tuple[torch.Tensor, torch.Tensor]
14 +
15
16 class OPTLearnedPositionalEmbedding(nn.Embedding):
17
@@ -31,17 +39,27 @@ class OPTAttention(nn.Module):
39 self.head_dim = embed_dim // num_heads
40 self.scaling = self.head_dim**-0.5
41
42 + # TODO(woosuk): Fuse the three linear layers into one QKV linear layer.
43 self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
44 self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
45 self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
46 self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
47
39 - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
40 - q = self.q_proj(hidden_states) * self.scaling
48 + self.attn = OPTCacheFlowAttention(scale=self.scaling)
49 +
50 + def forward(
51 + self,
52 + hidden_states: torch.Tensor,
53 + kv_cache: KVCache,
54 + input_metadata: InputMetadata,
55 + cache_event: Optional[torch.cuda.Event],
56 + ) -> torch.Tensor:
57 + q = self.q_proj(hidden_states)
58 k = self.k_proj(hidden_states)
59 v = self.v_proj(hidden_states)
43 - # TODO
44 - attn_output = None
60 + key_cache, value_cache = kv_cache
61 + attn_output = self.attn(
62 + q, k, v, key_cache, value_cache, input_metadata, cache_event)
63 output = self.out_proj(attn_output)
64 return output
65
@@ -66,13 +84,23 @@ class OPTDecoderLayer(nn.Module):
84 self.fc2 = nn.Linear(config.ffn_dim, self.embed_dim, bias=config.enable_bias)
85 self.final_layer_norm = nn.LayerNorm(self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine)
86
69 - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
87 + def forward(
88 + self,
89 + hidden_states: torch.Tensor,
90 + kv_cache: KVCache,
91 + input_metadata: InputMetadata,
92 + cache_event: Optional[torch.cuda.Event],
93 + ) -> torch.Tensor:
94 # Self Attention
95 residual = hidden_states
96 # 125m, 1.7B, ..., 175B applies layer norm BEFORE attention
97 if self.do_layer_norm_before:
98 hidden_states = self.self_attn_layer_norm(hidden_states)
75 - hidden_states = self.self_attn(hidden_states=hidden_states)
99 + hidden_states = self.self_attn(
100 + hidden_states=hidden_states,
101 + kv_cache=kv_cache,
102 + input_metadata=input_metadata,
103 + cache_event=cache_event)
104 hidden_states = residual + hidden_states
105 # 350m applies layer norm AFTER attention
106 if not self.do_layer_norm_before:
@@ -145,6 +173,9 @@ class OPTDecoder(OPTPreTrainedModel):
173 self,
174 input_ids: torch.LongTensor,
175 positions: torch.LongTensor,
176 + kv_caches: List[KVCache],
177 + input_metadata: InputMetadata,
178 + cache_events: Optional[List[torch.cuda.Event]],
179 ) -> torch.Tensor:
180 inputs_embeds = self.embed_tokens(input_ids)
181 pos_embeds = self.embed_positions(positions)
@@ -153,8 +184,14 @@ class OPTDecoder(OPTPreTrainedModel):
184 inputs_embeds = self.project_in(inputs_embeds)
185 hidden_states = inputs_embeds + pos_embeds
186
156 - for layer in self.layers:
157 - hidden_states = layer(hidden_states)
187 + for i in range(len(self.layers)):
188 + if cache_events is None:
189 + cache_event = None
190 + else:
191 + cache_event = cache_events[i]
192 + layer = self.layers[i]
193 + hidden_states = layer(
194 + hidden_states, kv_caches[i], input_metadata, cache_event)
195
196 if self.final_layer_norm is not None:
197 hidden_states = self.final_layer_norm(hidden_states)
@@ -175,8 +212,12 @@ class OPTModel(OPTPreTrainedModel):
212 self,
213 input_ids: torch.LongTensor,
214 positions: torch.LongTensor,
215 + kv_caches: List[KVCache],
216 + input_metadata: InputMetadata,
217 + cache_events: Optional[List[torch.cuda.Event]],
218 ) -> torch.Tensor:
179 - return self.decoder(input_ids, positions)
219 + return self.decoder(
220 + input_ids, positions, kv_caches, input_metadata, cache_events)
221
222
223 class OPTForCausalLM(OPTPreTrainedModel):
@@ -185,9 +226,9 @@ class OPTForCausalLM(OPTPreTrainedModel):
226 def __init__(self, config):
227 super().__init__(config)
228 self.model = OPTModel(config)
188 -
229 # the lm_head weight is automatically tied to the embed tokens weight
230 self.lm_head = nn.Linear(config.word_embed_proj_dim, config.vocab_size, bias=False)
231 + self.sampler = Sampler(embedding=self.lm_head.weight)
232
233 # Initialize weights and apply final processing
234 self.post_init()
@@ -196,7 +237,11 @@ class OPTForCausalLM(OPTPreTrainedModel):
237 self,
238 input_ids: torch.LongTensor,
239 positions: torch.LongTensor,
199 - ) -> torch.Tensor:
200 - hidden_states = self.model.decoder(input_ids, positions)
201 - logits = self.lm_head(hidden_states).contiguous()
202 - return logits
240 + kv_caches: List[KVCache],
241 + input_metadata: InputMetadata,
242 + cache_events: Optional[List[torch.cuda.Event]],
243 + ) -> Dict[int, Tuple[int, int]]:
244 + hidden_states = self.model(
245 + input_ids, positions, kv_caches, input_metadata, cache_events)
246 + next_tokens = self.sampler(hidden_states, input_metadata)
247 + return next_tokens