A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add support for GPT-NeoX (Pythia) (#50)
Woosuk Kwon committed
Apr 28, 2023 at 00:32 UTC
a96d63c21d18ad6610adfcabd3aae02c6357334e
9 files changed
+436
-71
cacheflow/models/attention.py
+11
-5
@@ -150,20 +150,20 @@ class OPTCacheFlowAttention(GPTCacheFlowAttention):
150
super().__init__(scale)
151
152
153
-class LlamaCacheFlowAttention(GPTCacheFlowAttention):
154
- """Llama uses GPT-NeoX style rotary embedding."""
153
+class GPTNeoXCacheFlowAttention(GPTCacheFlowAttention):
154
+ """Attention with GPT-NeoX style rotary embedding."""
155
156
def __init__(
157
self,
158
scale: float,
159
- head_size: int,
159
+ rotary_dim: int,
160
max_position: int = 8192,
161
base: int = 10000,
162
) -> None:
163
super().__init__(scale)
164
165
# Create the cos and sin cache.
166
- inv_freq = 1.0 / (base ** (torch.arange(0, head_size, 2) / head_size))
166
+ inv_freq = 1.0 / (base ** (torch.arange(0, rotary_dim, 2) / rotary_dim))
167
t = torch.arange(max_position).float()
168
freqs = torch.einsum('i,j -> ij', t, inv_freq.float())
169
cos = freqs.cos()
@@ -174,7 +174,7 @@ class LlamaCacheFlowAttention(GPTCacheFlowAttention):
174
# initializing the model. Make it more robust.
175
torch_dtype = torch.get_default_dtype()
176
cache = cache.to(torch_dtype)
177
- # Embedding size: [max_position, head_size]
177
+ # Embedding size: [max_position, rotary_dim]
178
self.register_buffer('cos_sin_cache', cache, persistent=False)
179
180
def forward(
@@ -190,10 +190,12 @@ class LlamaCacheFlowAttention(GPTCacheFlowAttention):
190
) -> torch.Tensor: # [num_tokens, num_heads * head_size]
191
# Apply rotary embedding to the query and key before passing them
192
# to the attention op.
193
+ head_size = value_cache.shape[2]
194
pos_encoding_ops.rotary_embedding_neox(
195
positions,
196
query,
197
key,
198
+ head_size,
199
self.cos_sin_cache,
200
)
201
return super().forward(
@@ -205,3 +207,7 @@ class LlamaCacheFlowAttention(GPTCacheFlowAttention):
207
input_metadata,
208
cache_event,
209
)
210
+
211
+
212
+class LlamaCacheFlowAttention(GPTNeoXCacheFlowAttention):
213
+ """LLaMA uses the GPT-NeoX style rotary embedding."""
cacheflow/models/gpt_neox.py
new
+278
@@ -0,0 +1,278 @@
1
+"""1D GPT-NeoX model compatible with HuggingFace weights."""
2
+import os
3
+import glob
4
+import filelock
5
+from tqdm import tqdm
6
+from typing import Dict, List, Optional, Tuple
7
+
8
+import numpy as np
9
+import torch
10
+from torch import nn
11
+from huggingface_hub import snapshot_download
12
+
13
+from cacheflow.models import InputMetadata
14
+from cacheflow.models.attention import GPTNeoXCacheFlowAttention
15
+from cacheflow.models.sample import Sampler
16
+from cacheflow.parallel_utils.parallel_state import (
17
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
18
+from cacheflow.parallel_utils.tensor_parallel import (VocabParallelEmbedding,
19
+ ColumnParallelLinear,
20
+ RowParallelLinear)
21
+from cacheflow.sequence import SequenceOutputs
22
+
23
+KVCache = Tuple[torch.Tensor, torch.Tensor]
24
+
25
+
26
+class GPTNeoXAttention(nn.Module):
27
+
28
+ def __init__(self, config):
29
+ super().__init__()
30
+ self.total_num_heads = config.num_attention_heads
31
+ self.hidden_size = config.hidden_size
32
+ self.head_size = self.hidden_size // self.total_num_heads
33
+
34
+ tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
35
+ assert self.total_num_heads % tensor_model_parallel_world_size == 0
36
+ self.num_heads = self.total_num_heads // tensor_model_parallel_world_size
37
+
38
+ self.query_key_value = ColumnParallelLinear(config.hidden_size,
39
+ 3 * config.hidden_size,
40
+ gather_output=False,
41
+ perform_initialization=False)
42
+ self.dense = RowParallelLinear(config.hidden_size, config.hidden_size,
43
+ input_is_parallel=True,
44
+ perform_initialization=False)
45
+
46
+ scaling = self.head_size ** -0.5
47
+ rotary_dim = int(self.head_size * config.rotary_pct)
48
+ assert rotary_dim % 2 == 0
49
+ self.attn = GPTNeoXCacheFlowAttention(scaling, rotary_dim)
50
+
51
+ def forward(
52
+ self,
53
+ position_ids: torch.LongTensor,
54
+ hidden_states: torch.Tensor,
55
+ kv_cache: KVCache,
56
+ input_metadata: InputMetadata,
57
+ cache_event: Optional[torch.cuda.Event],
58
+ ) -> torch.Tensor:
59
+ qkv, _ = self.query_key_value(hidden_states)
60
+
61
+ q, k, v = qkv.chunk(chunks=3, dim=-1)
62
+ k_cache, v_cache = kv_cache
63
+ attn_output = self.attn(
64
+ position_ids, q, k, v, k_cache, v_cache, input_metadata, cache_event)
65
+ output, _ = self.dense(attn_output)
66
+ return output
67
+
68
+
69
+class GPTNeoXMLP(nn.Module):
70
+ def __init__(self, config):
71
+ super().__init__()
72
+ self.dense_h_to_4h = ColumnParallelLinear(config.hidden_size,
73
+ config.intermediate_size,
74
+ gather_output=False,
75
+ perform_initialization=False)
76
+ self.dense_4h_to_h = RowParallelLinear(config.intermediate_size, config.hidden_size,
77
+ input_is_parallel=True,
78
+ perform_initialization=False)
79
+ if config.hidden_act != 'gelu':
80
+ raise ValueError(f'Unsupported activation: {config.hidden_act}. '
81
+ 'Only gelu is supported for now.')
82
+ self.act = torch.nn.GELU()
83
+
84
+ def forward(self, hidden_states):
85
+ hidden_states, _ = self.dense_h_to_4h(hidden_states)
86
+ hidden_states = self.act(hidden_states)
87
+ hidden_states, _ = self.dense_4h_to_h(hidden_states)
88
+ return hidden_states
89
+
90
+
91
+class GPTNeoXLayer(nn.Module):
92
+
93
+ def __init__(self, config):
94
+ super().__init__()
95
+ self.use_parallel_residual = config.use_parallel_residual
96
+ self.input_layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
97
+ self.post_attention_layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
98
+ self.attention = GPTNeoXAttention(config)
99
+ self.mlp = GPTNeoXMLP(config)
100
+
101
+ def forward(
102
+ self,
103
+ position_ids: torch.LongTensor,
104
+ hidden_states: torch.Tensor,
105
+ kv_cache: KVCache,
106
+ input_metadata: InputMetadata,
107
+ cache_event: Optional[torch.cuda.Event],
108
+ ) -> torch.Tensor:
109
+ attn_input = self.input_layernorm(hidden_states)
110
+ attn_output = self.attention(
111
+ position_ids=position_ids,
112
+ hidden_states=attn_input,
113
+ kv_cache=kv_cache,
114
+ input_metadata=input_metadata,
115
+ cache_event=cache_event,
116
+ )
117
+
118
+ if self.use_parallel_residual:
119
+ # pseudocode:
120
+ # x = x + attn(ln1(x)) + mlp(ln2(x))
121
+ mlp_input = self.post_attention_layernorm(hidden_states)
122
+ mlp_output = self.mlp(mlp_input)
123
+ hidden_states = mlp_output + attn_output + hidden_states
124
+ else:
125
+ # pseudocode:
126
+ # x = x + attn(ln1(x))
127
+ # x = x + mlp(ln2(x))
128
+ attn_output = attn_output + hidden_states
129
+ mlp_input = self.post_attention_layernorm(attn_output)
130
+ mlp_output = self.mlp(mlp_input)
131
+ hidden_states = mlp_output + attn_output
132
+ return hidden_states
133
+
134
+
135
+class GPTNeoXModel(nn.Module):
136
+ def __init__(self, config):
137
+ super().__init__()
138
+ self.config = config
139
+
140
+ self.embed_in = VocabParallelEmbedding(config.vocab_size, config.hidden_size,
141
+ perform_initialization=False)
142
+ self.layers = nn.ModuleList([GPTNeoXLayer(config) for _ in range(config.num_hidden_layers)])
143
+ self.final_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
144
+
145
+ def forward(
146
+ self,
147
+ input_ids: torch.LongTensor,
148
+ position_ids: torch.LongTensor,
149
+ kv_caches: List[KVCache],
150
+ input_metadata: InputMetadata,
151
+ cache_events: Optional[List[torch.cuda.Event]],
152
+ ) -> torch.Tensor:
153
+ hidden_states = self.embed_in(input_ids)
154
+ for i in range(len(self.layers)):
155
+ if cache_events is None:
156
+ cache_event = None
157
+ else:
158
+ cache_event = cache_events[i]
159
+ layer = self.layers[i]
160
+ hidden_states = layer(
161
+ position_ids,
162
+ hidden_states,
163
+ kv_caches[i],
164
+ input_metadata,
165
+ cache_event,
166
+ )
167
+ hidden_states = self.final_layer_norm(hidden_states)
168
+ return hidden_states
169
+
170
+
171
+class GPTNeoXForCausalLM(nn.Module):
172
+
173
+ def __init__(self, config):
174
+ super().__init__()
175
+ self.config = config
176
+ self.gpt_neox = GPTNeoXModel(config)
177
+ self.embed_out = ColumnParallelLinear(config.hidden_size, config.vocab_size,
178
+ bias=False, gather_output=False,
179
+ perform_initialization=False)
180
+ self.sampler = Sampler()
181
+
182
+ def forward(
183
+ self,
184
+ input_ids: torch.LongTensor,
185
+ positions: torch.LongTensor,
186
+ kv_caches: List[KVCache],
187
+ input_metadata: InputMetadata,
188
+ cache_events: Optional[List[torch.cuda.Event]],
189
+ ) -> Dict[int, SequenceOutputs]:
190
+ hidden_states = self.gpt_neox(
191
+ input_ids, positions, kv_caches, input_metadata, cache_events)
192
+ next_tokens = self.sampler(
193
+ self.embed_out.weight, hidden_states, input_metadata)
194
+ return next_tokens
195
+
196
+ _column_parallel_weights = ["embed_in.weight", "embed_out.weight", "dense_h_to_4h.weight", "dense_h_to_4h.bias"]
197
+ _row_parallel_weights = ["dense.weight", "dense_4h_to_h.weight"]
198
+
199
+ def load_weights(self, weights_path: str):
200
+ tensor_model_parallel_rank = get_tensor_model_parallel_rank()
201
+ state_dict = self.state_dict()
202
+ for name, param in state_dict.items():
203
+ if "query_key_value" in name:
204
+ # NOTE(woosuk): GPT-NeoX's fused QKV has the shape of
205
+ # [num_heads * 3 * head_size, num_heads * head_size], while the
206
+ # required shape is [3 * num_heads * head_size, num_heads * head_size].
207
+ # Thus, we need weight conversion.
208
+ loaded_weight = torch.from_numpy(
209
+ np.load(os.path.join(weights_path, name)))
210
+ shard_size = param.shape[0]
211
+ loaded_weight = loaded_weight[shard_size * tensor_model_parallel_rank
212
+ :shard_size * (tensor_model_parallel_rank + 1)]
213
+
214
+ num_heads = self.config.num_attention_heads
215
+ hidden_size = self.config.hidden_size
216
+ head_size = hidden_size // num_heads
217
+ if 'query_key_value.weight' in name:
218
+ loaded_weight = loaded_weight.view(-1, 3, head_size, hidden_size)
219
+ loaded_weight = loaded_weight.transpose(0, 1)
220
+ loaded_weight = loaded_weight.reshape(-1, hidden_size).contiguous()
221
+ elif 'query_key_value.bias' in name:
222
+ loaded_weight = loaded_weight.view(-1, 3, head_size)
223
+ loaded_weight = loaded_weight.transpose(0, 1)
224
+ loaded_weight = loaded_weight.reshape(-1).contiguous()
225
+ else:
226
+ assert False
227
+ else:
228
+ loaded_weight = torch.from_numpy(
229
+ np.load(os.path.join(weights_path, name)))
230
+ for p in self._column_parallel_weights:
231
+ if p in name:
232
+ shard_size = param.shape[0]
233
+ loaded_weight = loaded_weight[
234
+ shard_size * tensor_model_parallel_rank
235
+ :shard_size * (tensor_model_parallel_rank + 1)]
236
+ break
237
+ for p in self._row_parallel_weights:
238
+ if p in name:
239
+ shard_size = param.shape[1]
240
+ loaded_weight = loaded_weight[
241
+ :,
242
+ shard_size * tensor_model_parallel_rank
243
+ :shard_size * (tensor_model_parallel_rank + 1)]
244
+ break
245
+
246
+ assert param.shape == loaded_weight.shape
247
+ param.data.copy_(loaded_weight)
248
+
249
+ @staticmethod
250
+ def get_weights(model_name: str, path: str):
251
+ path = os.path.join(path, f"{model_name}-np")
252
+ path = os.path.abspath(os.path.expanduser(path))
253
+ os.makedirs(path, exist_ok=True)
254
+ lock_path = os.path.join(path, "file_lock")
255
+ lock = filelock.FileLock(lock_path)
256
+
257
+ with lock:
258
+ test_weight_path = os.path.join(
259
+ path, "gpt_neox.embed_in.weight")
260
+ if os.path.exists(test_weight_path):
261
+ return path
262
+
263
+ folder = snapshot_download(model_name, allow_patterns="*.bin",
264
+ cache_dir=os.path.join(path, "cache"))
265
+ bin_files = glob.glob(os.path.join(folder, "*.bin"))
266
+
267
+ for bin_file in tqdm(bin_files, desc="Convert format"):
268
+ state = torch.load(bin_file, map_location="cpu")
269
+ for name, param in tqdm(state.items(), leave=False):
270
+ param_path = os.path.join(path, name)
271
+ with open(param_path, "wb") as f:
272
+ np.save(f, param.cpu().detach().numpy())
273
+
274
+ return path
275
+
276
+ def initialize_dummy_weights(self) -> None:
277
+ for param in self.state_dict().values():
278
+ param.data.uniform_(-1e-3, 1e-3)
cacheflow/models/llama.py
+1
-1
@@ -289,4 +289,4 @@ class LlamaForCausalLM(nn.Module):
289
290
def initialize_dummy_weights(self) -> None:
291
for param in self.state_dict().values():
292
- param.data.uniform_(-0.1, 0.1)
292
+ param.data.uniform_(-1e-3, 1e-3)
cacheflow/models/memory_analyzer.py
+107
-50
@@ -40,6 +40,37 @@ class CacheFlowMemoryAnalyzer:
40
max_num_blocks = swap_space // self.get_cache_block_size()
41
return max_num_blocks
42
43
+ def get_param_size(self) -> int:
44
+ raise NotImplementedError()
45
+
46
+ def get_max_act_size(self, max_num_batched_tokens: int) -> int:
47
+ raise NotImplementedError()
48
+
49
+ def get_cache_block_size(self) -> int:
50
+ key_cache_block = self.block_size * self.hidden_size // self.tensor_parallel_size
51
+ value_cache_block = key_cache_block
52
+ total = self.num_layers * (key_cache_block + value_cache_block)
53
+ dtype_size = get_dtype_size(self.dtype)
54
+ return dtype_size * total
55
+
56
+ def get_max_num_gpu_blocks(
57
+ self,
58
+ max_num_batched_tokens: int,
59
+ memory_utilization: float = 0.95,
60
+ ) -> int:
61
+ # NOTE(woosuk): This assumes that the machine has homogeneous GPUs.
62
+ usable_memory = int(memory_utilization * self.gpu_memory)
63
+
64
+ param_size = self.get_param_size()
65
+ act_size = self.get_max_act_size(max_num_batched_tokens)
66
+ workspace_size = self.get_workspace_size()
67
+
68
+ max_cache_size = usable_memory - (param_size + act_size + workspace_size)
69
+ if max_cache_size <= 0:
70
+ raise RuntimeError('Not enough GPU memory.')
71
+ max_num_blocks = max_cache_size // self.get_cache_block_size()
72
+ return max_num_blocks
73
+
74
75
class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
76
@@ -69,7 +100,7 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
100
self.vocab_size = config.vocab_size
101
self.max_position = config.max_position_embeddings
102
72
- def _get_param_size(self) -> int:
103
+ def get_param_size(self) -> int:
104
word_embedding = self.vocab_size * self.embedding_size // self.tensor_parallel_size
105
if self.embedding_size != self.hidden_size:
106
# Project in/out.
@@ -93,7 +124,7 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
124
dtype_size = get_dtype_size(self.dtype)
125
return dtype_size * total
126
96
- def _get_max_act_size(
127
+ def get_max_act_size(
128
self,
129
max_num_batched_tokens: int,
130
) -> int:
@@ -114,31 +145,6 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
145
dtype_size = get_dtype_size(self.dtype)
146
return dtype_size * max_act
147
117
- def get_cache_block_size(self) -> int:
118
- key_cache_block = self.block_size * self.hidden_size // self.tensor_parallel_size
119
- value_cache_block = key_cache_block
120
- total = self.num_layers * (key_cache_block + value_cache_block)
121
- dtype_size = get_dtype_size(self.dtype)
122
- return dtype_size * total
123
-
124
- def get_max_num_gpu_blocks(
125
- self,
126
- max_num_batched_tokens: int,
127
- memory_utilization: float = 0.95,
128
- ) -> int:
129
- # NOTE(woosuk): This assumes that the machine has homogeneous GPUs.
130
- usable_memory = int(memory_utilization * self.gpu_memory)
131
-
132
- param_size = self._get_param_size()
133
- act_size = self._get_max_act_size(max_num_batched_tokens)
134
- workspace_size = self.get_workspace_size()
135
-
136
- max_cache_size = usable_memory - (param_size + act_size + workspace_size)
137
- if max_cache_size <= 0:
138
- raise RuntimeError('Not enough GPU memory.')
139
- max_num_blocks = max_cache_size // self.get_cache_block_size()
140
- return max_num_blocks
141
-
148
149
class LlamaMemoryAnalyzer(CacheFlowMemoryAnalyzer):
150
@@ -167,9 +173,10 @@ class LlamaMemoryAnalyzer(CacheFlowMemoryAnalyzer):
173
self.vocab_size = config.vocab_size
174
self.max_position = 8192
175
170
- def _get_param_size(self) -> int:
176
+ def get_param_size(self) -> int:
177
+ # NOTE: LLaMA does not tie the two embeddings.
178
word_embedding = self.vocab_size * self.hidden_size // self.tensor_parallel_size
172
- position_embedding = self.max_position * self.hidden_size
179
+ lm_head = self.vocab_size * self.hidden_size // self.tensor_parallel_size
180
181
# NOTE: LLaMA does not have bias terms.
182
ln1 = self.hidden_size
@@ -188,11 +195,11 @@ class LlamaMemoryAnalyzer(CacheFlowMemoryAnalyzer):
195
up = self.hidden_size * self.ffn_size // self.tensor_parallel_size
196
ffn = ln2 + gate + down + up
197
191
- total = (word_embedding + position_embedding + self.num_layers * (mha + ffn))
198
+ total = word_embedding + self.num_layers * (mha + ffn) + lm_head
199
dtype_size = get_dtype_size(self.dtype)
200
return dtype_size * total
201
195
- def _get_max_act_size(
202
+ def get_max_act_size(
203
self,
204
max_num_batched_tokens: int,
205
) -> int:
@@ -213,28 +220,78 @@ class LlamaMemoryAnalyzer(CacheFlowMemoryAnalyzer):
220
dtype_size = get_dtype_size(self.dtype)
221
return dtype_size * max_act
222
216
- def get_cache_block_size(self) -> int:
217
- key_cache_block = self.block_size * self.hidden_size // self.tensor_parallel_size
218
- value_cache_block = key_cache_block
219
- total = self.num_layers * (key_cache_block + value_cache_block)
223
+
224
+class GPTNeoXMemoryAnalyzer(CacheFlowMemoryAnalyzer):
225
+
226
+ def __init__(
227
+ self,
228
+ model_name: str,
229
+ block_size: int,
230
+ dtype: torch.dtype,
231
+ gpu_memory: int,
232
+ cpu_memory: int,
233
+ tensor_parallel_size: int,
234
+ ) -> None:
235
+ self.model_name = model_name
236
+ self.block_size = block_size
237
+ self.dtype = dtype
238
+ self.gpu_memory = gpu_memory
239
+ self.cpu_memory = cpu_memory
240
+ self.tensor_parallel_size = tensor_parallel_size
241
+
242
+ config = AutoConfig.from_pretrained(model_name)
243
+ self.num_layers = config.num_hidden_layers
244
+ self.hidden_size = config.hidden_size
245
+ self.num_heads = config.num_attention_heads
246
+ self.head_size = config.hidden_size // self.num_heads
247
+ self.ffn_size = config.intermediate_size
248
+ self.vocab_size = config.vocab_size
249
+ self.max_position = 8192
250
+ self.tie_word_embeddings = config.tie_word_embeddings
251
+
252
+ def get_param_size(self) -> int:
253
+ word_embedding = self.vocab_size * self.hidden_size // self.tensor_parallel_size
254
+ if self.tie_word_embeddings:
255
+ lm_head = 0
256
+ else:
257
+ lm_head = self.vocab_size * self.hidden_size // self.tensor_parallel_size
258
+
259
+ ln1 = 2 * self.hidden_size
260
+ q = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
261
+ k = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
262
+ v = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
263
+ out = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
264
+ # Rotary embedding.
265
+ # TODO(woosuk): Share the rotary embedding between layers.
266
+ rot = self.max_position * self.head_size
267
+ mha = ln1 + q + k + v + out + rot
268
+
269
+ ln2 = 2 * self.hidden_size
270
+ ffn1 = self.hidden_size * self.ffn_size // self.tensor_parallel_size + self.ffn_size
271
+ ffn2 = self.ffn_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
272
+ ffn = ln2 + ffn1 + ffn2
273
+
274
+ total = word_embedding + self.num_layers * (mha + ffn) + lm_head
275
dtype_size = get_dtype_size(self.dtype)
276
return dtype_size * total
277
223
- def get_max_num_gpu_blocks(
278
+ def get_max_act_size(
279
self,
280
max_num_batched_tokens: int,
226
- memory_utilization: float = 0.95,
281
) -> int:
228
- # NOTE(woosuk): This assumes that the machine has homogeneous GPUs.
229
- gpu_memory = self.gpu_memory
230
- usable_memory = int(memory_utilization * gpu_memory)
231
-
232
- param_size = self._get_param_size()
233
- act_size = self._get_max_act_size(max_num_batched_tokens)
234
- workspace_size = self.get_workspace_size()
235
-
236
- max_cache_size = usable_memory - (param_size + act_size + workspace_size)
237
- if max_cache_size <= 0:
238
- raise RuntimeError('Not enough GPU memory.')
239
- max_num_blocks = max_cache_size // self.get_cache_block_size()
240
- return max_num_blocks
282
+ # NOTE: We approxmiately calculate the maximum activation size by
283
+ # estimating
284
+ # 1) the maximum activation tensor size during inference
285
+ # 2) the residual tensor size during inference
286
+ # Here, we assume that FlashAttention is used and
287
+ # thus the attention maps are never materialized in GPU DRAM.
288
+ residual = max_num_batched_tokens * self.hidden_size
289
+ qkv = 3 * (max_num_batched_tokens * self.hidden_size) // self.tensor_parallel_size
290
+ ffn = 2 * (max_num_batched_tokens * self.ffn_size) // self.tensor_parallel_size
291
+ # Double the activation size for input and output.
292
+ max_act = 2 * (max(qkv, ffn) + residual)
293
+ # Size of output logits.
294
+ output_logits = 2 * (max_num_batched_tokens * self.vocab_size)
295
+ max_act = max(max_act, output_logits)
296
+ dtype_size = get_dtype_size(self.dtype)
297
+ return dtype_size * max_act
cacheflow/models/model_utils.py
+6
-1
@@ -1,13 +1,14 @@
1
from typing import Union
2
3
-import numpy as np
3
import torch
4
import torch.nn as nn
5
from transformers import AutoConfig
6
7
from cacheflow.models.memory_analyzer import CacheFlowMemoryAnalyzer
8
+from cacheflow.models.memory_analyzer import GPTNeoXMemoryAnalyzer
9
from cacheflow.models.memory_analyzer import LlamaMemoryAnalyzer
10
from cacheflow.models.memory_analyzer import OPTMemoryAnalyzer
11
+from cacheflow.models.gpt_neox import GPTNeoXForCausalLM
12
from cacheflow.models.llama import LlamaForCausalLM
13
from cacheflow.models.opt import OPTForCausalLM
14
from cacheflow.models.utils import get_torch_dtype
@@ -16,11 +17,15 @@ from cacheflow.models.utils import get_torch_dtype
17
_MODELS = {
18
'llama': LlamaForCausalLM,
19
'opt': OPTForCausalLM,
20
+ 'stablelm': GPTNeoXForCausalLM,
21
+ 'pythia': GPTNeoXForCausalLM,
22
}
23
24
_MEMORY_ANALYZERS = {
25
'llama': LlamaMemoryAnalyzer,
26
'opt': OPTMemoryAnalyzer,
27
+ 'stablelm': GPTNeoXMemoryAnalyzer,
28
+ 'pythia': GPTNeoXMemoryAnalyzer,
29
}
30
31
cacheflow/models/opt.py
+1
-1
@@ -327,4 +327,4 @@ class OPTForCausalLM(nn.Module):
327
328
def initialize_dummy_weights(self) -> None:
329
for param in self.state_dict().values():
330
- param.data.uniform_(-0.1, 0.1)
330
+ param.data.uniform_(-1e-3, 1e-3)
csrc/pos_encoding.cpp
+1
@@ -4,6 +4,7 @@ void rotary_embedding_neox(
4
torch::Tensor& positions,
5
torch::Tensor& query,
6
torch::Tensor& key,
7
+ int head_size,
8
torch::Tensor& cos_sin_cache);
9
10
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
csrc/pos_encoding_kernels.cu
+9
-6
@@ -8,16 +8,17 @@ __global__ void rotary_embedding_neox_kernel(
8
const int64_t* __restrict__ positions, // [num_tokens]
9
scalar_t* __restrict__ query, // [num_tokens, num_heads, head_size]
10
scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
11
- const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, head_size // 2]
11
+ const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
12
+ const int rot_dim,
13
const int stride,
14
const int num_heads,
15
const int head_size) {
16
// Each thread block is responsible for one token.
17
const int token_idx = blockIdx.x;
18
int64_t pos = positions[token_idx];
18
- const scalar_t* cache_ptr = cos_sin_cache + pos * head_size;
19
+ const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim;
20
20
- const int embed_dim = head_size / 2;
21
+ const int embed_dim = rot_dim / 2;
22
const int n = num_heads * embed_dim;
23
for (int i = threadIdx.x; i < n; i += blockDim.x) {
24
const int head_idx = i / embed_dim;
@@ -51,16 +52,17 @@ void rotary_embedding_neox(
52
torch::Tensor& positions, // [num_tokens]
53
torch::Tensor& query, // [num_tokens, num_heads * head_size]
54
torch::Tensor& key, // [num_tokens, num_heads * head_size]
54
- torch::Tensor& cos_sin_cache) // [max_position, head_size]
55
+ int head_size,
56
+ torch::Tensor& cos_sin_cache) // [max_position, rot_dim]
57
{
58
int num_tokens = query.size(0);
57
- int head_size = cos_sin_cache.size(1);
59
+ int rot_dim = cos_sin_cache.size(1);
60
int num_heads = query.size(1) / head_size;
61
int stride = query.stride(0);
62
TORCH_CHECK(stride == key.stride(0));
63
64
dim3 grid(num_tokens);
63
- dim3 block(std::min(num_heads * head_size / 2, 512));
65
+ dim3 block(std::min(num_heads * rot_dim / 2, 512));
66
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
67
AT_DISPATCH_FLOATING_TYPES_AND_HALF(
68
query.scalar_type(),
@@ -71,6 +73,7 @@ void rotary_embedding_neox(
73
query.data_ptr<scalar_t>(),
74
key.data_ptr<scalar_t>(),
75
cos_sin_cache.data_ptr<scalar_t>(),
76
+ rot_dim,
77
stride,
78
num_heads,
79
head_size);
tests/kernels/pos_encoding.py
+22
-7
@@ -34,6 +34,7 @@ class RefRotaryEmbeddingNeox(nn.Module):
34
base: int = 10000,
35
) -> None:
36
super().__init__()
37
+ self.rotary_dim = dim
38
self.max_position_embeddings = max_position_embeddings
39
40
# Create cos and sin embeddings.
@@ -52,13 +53,24 @@ class RefRotaryEmbeddingNeox(nn.Module):
53
query: torch.Tensor, # [num_tokens, num_heads, head_size]
54
key: torch.Tensor, # [num_tokens, num_heads, head_size]
55
) -> Tuple[torch.Tensor, torch.Tensor]:
56
+
57
+ query_rot = query[..., : self.rotary_dim]
58
+ query_pass = query[..., self.rotary_dim :]
59
+ key_rot = key[..., : self.rotary_dim]
60
+ key_pass = key[..., self.rotary_dim :]
61
+
62
+
63
+ query_rot = query_rot.transpose(0, 1)
64
+ key_rot = key_rot.transpose(0, 1)
65
cos = F.embedding(positions, self.cos_cached)
66
sin = F.embedding(positions, self.sin_cached)
57
- query = query.transpose(0, 1)
58
- key = key.transpose(0, 1)
59
- query, key = apply_rotary_pos_emb(query, key, cos, sin)
60
- query = query.transpose(0, 1).contiguous()
61
- key = key.transpose(0, 1).contiguous()
67
+ query_rot, key_rot = apply_rotary_pos_emb(query_rot, key_rot, cos, sin)
68
+ query_rot = query_rot.transpose(0, 1).contiguous()
69
+ key_rot = key_rot.transpose(0, 1).contiguous()
70
+
71
+ query = torch.cat((query_rot, query_pass), dim=-1)
72
+ key = torch.cat((key_rot, key_pass), dim=-1)
73
+
74
# Output query/key shape: [num_tokens, num_tokens, head_size]
75
return query, key
76
@@ -69,6 +81,7 @@ def test_rotary_embedding_neox(
81
num_heads: int,
82
head_size: int,
83
max_position: int,
84
+ rotary_dim: int,
85
dtype: torch.dtype,
86
base: int = 10000,
87
) -> None:
@@ -77,7 +90,7 @@ def test_rotary_embedding_neox(
90
key = torch.randn(num_tokens, num_heads * head_size, dtype=dtype, device='cuda')
91
92
# Create the rotary embedding.
80
- inv_freq = 1.0 / (base ** (torch.arange(0, head_size, 2) / head_size))
93
+ inv_freq = 1.0 / (base ** (torch.arange(0, rotary_dim, 2) / rotary_dim))
94
t = torch.arange(max_position).float()
95
freqs = torch.einsum('i,j -> ij', t, inv_freq.float())
96
cos = freqs.cos()
@@ -92,12 +105,13 @@ def test_rotary_embedding_neox(
105
positions,
106
out_query,
107
out_key,
108
+ head_size,
109
cos_sin_cache,
110
)
111
112
# Run the reference implementation.
113
ref_rotary_embedding = RefRotaryEmbeddingNeox(
100
- dim=head_size,
114
+ dim=rotary_dim,
115
max_position_embeddings=max_position,
116
base=base,
117
).to(dtype=dtype, device='cuda')
@@ -123,5 +137,6 @@ if __name__ == '__main__':
137
num_heads=5,
138
head_size=head_size,
139
max_position=8192,
140
+ rotary_dim=int(head_size * 0.25),
141
dtype=dtype,
142
)