@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a96d63c21d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add support for GPT-NeoX (Pythia) (#50)

Woosuk Kwon committed Apr 28, 2023 at 00:32 UTC a96d63c21d18ad6610adfcabd3aae02c6357334e
9 files changed +436 -71
cacheflow/models/attention.py
+11 -5
@@ -150,20 +150,20 @@ class OPTCacheFlowAttention(GPTCacheFlowAttention):
150 super().__init__(scale)
151
152
153 -class LlamaCacheFlowAttention(GPTCacheFlowAttention):
154 - """Llama uses GPT-NeoX style rotary embedding."""
153 +class GPTNeoXCacheFlowAttention(GPTCacheFlowAttention):
154 + """Attention with GPT-NeoX style rotary embedding."""
155
156 def __init__(
157 self,
158 scale: float,
159 - head_size: int,
159 + rotary_dim: int,
160 max_position: int = 8192,
161 base: int = 10000,
162 ) -> None:
163 super().__init__(scale)
164
165 # Create the cos and sin cache.
166 - inv_freq = 1.0 / (base ** (torch.arange(0, head_size, 2) / head_size))
166 + inv_freq = 1.0 / (base ** (torch.arange(0, rotary_dim, 2) / rotary_dim))
167 t = torch.arange(max_position).float()
168 freqs = torch.einsum('i,j -> ij', t, inv_freq.float())
169 cos = freqs.cos()
@@ -174,7 +174,7 @@ class LlamaCacheFlowAttention(GPTCacheFlowAttention):
174 # initializing the model. Make it more robust.
175 torch_dtype = torch.get_default_dtype()
176 cache = cache.to(torch_dtype)
177 - # Embedding size: [max_position, head_size]
177 + # Embedding size: [max_position, rotary_dim]
178 self.register_buffer('cos_sin_cache', cache, persistent=False)
179
180 def forward(
@@ -190,10 +190,12 @@ class LlamaCacheFlowAttention(GPTCacheFlowAttention):
190 ) -> torch.Tensor: # [num_tokens, num_heads * head_size]
191 # Apply rotary embedding to the query and key before passing them
192 # to the attention op.
193 + head_size = value_cache.shape[2]
194 pos_encoding_ops.rotary_embedding_neox(
195 positions,
196 query,
197 key,
198 + head_size,
199 self.cos_sin_cache,
200 )
201 return super().forward(
@@ -205,3 +207,7 @@ class LlamaCacheFlowAttention(GPTCacheFlowAttention):
207 input_metadata,
208 cache_event,
209 )
210 +
211 +
212 +class LlamaCacheFlowAttention(GPTNeoXCacheFlowAttention):
213 + """LLaMA uses the GPT-NeoX style rotary embedding."""
cacheflow/models/gpt_neox.py new
+278
@@ -0,0 +1,278 @@
1 +"""1D GPT-NeoX model compatible with HuggingFace weights."""
2 +import os
3 +import glob
4 +import filelock
5 +from tqdm import tqdm
6 +from typing import Dict, List, Optional, Tuple
7 +
8 +import numpy as np
9 +import torch
10 +from torch import nn
11 +from huggingface_hub import snapshot_download
12 +
13 +from cacheflow.models import InputMetadata
14 +from cacheflow.models.attention import GPTNeoXCacheFlowAttention
15 +from cacheflow.models.sample import Sampler
16 +from cacheflow.parallel_utils.parallel_state import (
17 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
18 +from cacheflow.parallel_utils.tensor_parallel import (VocabParallelEmbedding,
19 + ColumnParallelLinear,
20 + RowParallelLinear)
21 +from cacheflow.sequence import SequenceOutputs
22 +
23 +KVCache = Tuple[torch.Tensor, torch.Tensor]
24 +
25 +
26 +class GPTNeoXAttention(nn.Module):
27 +
28 + def __init__(self, config):
29 + super().__init__()
30 + self.total_num_heads = config.num_attention_heads
31 + self.hidden_size = config.hidden_size
32 + self.head_size = self.hidden_size // self.total_num_heads
33 +
34 + tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
35 + assert self.total_num_heads % tensor_model_parallel_world_size == 0
36 + self.num_heads = self.total_num_heads // tensor_model_parallel_world_size
37 +
38 + self.query_key_value = ColumnParallelLinear(config.hidden_size,
39 + 3 * config.hidden_size,
40 + gather_output=False,
41 + perform_initialization=False)
42 + self.dense = RowParallelLinear(config.hidden_size, config.hidden_size,
43 + input_is_parallel=True,
44 + perform_initialization=False)
45 +
46 + scaling = self.head_size ** -0.5
47 + rotary_dim = int(self.head_size * config.rotary_pct)
48 + assert rotary_dim % 2 == 0
49 + self.attn = GPTNeoXCacheFlowAttention(scaling, rotary_dim)
50 +
51 + def forward(
52 + self,
53 + position_ids: torch.LongTensor,
54 + hidden_states: torch.Tensor,
55 + kv_cache: KVCache,
56 + input_metadata: InputMetadata,
57 + cache_event: Optional[torch.cuda.Event],
58 + ) -> torch.Tensor:
59 + qkv, _ = self.query_key_value(hidden_states)
60 +
61 + q, k, v = qkv.chunk(chunks=3, dim=-1)
62 + k_cache, v_cache = kv_cache
63 + attn_output = self.attn(
64 + position_ids, q, k, v, k_cache, v_cache, input_metadata, cache_event)
65 + output, _ = self.dense(attn_output)
66 + return output
67 +
68 +
69 +class GPTNeoXMLP(nn.Module):
70 + def __init__(self, config):
71 + super().__init__()
72 + self.dense_h_to_4h = ColumnParallelLinear(config.hidden_size,
73 + config.intermediate_size,
74 + gather_output=False,
75 + perform_initialization=False)
76 + self.dense_4h_to_h = RowParallelLinear(config.intermediate_size, config.hidden_size,
77 + input_is_parallel=True,
78 + perform_initialization=False)
79 + if config.hidden_act != 'gelu':
80 + raise ValueError(f'Unsupported activation: {config.hidden_act}. '
81 + 'Only gelu is supported for now.')
82 + self.act = torch.nn.GELU()
83 +
84 + def forward(self, hidden_states):
85 + hidden_states, _ = self.dense_h_to_4h(hidden_states)
86 + hidden_states = self.act(hidden_states)
87 + hidden_states, _ = self.dense_4h_to_h(hidden_states)
88 + return hidden_states
89 +
90 +
91 +class GPTNeoXLayer(nn.Module):
92 +
93 + def __init__(self, config):
94 + super().__init__()
95 + self.use_parallel_residual = config.use_parallel_residual
96 + self.input_layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
97 + self.post_attention_layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
98 + self.attention = GPTNeoXAttention(config)
99 + self.mlp = GPTNeoXMLP(config)
100 +
101 + def forward(
102 + self,
103 + position_ids: torch.LongTensor,
104 + hidden_states: torch.Tensor,
105 + kv_cache: KVCache,
106 + input_metadata: InputMetadata,
107 + cache_event: Optional[torch.cuda.Event],
108 + ) -> torch.Tensor:
109 + attn_input = self.input_layernorm(hidden_states)
110 + attn_output = self.attention(
111 + position_ids=position_ids,
112 + hidden_states=attn_input,
113 + kv_cache=kv_cache,
114 + input_metadata=input_metadata,
115 + cache_event=cache_event,
116 + )
117 +
118 + if self.use_parallel_residual:
119 + # pseudocode:
120 + # x = x + attn(ln1(x)) + mlp(ln2(x))
121 + mlp_input = self.post_attention_layernorm(hidden_states)
122 + mlp_output = self.mlp(mlp_input)
123 + hidden_states = mlp_output + attn_output + hidden_states
124 + else:
125 + # pseudocode:
126 + # x = x + attn(ln1(x))
127 + # x = x + mlp(ln2(x))
128 + attn_output = attn_output + hidden_states
129 + mlp_input = self.post_attention_layernorm(attn_output)
130 + mlp_output = self.mlp(mlp_input)
131 + hidden_states = mlp_output + attn_output
132 + return hidden_states
133 +
134 +
135 +class GPTNeoXModel(nn.Module):
136 + def __init__(self, config):
137 + super().__init__()
138 + self.config = config
139 +
140 + self.embed_in = VocabParallelEmbedding(config.vocab_size, config.hidden_size,
141 + perform_initialization=False)
142 + self.layers = nn.ModuleList([GPTNeoXLayer(config) for _ in range(config.num_hidden_layers)])
143 + self.final_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
144 +
145 + def forward(
146 + self,
147 + input_ids: torch.LongTensor,
148 + position_ids: torch.LongTensor,
149 + kv_caches: List[KVCache],
150 + input_metadata: InputMetadata,
151 + cache_events: Optional[List[torch.cuda.Event]],
152 + ) -> torch.Tensor:
153 + hidden_states = self.embed_in(input_ids)
154 + for i in range(len(self.layers)):
155 + if cache_events is None:
156 + cache_event = None
157 + else:
158 + cache_event = cache_events[i]
159 + layer = self.layers[i]
160 + hidden_states = layer(
161 + position_ids,
162 + hidden_states,
163 + kv_caches[i],
164 + input_metadata,
165 + cache_event,
166 + )
167 + hidden_states = self.final_layer_norm(hidden_states)
168 + return hidden_states
169 +
170 +
171 +class GPTNeoXForCausalLM(nn.Module):
172 +
173 + def __init__(self, config):
174 + super().__init__()
175 + self.config = config
176 + self.gpt_neox = GPTNeoXModel(config)
177 + self.embed_out = ColumnParallelLinear(config.hidden_size, config.vocab_size,
178 + bias=False, gather_output=False,
179 + perform_initialization=False)
180 + self.sampler = Sampler()
181 +
182 + def forward(
183 + self,
184 + input_ids: torch.LongTensor,
185 + positions: torch.LongTensor,
186 + kv_caches: List[KVCache],
187 + input_metadata: InputMetadata,
188 + cache_events: Optional[List[torch.cuda.Event]],
189 + ) -> Dict[int, SequenceOutputs]:
190 + hidden_states = self.gpt_neox(
191 + input_ids, positions, kv_caches, input_metadata, cache_events)
192 + next_tokens = self.sampler(
193 + self.embed_out.weight, hidden_states, input_metadata)
194 + return next_tokens
195 +
196 + _column_parallel_weights = ["embed_in.weight", "embed_out.weight", "dense_h_to_4h.weight", "dense_h_to_4h.bias"]
197 + _row_parallel_weights = ["dense.weight", "dense_4h_to_h.weight"]
198 +
199 + def load_weights(self, weights_path: str):
200 + tensor_model_parallel_rank = get_tensor_model_parallel_rank()
201 + state_dict = self.state_dict()
202 + for name, param in state_dict.items():
203 + if "query_key_value" in name:
204 + # NOTE(woosuk): GPT-NeoX's fused QKV has the shape of
205 + # [num_heads * 3 * head_size, num_heads * head_size], while the
206 + # required shape is [3 * num_heads * head_size, num_heads * head_size].
207 + # Thus, we need weight conversion.
208 + loaded_weight = torch.from_numpy(
209 + np.load(os.path.join(weights_path, name)))
210 + shard_size = param.shape[0]
211 + loaded_weight = loaded_weight[shard_size * tensor_model_parallel_rank
212 + :shard_size * (tensor_model_parallel_rank + 1)]
213 +
214 + num_heads = self.config.num_attention_heads
215 + hidden_size = self.config.hidden_size
216 + head_size = hidden_size // num_heads
217 + if 'query_key_value.weight' in name:
218 + loaded_weight = loaded_weight.view(-1, 3, head_size, hidden_size)
219 + loaded_weight = loaded_weight.transpose(0, 1)
220 + loaded_weight = loaded_weight.reshape(-1, hidden_size).contiguous()
221 + elif 'query_key_value.bias' in name:
222 + loaded_weight = loaded_weight.view(-1, 3, head_size)
223 + loaded_weight = loaded_weight.transpose(0, 1)
224 + loaded_weight = loaded_weight.reshape(-1).contiguous()
225 + else:
226 + assert False
227 + else:
228 + loaded_weight = torch.from_numpy(
229 + np.load(os.path.join(weights_path, name)))
230 + for p in self._column_parallel_weights:
231 + if p in name:
232 + shard_size = param.shape[0]
233 + loaded_weight = loaded_weight[
234 + shard_size * tensor_model_parallel_rank
235 + :shard_size * (tensor_model_parallel_rank + 1)]
236 + break
237 + for p in self._row_parallel_weights:
238 + if p in name:
239 + shard_size = param.shape[1]
240 + loaded_weight = loaded_weight[
241 + :,
242 + shard_size * tensor_model_parallel_rank
243 + :shard_size * (tensor_model_parallel_rank + 1)]
244 + break
245 +
246 + assert param.shape == loaded_weight.shape
247 + param.data.copy_(loaded_weight)
248 +
249 + @staticmethod
250 + def get_weights(model_name: str, path: str):
251 + path = os.path.join(path, f"{model_name}-np")
252 + path = os.path.abspath(os.path.expanduser(path))
253 + os.makedirs(path, exist_ok=True)
254 + lock_path = os.path.join(path, "file_lock")
255 + lock = filelock.FileLock(lock_path)
256 +
257 + with lock:
258 + test_weight_path = os.path.join(
259 + path, "gpt_neox.embed_in.weight")
260 + if os.path.exists(test_weight_path):
261 + return path
262 +
263 + folder = snapshot_download(model_name, allow_patterns="*.bin",
264 + cache_dir=os.path.join(path, "cache"))
265 + bin_files = glob.glob(os.path.join(folder, "*.bin"))
266 +
267 + for bin_file in tqdm(bin_files, desc="Convert format"):
268 + state = torch.load(bin_file, map_location="cpu")
269 + for name, param in tqdm(state.items(), leave=False):
270 + param_path = os.path.join(path, name)
271 + with open(param_path, "wb") as f:
272 + np.save(f, param.cpu().detach().numpy())
273 +
274 + return path
275 +
276 + def initialize_dummy_weights(self) -> None:
277 + for param in self.state_dict().values():
278 + param.data.uniform_(-1e-3, 1e-3)
cacheflow/models/llama.py
+1 -1
@@ -289,4 +289,4 @@ class LlamaForCausalLM(nn.Module):
289
290 def initialize_dummy_weights(self) -> None:
291 for param in self.state_dict().values():
292 - param.data.uniform_(-0.1, 0.1)
292 + param.data.uniform_(-1e-3, 1e-3)
cacheflow/models/memory_analyzer.py
+107 -50
@@ -40,6 +40,37 @@ class CacheFlowMemoryAnalyzer:
40 max_num_blocks = swap_space // self.get_cache_block_size()
41 return max_num_blocks
42
43 + def get_param_size(self) -> int:
44 + raise NotImplementedError()
45 +
46 + def get_max_act_size(self, max_num_batched_tokens: int) -> int:
47 + raise NotImplementedError()
48 +
49 + def get_cache_block_size(self) -> int:
50 + key_cache_block = self.block_size * self.hidden_size // self.tensor_parallel_size
51 + value_cache_block = key_cache_block
52 + total = self.num_layers * (key_cache_block + value_cache_block)
53 + dtype_size = get_dtype_size(self.dtype)
54 + return dtype_size * total
55 +
56 + def get_max_num_gpu_blocks(
57 + self,
58 + max_num_batched_tokens: int,
59 + memory_utilization: float = 0.95,
60 + ) -> int:
61 + # NOTE(woosuk): This assumes that the machine has homogeneous GPUs.
62 + usable_memory = int(memory_utilization * self.gpu_memory)
63 +
64 + param_size = self.get_param_size()
65 + act_size = self.get_max_act_size(max_num_batched_tokens)
66 + workspace_size = self.get_workspace_size()
67 +
68 + max_cache_size = usable_memory - (param_size + act_size + workspace_size)
69 + if max_cache_size <= 0:
70 + raise RuntimeError('Not enough GPU memory.')
71 + max_num_blocks = max_cache_size // self.get_cache_block_size()
72 + return max_num_blocks
73 +
74
75 class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
76
@@ -69,7 +100,7 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
100 self.vocab_size = config.vocab_size
101 self.max_position = config.max_position_embeddings
102
72 - def _get_param_size(self) -> int:
103 + def get_param_size(self) -> int:
104 word_embedding = self.vocab_size * self.embedding_size // self.tensor_parallel_size
105 if self.embedding_size != self.hidden_size:
106 # Project in/out.
@@ -93,7 +124,7 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
124 dtype_size = get_dtype_size(self.dtype)
125 return dtype_size * total
126
96 - def _get_max_act_size(
127 + def get_max_act_size(
128 self,
129 max_num_batched_tokens: int,
130 ) -> int:
@@ -114,31 +145,6 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
145 dtype_size = get_dtype_size(self.dtype)
146 return dtype_size * max_act
147
117 - def get_cache_block_size(self) -> int:
118 - key_cache_block = self.block_size * self.hidden_size // self.tensor_parallel_size
119 - value_cache_block = key_cache_block
120 - total = self.num_layers * (key_cache_block + value_cache_block)
121 - dtype_size = get_dtype_size(self.dtype)
122 - return dtype_size * total
123 -
124 - def get_max_num_gpu_blocks(
125 - self,
126 - max_num_batched_tokens: int,
127 - memory_utilization: float = 0.95,
128 - ) -> int:
129 - # NOTE(woosuk): This assumes that the machine has homogeneous GPUs.
130 - usable_memory = int(memory_utilization * self.gpu_memory)
131 -
132 - param_size = self._get_param_size()
133 - act_size = self._get_max_act_size(max_num_batched_tokens)
134 - workspace_size = self.get_workspace_size()
135 -
136 - max_cache_size = usable_memory - (param_size + act_size + workspace_size)
137 - if max_cache_size <= 0:
138 - raise RuntimeError('Not enough GPU memory.')
139 - max_num_blocks = max_cache_size // self.get_cache_block_size()
140 - return max_num_blocks
141 -
148
149 class LlamaMemoryAnalyzer(CacheFlowMemoryAnalyzer):
150
@@ -167,9 +173,10 @@ class LlamaMemoryAnalyzer(CacheFlowMemoryAnalyzer):
173 self.vocab_size = config.vocab_size
174 self.max_position = 8192
175
170 - def _get_param_size(self) -> int:
176 + def get_param_size(self) -> int:
177 + # NOTE: LLaMA does not tie the two embeddings.
178 word_embedding = self.vocab_size * self.hidden_size // self.tensor_parallel_size
172 - position_embedding = self.max_position * self.hidden_size
179 + lm_head = self.vocab_size * self.hidden_size // self.tensor_parallel_size
180
181 # NOTE: LLaMA does not have bias terms.
182 ln1 = self.hidden_size
@@ -188,11 +195,11 @@ class LlamaMemoryAnalyzer(CacheFlowMemoryAnalyzer):
195 up = self.hidden_size * self.ffn_size // self.tensor_parallel_size
196 ffn = ln2 + gate + down + up
197
191 - total = (word_embedding + position_embedding + self.num_layers * (mha + ffn))
198 + total = word_embedding + self.num_layers * (mha + ffn) + lm_head
199 dtype_size = get_dtype_size(self.dtype)
200 return dtype_size * total
201
195 - def _get_max_act_size(
202 + def get_max_act_size(
203 self,
204 max_num_batched_tokens: int,
205 ) -> int:
@@ -213,28 +220,78 @@ class LlamaMemoryAnalyzer(CacheFlowMemoryAnalyzer):
220 dtype_size = get_dtype_size(self.dtype)
221 return dtype_size * max_act
222
216 - def get_cache_block_size(self) -> int:
217 - key_cache_block = self.block_size * self.hidden_size // self.tensor_parallel_size
218 - value_cache_block = key_cache_block
219 - total = self.num_layers * (key_cache_block + value_cache_block)
223 +
224 +class GPTNeoXMemoryAnalyzer(CacheFlowMemoryAnalyzer):
225 +
226 + def __init__(
227 + self,
228 + model_name: str,
229 + block_size: int,
230 + dtype: torch.dtype,
231 + gpu_memory: int,
232 + cpu_memory: int,
233 + tensor_parallel_size: int,
234 + ) -> None:
235 + self.model_name = model_name
236 + self.block_size = block_size
237 + self.dtype = dtype
238 + self.gpu_memory = gpu_memory
239 + self.cpu_memory = cpu_memory
240 + self.tensor_parallel_size = tensor_parallel_size
241 +
242 + config = AutoConfig.from_pretrained(model_name)
243 + self.num_layers = config.num_hidden_layers
244 + self.hidden_size = config.hidden_size
245 + self.num_heads = config.num_attention_heads
246 + self.head_size = config.hidden_size // self.num_heads
247 + self.ffn_size = config.intermediate_size
248 + self.vocab_size = config.vocab_size
249 + self.max_position = 8192
250 + self.tie_word_embeddings = config.tie_word_embeddings
251 +
252 + def get_param_size(self) -> int:
253 + word_embedding = self.vocab_size * self.hidden_size // self.tensor_parallel_size
254 + if self.tie_word_embeddings:
255 + lm_head = 0
256 + else:
257 + lm_head = self.vocab_size * self.hidden_size // self.tensor_parallel_size
258 +
259 + ln1 = 2 * self.hidden_size
260 + q = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
261 + k = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
262 + v = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
263 + out = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
264 + # Rotary embedding.
265 + # TODO(woosuk): Share the rotary embedding between layers.
266 + rot = self.max_position * self.head_size
267 + mha = ln1 + q + k + v + out + rot
268 +
269 + ln2 = 2 * self.hidden_size
270 + ffn1 = self.hidden_size * self.ffn_size // self.tensor_parallel_size + self.ffn_size
271 + ffn2 = self.ffn_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
272 + ffn = ln2 + ffn1 + ffn2
273 +
274 + total = word_embedding + self.num_layers * (mha + ffn) + lm_head
275 dtype_size = get_dtype_size(self.dtype)
276 return dtype_size * total
277
223 - def get_max_num_gpu_blocks(
278 + def get_max_act_size(
279 self,
280 max_num_batched_tokens: int,
226 - memory_utilization: float = 0.95,
281 ) -> int:
228 - # NOTE(woosuk): This assumes that the machine has homogeneous GPUs.
229 - gpu_memory = self.gpu_memory
230 - usable_memory = int(memory_utilization * gpu_memory)
231 -
232 - param_size = self._get_param_size()
233 - act_size = self._get_max_act_size(max_num_batched_tokens)
234 - workspace_size = self.get_workspace_size()
235 -
236 - max_cache_size = usable_memory - (param_size + act_size + workspace_size)
237 - if max_cache_size <= 0:
238 - raise RuntimeError('Not enough GPU memory.')
239 - max_num_blocks = max_cache_size // self.get_cache_block_size()
240 - return max_num_blocks
282 + # NOTE: We approxmiately calculate the maximum activation size by
283 + # estimating
284 + # 1) the maximum activation tensor size during inference
285 + # 2) the residual tensor size during inference
286 + # Here, we assume that FlashAttention is used and
287 + # thus the attention maps are never materialized in GPU DRAM.
288 + residual = max_num_batched_tokens * self.hidden_size
289 + qkv = 3 * (max_num_batched_tokens * self.hidden_size) // self.tensor_parallel_size
290 + ffn = 2 * (max_num_batched_tokens * self.ffn_size) // self.tensor_parallel_size
291 + # Double the activation size for input and output.
292 + max_act = 2 * (max(qkv, ffn) + residual)
293 + # Size of output logits.
294 + output_logits = 2 * (max_num_batched_tokens * self.vocab_size)
295 + max_act = max(max_act, output_logits)
296 + dtype_size = get_dtype_size(self.dtype)
297 + return dtype_size * max_act
cacheflow/models/model_utils.py
+6 -1
@@ -1,13 +1,14 @@
1 from typing import Union
2
3 -import numpy as np
3 import torch
4 import torch.nn as nn
5 from transformers import AutoConfig
6
7 from cacheflow.models.memory_analyzer import CacheFlowMemoryAnalyzer
8 +from cacheflow.models.memory_analyzer import GPTNeoXMemoryAnalyzer
9 from cacheflow.models.memory_analyzer import LlamaMemoryAnalyzer
10 from cacheflow.models.memory_analyzer import OPTMemoryAnalyzer
11 +from cacheflow.models.gpt_neox import GPTNeoXForCausalLM
12 from cacheflow.models.llama import LlamaForCausalLM
13 from cacheflow.models.opt import OPTForCausalLM
14 from cacheflow.models.utils import get_torch_dtype
@@ -16,11 +17,15 @@ from cacheflow.models.utils import get_torch_dtype
17 _MODELS = {
18 'llama': LlamaForCausalLM,
19 'opt': OPTForCausalLM,
20 + 'stablelm': GPTNeoXForCausalLM,
21 + 'pythia': GPTNeoXForCausalLM,
22 }
23
24 _MEMORY_ANALYZERS = {
25 'llama': LlamaMemoryAnalyzer,
26 'opt': OPTMemoryAnalyzer,
27 + 'stablelm': GPTNeoXMemoryAnalyzer,
28 + 'pythia': GPTNeoXMemoryAnalyzer,
29 }
30
31
cacheflow/models/opt.py
+1 -1
@@ -327,4 +327,4 @@ class OPTForCausalLM(nn.Module):
327
328 def initialize_dummy_weights(self) -> None:
329 for param in self.state_dict().values():
330 - param.data.uniform_(-0.1, 0.1)
330 + param.data.uniform_(-1e-3, 1e-3)
csrc/pos_encoding.cpp
+1
@@ -4,6 +4,7 @@ void rotary_embedding_neox(
4 torch::Tensor& positions,
5 torch::Tensor& query,
6 torch::Tensor& key,
7 + int head_size,
8 torch::Tensor& cos_sin_cache);
9
10 PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
csrc/pos_encoding_kernels.cu
+9 -6
@@ -8,16 +8,17 @@ __global__ void rotary_embedding_neox_kernel(
8 const int64_t* __restrict__ positions, // [num_tokens]
9 scalar_t* __restrict__ query, // [num_tokens, num_heads, head_size]
10 scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
11 - const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, head_size // 2]
11 + const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
12 + const int rot_dim,
13 const int stride,
14 const int num_heads,
15 const int head_size) {
16 // Each thread block is responsible for one token.
17 const int token_idx = blockIdx.x;
18 int64_t pos = positions[token_idx];
18 - const scalar_t* cache_ptr = cos_sin_cache + pos * head_size;
19 + const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim;
20
20 - const int embed_dim = head_size / 2;
21 + const int embed_dim = rot_dim / 2;
22 const int n = num_heads * embed_dim;
23 for (int i = threadIdx.x; i < n; i += blockDim.x) {
24 const int head_idx = i / embed_dim;
@@ -51,16 +52,17 @@ void rotary_embedding_neox(
52 torch::Tensor& positions, // [num_tokens]
53 torch::Tensor& query, // [num_tokens, num_heads * head_size]
54 torch::Tensor& key, // [num_tokens, num_heads * head_size]
54 - torch::Tensor& cos_sin_cache) // [max_position, head_size]
55 + int head_size,
56 + torch::Tensor& cos_sin_cache) // [max_position, rot_dim]
57 {
58 int num_tokens = query.size(0);
57 - int head_size = cos_sin_cache.size(1);
59 + int rot_dim = cos_sin_cache.size(1);
60 int num_heads = query.size(1) / head_size;
61 int stride = query.stride(0);
62 TORCH_CHECK(stride == key.stride(0));
63
64 dim3 grid(num_tokens);
63 - dim3 block(std::min(num_heads * head_size / 2, 512));
65 + dim3 block(std::min(num_heads * rot_dim / 2, 512));
66 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
67 AT_DISPATCH_FLOATING_TYPES_AND_HALF(
68 query.scalar_type(),
@@ -71,6 +73,7 @@ void rotary_embedding_neox(
73 query.data_ptr<scalar_t>(),
74 key.data_ptr<scalar_t>(),
75 cos_sin_cache.data_ptr<scalar_t>(),
76 + rot_dim,
77 stride,
78 num_heads,
79 head_size);
tests/kernels/pos_encoding.py
+22 -7
@@ -34,6 +34,7 @@ class RefRotaryEmbeddingNeox(nn.Module):
34 base: int = 10000,
35 ) -> None:
36 super().__init__()
37 + self.rotary_dim = dim
38 self.max_position_embeddings = max_position_embeddings
39
40 # Create cos and sin embeddings.
@@ -52,13 +53,24 @@ class RefRotaryEmbeddingNeox(nn.Module):
53 query: torch.Tensor, # [num_tokens, num_heads, head_size]
54 key: torch.Tensor, # [num_tokens, num_heads, head_size]
55 ) -> Tuple[torch.Tensor, torch.Tensor]:
56 +
57 + query_rot = query[..., : self.rotary_dim]
58 + query_pass = query[..., self.rotary_dim :]
59 + key_rot = key[..., : self.rotary_dim]
60 + key_pass = key[..., self.rotary_dim :]
61 +
62 +
63 + query_rot = query_rot.transpose(0, 1)
64 + key_rot = key_rot.transpose(0, 1)
65 cos = F.embedding(positions, self.cos_cached)
66 sin = F.embedding(positions, self.sin_cached)
57 - query = query.transpose(0, 1)
58 - key = key.transpose(0, 1)
59 - query, key = apply_rotary_pos_emb(query, key, cos, sin)
60 - query = query.transpose(0, 1).contiguous()
61 - key = key.transpose(0, 1).contiguous()
67 + query_rot, key_rot = apply_rotary_pos_emb(query_rot, key_rot, cos, sin)
68 + query_rot = query_rot.transpose(0, 1).contiguous()
69 + key_rot = key_rot.transpose(0, 1).contiguous()
70 +
71 + query = torch.cat((query_rot, query_pass), dim=-1)
72 + key = torch.cat((key_rot, key_pass), dim=-1)
73 +
74 # Output query/key shape: [num_tokens, num_tokens, head_size]
75 return query, key
76
@@ -69,6 +81,7 @@ def test_rotary_embedding_neox(
81 num_heads: int,
82 head_size: int,
83 max_position: int,
84 + rotary_dim: int,
85 dtype: torch.dtype,
86 base: int = 10000,
87 ) -> None:
@@ -77,7 +90,7 @@ def test_rotary_embedding_neox(
90 key = torch.randn(num_tokens, num_heads * head_size, dtype=dtype, device='cuda')
91
92 # Create the rotary embedding.
80 - inv_freq = 1.0 / (base ** (torch.arange(0, head_size, 2) / head_size))
93 + inv_freq = 1.0 / (base ** (torch.arange(0, rotary_dim, 2) / rotary_dim))
94 t = torch.arange(max_position).float()
95 freqs = torch.einsum('i,j -> ij', t, inv_freq.float())
96 cos = freqs.cos()
@@ -92,12 +105,13 @@ def test_rotary_embedding_neox(
105 positions,
106 out_query,
107 out_key,
108 + head_size,
109 cos_sin_cache,
110 )
111
112 # Run the reference implementation.
113 ref_rotary_embedding = RefRotaryEmbeddingNeox(
100 - dim=head_size,
114 + dim=rotary_dim,
115 max_position_embeddings=max_position,
116 base=base,
117 ).to(dtype=dtype, device='cuda')
@@ -123,5 +137,6 @@ if __name__ == '__main__':
137 num_heads=5,
138 head_size=head_size,
139 max_position=8192,
140 + rotary_dim=int(head_size * 0.25),
141 dtype=dtype,
142 )