A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Implement LLaMA (#9)
Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
Woosuk Kwon committed
Mar 29, 2023 at 21:25 UTC
80a2f812f17add5838f84288054fbe0b915622cc
7 files changed
+500
-35
README.md
+4
-2
@@ -3,8 +3,10 @@
3
## Installation
4
5
```bash
6
-pip install psutil numpy torch transformers
7
-pip install flash-attn # This may take up to 10 mins.
6
+pip install psutil numpy ray torch
7
+pip install git+https://github.com/huggingface/transformers # Required for LLaMA.
8
+pip install sentencepiece # Required for LlamaTokenizer.
9
+pip install flash-attn # This may take up to 20 mins.
10
pip install -e .
11
```
12
cacheflow/master/simple_frontend.py
+1
@@ -61,4 +61,5 @@ class SimpleFrontend:
61
for seq in seq_group.seqs:
62
token_ids = seq.get_token_ids()
63
output = self.tokenizer.decode(token_ids, skip_special_tokens=True)
64
+ output = output.strip()
65
print(f'Seq {seq.seq_id}: {output!r}')
cacheflow/models/llama.py
new
+357
@@ -0,0 +1,357 @@
1
+"""1D LLaMA model compatible with HuggingFace weights."""
2
+import os
3
+import glob
4
+import filelock
5
+from tqdm import tqdm
6
+from typing import Dict, List, Optional, Tuple
7
+
8
+import numpy as np
9
+import torch
10
+from torch import nn
11
+import torch.nn.functional as F
12
+from transformers import LlamaConfig
13
+from transformers import PreTrainedModel
14
+
15
+from cacheflow.models import InputMetadata
16
+from cacheflow.models.attention import OPTCacheFlowAttention
17
+from cacheflow.models.sample import Sampler
18
+from cacheflow.parallel_utils.parallel_state import (
19
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
20
+from cacheflow.parallel_utils.tensor_parallel import (VocabParallelEmbedding,
21
+ ColumnParallelLinear,
22
+ RowParallelLinear)
23
+from cacheflow.sequence import SequenceOutputs
24
+
25
+KVCache = Tuple[torch.Tensor, torch.Tensor]
26
+
27
+
28
+class LlamaRMSNorm(nn.Module):
29
+
30
+ def __init__(self, hidden_size, eps=1e-6):
31
+ super().__init__()
32
+ self.weight = nn.Parameter(torch.ones(hidden_size))
33
+ self.variance_epsilon = eps
34
+
35
+ def forward(self, hidden_states):
36
+ variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True)
37
+ hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
38
+ # convert into half-precision if necessary
39
+ if self.weight.dtype in [torch.float16, torch.bfloat16]:
40
+ hidden_states = hidden_states.to(self.weight.dtype)
41
+ return self.weight * hidden_states
42
+
43
+
44
+class LlamaRotaryEmbedding(torch.nn.Module):
45
+
46
+ def __init__(self, dim, max_position_embeddings=2048, base=10000):
47
+ super().__init__()
48
+ self.max_position_embeddings = max_position_embeddings
49
+
50
+ inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2) / dim))
51
+ self.register_buffer("inv_freq", inv_freq)
52
+
53
+ # Create cos and sin embeddings.
54
+ t = torch.arange(max_position_embeddings).float()
55
+ freqs = torch.einsum("i,j->ij", t, self.inv_freq.float())
56
+ emb = torch.cat((freqs, freqs), dim=-1)
57
+ cos = emb.cos().to(dtype=self.inv_freq.dtype)
58
+ sin = emb.sin().to(dtype=self.inv_freq.dtype)
59
+ self.register_buffer("cos_cached", cos, persistent=False)
60
+ self.register_buffer("sin_cached", sin, persistent=False)
61
+
62
+ def forward(
63
+ self,
64
+ positions: torch.LongTensor,
65
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
66
+ cos = F.embedding(positions, self.cos_cached)
67
+ sin = F.embedding(positions, self.sin_cached)
68
+ return cos, sin
69
+
70
+
71
+def rotate_half(x):
72
+ """Rotates half the hidden dims of the input."""
73
+ x1 = x[..., : x.shape[-1] // 2]
74
+ x2 = x[..., x.shape[-1] // 2 :]
75
+ return torch.cat((-x2, x1), dim=-1)
76
+
77
+
78
+def apply_rotary_pos_emb(q, k, cos, sin):
79
+ # TODO: Optimize.
80
+ q_embed = (q * cos) + (rotate_half(q) * sin)
81
+ k_embed = (k * cos) + (rotate_half(k) * sin)
82
+ return q_embed, k_embed
83
+
84
+
85
+class LlamaMLP(nn.Module):
86
+ def __init__(
87
+ self,
88
+ hidden_size: int,
89
+ intermediate_size: int,
90
+ hidden_act: str,
91
+ ):
92
+ super().__init__()
93
+ # TODO: Merge the gate and down linear layers.
94
+ self.gate_proj = ColumnParallelLinear(hidden_size, intermediate_size,
95
+ bias=False, gather_output=False,
96
+ perform_initialization=False)
97
+ self.down_proj = RowParallelLinear(intermediate_size, hidden_size,
98
+ bias=False, input_is_parallel=True,
99
+ perform_initialization=False)
100
+ self.up_proj = ColumnParallelLinear(hidden_size, intermediate_size,
101
+ bias=False, gather_output=False,
102
+ perform_initialization=False)
103
+ assert hidden_act == 'silu'
104
+ self.act_fn = nn.SiLU()
105
+
106
+ def forward(self, x):
107
+ gate, _ = self.gate_proj(x)
108
+ up, _ = self.up_proj(x)
109
+ x = self.act_fn(gate) * up
110
+ x, _ = self.down_proj(x)
111
+ return x
112
+
113
+
114
+class LlamaAttention(nn.Module):
115
+
116
+ def __init__(
117
+ self,
118
+ hidden_size: int,
119
+ num_heads: int,
120
+ ):
121
+ super().__init__()
122
+ self.hidden_size = hidden_size
123
+ tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
124
+ self.total_num_heads = num_heads
125
+ assert self.total_num_heads % tensor_model_parallel_world_size == 0
126
+ self.num_heads = self.total_num_heads // tensor_model_parallel_world_size
127
+ self.head_dim = hidden_size // self.total_num_heads
128
+ self.scaling = self.head_dim ** -0.5
129
+
130
+ # TODO: Merge the QKV linear layers.
131
+ self.q_proj = ColumnParallelLinear(
132
+ hidden_size,
133
+ self.total_num_heads * self.head_dim,
134
+ bias=False,
135
+ gather_output=False,
136
+ perform_initialization=False,
137
+ )
138
+ self.k_proj = ColumnParallelLinear(
139
+ hidden_size,
140
+ self.total_num_heads * self.head_dim,
141
+ bias=False,
142
+ gather_output=False,
143
+ perform_initialization=False,
144
+ )
145
+ self.v_proj = ColumnParallelLinear(
146
+ hidden_size,
147
+ self.total_num_heads * self.head_dim,
148
+ bias=False,
149
+ gather_output=False,
150
+ perform_initialization=False,
151
+ )
152
+ self.o_proj = RowParallelLinear(
153
+ self.total_num_heads * self.head_dim,
154
+ hidden_size,
155
+ bias=False,
156
+ input_is_parallel=True,
157
+ perform_initialization=False,
158
+ )
159
+ self.rotary_emb = LlamaRotaryEmbedding(self.head_dim)
160
+ # FIXME(woosuk): Rename this.
161
+ self.attn = OPTCacheFlowAttention(scale=self.scaling)
162
+
163
+ def forward(
164
+ self,
165
+ positions: torch.LongTensor,
166
+ hidden_states: torch.Tensor,
167
+ kv_cache: KVCache,
168
+ input_metadata: InputMetadata,
169
+ cache_event: Optional[torch.cuda.Event],
170
+ ) -> torch.Tensor:
171
+ q, _ = self.q_proj(hidden_states)
172
+ k, _ = self.k_proj(hidden_states)
173
+ v, _ = self.v_proj(hidden_states)
174
+
175
+ # Apply rotrary embedding.
176
+ # TODO: Optimize.
177
+ q = q.view(-1, self.num_heads, self.head_dim).transpose(0, 1)
178
+ k = k.view(-1, self.num_heads, self.head_dim).transpose(0, 1)
179
+ cos, sin = self.rotary_emb(positions)
180
+ q, k = apply_rotary_pos_emb(q, k, cos, sin)
181
+ q = q.transpose(0, 1).contiguous().view(-1, self.num_heads * self.head_dim)
182
+ k = k.transpose(0, 1).contiguous().view(-1, self.num_heads * self.head_dim)
183
+
184
+ key_cache, value_cache = kv_cache
185
+ attn_output = self.attn(
186
+ q, k, v, key_cache, value_cache, input_metadata, cache_event)
187
+ output, _ = self.o_proj(attn_output)
188
+ return output
189
+
190
+
191
+class LlamaDecoderLayer(nn.Module):
192
+
193
+ def __init__(self, config: LlamaConfig):
194
+ super().__init__()
195
+ self.hidden_size = config.hidden_size
196
+ self.self_attn = LlamaAttention(
197
+ hidden_size=self.hidden_size,
198
+ num_heads=config.num_attention_heads,
199
+ )
200
+ self.mlp = LlamaMLP(
201
+ hidden_size=self.hidden_size,
202
+ intermediate_size=config.intermediate_size,
203
+ hidden_act=config.hidden_act,
204
+ )
205
+ self.input_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
206
+ self.post_attention_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
207
+
208
+ def forward(
209
+ self,
210
+ positions: torch.LongTensor,
211
+ hidden_states: torch.Tensor,
212
+ kv_cache: KVCache,
213
+ input_metadata: InputMetadata,
214
+ cache_event: Optional[torch.cuda.Event],
215
+ ) -> torch.Tensor:
216
+ # Self Attention
217
+ residual = hidden_states
218
+ hidden_states = self.input_layernorm(hidden_states)
219
+ hidden_states = self.self_attn(
220
+ positions=positions,
221
+ hidden_states=hidden_states,
222
+ kv_cache=kv_cache,
223
+ input_metadata=input_metadata,
224
+ cache_event=cache_event,
225
+ )
226
+ hidden_states = residual + hidden_states
227
+
228
+ # Fully Connected
229
+ residual = hidden_states
230
+ hidden_states = self.post_attention_layernorm(hidden_states)
231
+ hidden_states = self.mlp(hidden_states)
232
+ hidden_states = residual + hidden_states
233
+ return hidden_states
234
+
235
+
236
+class LlamaModel(nn.Module):
237
+
238
+ def __init__(self, config: LlamaConfig):
239
+ super().__init__()
240
+ self.config = config
241
+ self.padding_idx = config.pad_token_id
242
+ self.vocab_size = config.vocab_size
243
+
244
+ self.embed_tokens = VocabParallelEmbedding(config.vocab_size, config.hidden_size,
245
+ perform_initialization=False)
246
+ self.layers = nn.ModuleList([LlamaDecoderLayer(config) for _ in range(config.num_hidden_layers)])
247
+ self.norm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
248
+
249
+ def forward(
250
+ self,
251
+ input_ids: torch.LongTensor,
252
+ positions: torch.LongTensor,
253
+ kv_caches: List[KVCache],
254
+ input_metadata: InputMetadata,
255
+ cache_events: Optional[List[torch.cuda.Event]],
256
+ ) -> torch.Tensor:
257
+ hidden_states = self.embed_tokens(input_ids)
258
+ for i in range(len(self.layers)):
259
+ if cache_events is None:
260
+ cache_event = None
261
+ else:
262
+ cache_event = cache_events[i]
263
+ layer = self.layers[i]
264
+ hidden_states = layer(
265
+ positions,
266
+ hidden_states,
267
+ kv_caches[i],
268
+ input_metadata,
269
+ cache_event,
270
+ )
271
+ hidden_states = self.norm(hidden_states)
272
+ return hidden_states
273
+
274
+
275
+class LlamaForCausalLM(nn.Module):
276
+ def __init__(self, config):
277
+ super().__init__()
278
+ self.config = config
279
+ self.model = LlamaModel(config)
280
+ self.lm_head = ColumnParallelLinear(config.hidden_size,
281
+ config.vocab_size,
282
+ bias=False,
283
+ gather_output=False,
284
+ perform_initialization=False)
285
+ self.sampler = Sampler()
286
+
287
+ def forward(
288
+ self,
289
+ input_ids: torch.LongTensor,
290
+ positions: torch.LongTensor,
291
+ kv_caches: List[KVCache],
292
+ input_metadata: InputMetadata,
293
+ cache_events: Optional[List[torch.cuda.Event]],
294
+ ) -> Dict[int, SequenceOutputs]:
295
+ hidden_states = self.model(
296
+ input_ids, positions, kv_caches, input_metadata, cache_events)
297
+ next_tokens = self.sampler(
298
+ self.lm_head.weight, hidden_states, input_metadata)
299
+ return next_tokens
300
+
301
+ _column_parallel_weights = ["embed_tokens.weight", "lm_head.weight",
302
+ "q_proj.weight", "k_proj.weight",
303
+ "v_proj.weight", "gate_proj.weight",
304
+ "up_proj.weight"]
305
+ _row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
306
+
307
+ def load_weights(self, weights_path: str):
308
+ tensor_model_parallel_rank = get_tensor_model_parallel_rank()
309
+ state_dict = self.state_dict()
310
+ for name, param in state_dict.items():
311
+ loaded_weight = torch.from_numpy(np.load(os.path.join(weights_path,
312
+ name)))
313
+ for p in self._column_parallel_weights:
314
+ if p in name:
315
+ shard_size = param.shape[0]
316
+ loaded_weight = loaded_weight[
317
+ shard_size * tensor_model_parallel_rank
318
+ :shard_size * (tensor_model_parallel_rank + 1)]
319
+ break
320
+ for p in self._row_parallel_weights:
321
+ if p in name:
322
+ shard_size = param.shape[1]
323
+ loaded_weight = loaded_weight[
324
+ :,
325
+ shard_size * tensor_model_parallel_rank
326
+ :shard_size * (tensor_model_parallel_rank + 1)]
327
+ break
328
+
329
+ assert param.shape == loaded_weight.shape
330
+ param.data.copy_(loaded_weight)
331
+
332
+ @staticmethod
333
+ def get_weights(model_name: str, path: str):
334
+ if not os.path.isfile(os.path.join(model_name, "config.json")):
335
+ raise ValueError("LLaMA model's model_name has to be a path"
336
+ "to the huggingface model's directory.")
337
+ path = os.path.join(model_name, f"np")
338
+ path = os.path.abspath(os.path.expanduser(path))
339
+ os.makedirs(path, exist_ok=True)
340
+ lock_path = os.path.join(path, "file_lock")
341
+ lock = filelock.FileLock(lock_path)
342
+
343
+ with lock:
344
+ test_weight_path = os.path.join(path, "model.embed_tokens.weight")
345
+ if os.path.exists(test_weight_path):
346
+ return path
347
+
348
+ bin_files = glob.glob(os.path.join(model_name, "*.bin"))
349
+
350
+ for bin_file in tqdm(bin_files, desc="Convert format"):
351
+ state = torch.load(bin_file, map_location="cpu")
352
+ for name, param in tqdm(state.items(), leave=False):
353
+ param_path = os.path.join(path, name)
354
+ with open(param_path, "wb") as f:
355
+ np.save(f, param.cpu().detach().numpy())
356
+
357
+ return path
cacheflow/models/memory_analyzer.py
+130
-26
@@ -15,11 +15,30 @@ class CacheFlowMemoryAnalyzer:
15
) -> int:
16
raise NotImplementedError()
17
18
+ def get_workspace_size(self) -> int:
19
+ return 1 * _GiB
20
+
21
+ def get_cache_block_size(self) -> int:
22
+ raise NotImplementedError()
23
+
24
def get_max_num_cpu_blocks(
25
self,
20
- memory_utilization: float,
26
+ swap_space: int,
27
) -> int:
22
- raise NotImplementedError()
28
+ swap_space = swap_space * _GiB
29
+ cpu_memory = self.cpu_memory
30
+ if swap_space > 0.8 * cpu_memory:
31
+ raise ValueError(f'The swap space ({swap_space / _GiB:.2f} GiB) '
32
+ 'takes more than 80% of the available memory '
33
+ f'({cpu_memory / _GiB:.2f} GiB).'
34
+ 'Please check the swap space size.')
35
+ if swap_space > 0.5 * cpu_memory:
36
+ print(f'WARNING: The swap space ({swap_space / _GiB:.2f} GiB) '
37
+ 'takes more than 50% of the available memory '
38
+ f'({cpu_memory / _GiB:.2f} GiB).'
39
+ 'This may slow the system performance.')
40
+ max_num_blocks = swap_space // self.get_cache_block_size()
41
+ return max_num_blocks
42
43
44
class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
@@ -52,9 +71,9 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
71
72
def _get_param_size(self) -> int:
73
word_embedding = self.vocab_size * self.embedding_size // self.tensor_parallel_size
55
- if self.embedding_size != self.vocab_size:
74
+ if self.embedding_size != self.hidden_size:
75
# Project in/out.
57
- word_embedding += 2 * self.embedding_size * self.vocab_size
76
+ word_embedding += 2 * self.embedding_size * self.hidden_size
77
position_embedding = self.max_position * self.hidden_size
78
79
ln1 = 2 * self.hidden_size
@@ -89,15 +108,15 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
108
ffn = max_num_batched_tokens * self.ffn_size // self.tensor_parallel_size
109
# Double the activation size for input and output.
110
max_act = 2 * (max(qkv, ffn) + residual)
111
+ # Size of output logits.
112
+ output_logits = 2 * (max_num_batched_tokens * self.vocab_size)
113
+ max_act = max(max_act, output_logits)
114
dtype_size = get_dtype_size(self.dtype)
115
return dtype_size * max_act
116
95
- def _get_workspace_size(self) -> int:
96
- return 1 * _GiB
97
-
98
- def _get_cache_block_size(self) -> int:
99
- key_cache_block = self.block_size * self.num_heads * self.head_size
100
- value_cache_block = self.block_size * self.num_heads * self.head_size
117
+ def get_cache_block_size(self) -> int:
118
+ key_cache_block = self.block_size * self.hidden_size // self.tensor_parallel_size
119
+ value_cache_block = key_cache_block
120
total = self.num_layers * (key_cache_block + value_cache_block)
121
dtype_size = get_dtype_size(self.dtype)
122
return dtype_size * total
@@ -112,26 +131,111 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
131
132
param_size = self._get_param_size()
133
act_size = self._get_max_act_size(max_num_batched_tokens)
115
- workspace_size = self._get_workspace_size()
134
+ workspace_size = self.get_workspace_size()
135
136
max_cache_size = usable_memory - (param_size + act_size + workspace_size)
118
- max_num_blocks = max_cache_size // self._get_cache_block_size()
137
+ if max_cache_size <= 0:
138
+ raise RuntimeError('Not enough GPU memory.')
139
+ max_num_blocks = max_cache_size // self.get_cache_block_size()
140
return max_num_blocks
141
121
- def get_max_num_cpu_blocks(
142
+
143
+class LlamaMemoryAnalyzer(CacheFlowMemoryAnalyzer):
144
+
145
+ def __init__(
146
self,
123
- swap_space: int,
147
+ model_name: str,
148
+ block_size: int,
149
+ dtype: torch.dtype,
150
+ gpu_memory: int,
151
+ cpu_memory: int,
152
+ tensor_parallel_size: int,
153
+ ) -> None:
154
+ self.model_name = model_name
155
+ self.block_size = block_size
156
+ self.dtype = dtype
157
+ self.gpu_memory = gpu_memory
158
+ self.cpu_memory = cpu_memory
159
+ self.tensor_parallel_size = tensor_parallel_size
160
+
161
+ config = AutoConfig.from_pretrained(model_name)
162
+ self.num_layers = config.num_hidden_layers
163
+ self.hidden_size = config.hidden_size
164
+ self.num_heads = config.num_attention_heads
165
+ self.head_size = config.hidden_size // self.num_heads
166
+ self.ffn_size = config.intermediate_size
167
+ self.vocab_size = config.vocab_size
168
+ # FIXME
169
+ self.max_position = 2048
170
+
171
+ def _get_param_size(self) -> int:
172
+ word_embedding = self.vocab_size * self.hidden_size // self.tensor_parallel_size
173
+ position_embedding = self.max_position * self.hidden_size
174
+
175
+ # NOTE: LLaMA does not have bias terms.
176
+ ln1 = self.hidden_size
177
+ q = self.hidden_size * self.hidden_size // self.tensor_parallel_size
178
+ k = self.hidden_size * self.hidden_size // self.tensor_parallel_size
179
+ v = self.hidden_size * self.hidden_size // self.tensor_parallel_size
180
+ out = self.hidden_size * self.hidden_size // self.tensor_parallel_size
181
+ # Rotary embedding.
182
+ # TODO(woosuk): Share the rotary embedding between layers.
183
+ rot = self.max_position * self.head_size
184
+ mha = ln1 + q + k + v + out + rot
185
+
186
+ ln2 = self.hidden_size
187
+ gate = self.hidden_size * self.ffn_size // self.tensor_parallel_size
188
+ down = self.ffn_size * self.hidden_size // self.tensor_parallel_size
189
+ up = self.hidden_size * self.ffn_size // self.tensor_parallel_size
190
+ ffn = ln2 + gate + down + up
191
+
192
+ total = (word_embedding + position_embedding + self.num_layers * (mha + ffn))
193
+ dtype_size = get_dtype_size(self.dtype)
194
+ return dtype_size * total
195
+
196
+ def _get_max_act_size(
197
+ self,
198
+ max_num_batched_tokens: int,
199
) -> int:
125
- swap_space = swap_space * _GiB
126
- if swap_space > 0.8 * self.cpu_memory:
127
- raise ValueError(f'The swap space ({swap_space / _GiB:.2f} GiB) '
128
- 'takes more than 80% of the available memory '
129
- f'({self.cpu_memory / _GiB:.2f} GiB).'
130
- 'Please check the swap space size.')
131
- if swap_space > 0.5 * self.cpu_memory:
132
- print(f'WARNING: The swap space ({swap_space / _GiB:.2f} GiB) '
133
- 'takes more than 50% of the available memory '
134
- f'({self.cpu_memory / _GiB:.2f} GiB).'
135
- 'This may slow the system performance.')
136
- max_num_blocks = swap_space // self._get_cache_block_size()
200
+ # NOTE: We approxmiately calculate the maximum activation size by
201
+ # estimating
202
+ # 1) the maximum activation tensor size during inference
203
+ # 2) the residual tensor size during inference
204
+ # Here, we assume that FlashAttention is used and
205
+ # thus the attention maps are never materialized in GPU DRAM.
206
+ residual = max_num_batched_tokens * self.hidden_size
207
+ qkv = 3 * (max_num_batched_tokens * self.hidden_size) // self.tensor_parallel_size
208
+ ffn = 2 * (max_num_batched_tokens * self.ffn_size) // self.tensor_parallel_size
209
+ # Double the activation size for input and output.
210
+ max_act = 2 * (max(qkv, ffn) + residual)
211
+ # Size of output logits.
212
+ output_logits = 2 * (max_num_batched_tokens * self.vocab_size)
213
+ max_act = max(max_act, output_logits)
214
+ dtype_size = get_dtype_size(self.dtype)
215
+ return dtype_size * max_act
216
+
217
+ def get_cache_block_size(self) -> int:
218
+ key_cache_block = self.block_size * self.hidden_size // self.tensor_parallel_size
219
+ value_cache_block = key_cache_block
220
+ total = self.num_layers * (key_cache_block + value_cache_block)
221
+ dtype_size = get_dtype_size(self.dtype)
222
+ return dtype_size * total
223
+
224
+ def get_max_num_gpu_blocks(
225
+ self,
226
+ max_num_batched_tokens: int,
227
+ memory_utilization: float = 0.95,
228
+ ) -> int:
229
+ # NOTE(woosuk): This assumes that the machine has homogeneous GPUs.
230
+ gpu_memory = self.gpu_memory
231
+ usable_memory = int(memory_utilization * gpu_memory)
232
+
233
+ param_size = self._get_param_size()
234
+ act_size = self._get_max_act_size(max_num_batched_tokens)
235
+ workspace_size = self.get_workspace_size()
236
+
237
+ max_cache_size = usable_memory - (param_size + act_size + workspace_size)
238
+ if max_cache_size <= 0:
239
+ raise RuntimeError('Not enough GPU memory.')
240
+ max_num_blocks = max_cache_size // self.get_cache_block_size()
241
return max_num_blocks
cacheflow/models/model_utils.py
+5
-1
@@ -6,16 +6,20 @@ import torch.nn as nn
6
from transformers import AutoConfig
7
8
from cacheflow.models.memory_analyzer import CacheFlowMemoryAnalyzer
9
+from cacheflow.models.memory_analyzer import LlamaMemoryAnalyzer
10
from cacheflow.models.memory_analyzer import OPTMemoryAnalyzer
11
+from cacheflow.models.llama import LlamaForCausalLM
12
from cacheflow.models.opt import OPTForCausalLM
13
from cacheflow.models.utils import get_torch_dtype
14
15
16
_MODELS = {
17
+ 'llama': LlamaForCausalLM,
18
'opt': OPTForCausalLM,
19
}
20
21
_MEMORY_ANALYZERS = {
22
+ 'llama': LlamaMemoryAnalyzer,
23
'opt': OPTMemoryAnalyzer,
24
}
25
@@ -31,7 +35,7 @@ def get_model(
35
for model_class_name, model_class in _MODELS.items():
36
if model_class_name in model_name:
37
# Download model weights if it's not cached.
34
- weights_dir = model_class.download_weights(model_name, path=path)
38
+ weights_dir = model_class.get_weights(model_name, path=path)
39
# Create a model instance.
40
model = model_class(config)
41
# Load the weights from the cached or downloaded files.
cacheflow/models/opt.py
+2
-5
@@ -299,7 +299,7 @@ class OPTForCausalLM(nn.Module):
299
param.data.copy_(loaded_weight)
300
301
@staticmethod
302
- def download_weights(model_name: str, path: str):
302
+ def get_weights(model_name: str, path: str):
303
path = os.path.join(path, f"{model_name}-np")
304
path = os.path.abspath(os.path.expanduser(path))
305
os.makedirs(path, exist_ok=True)
@@ -316,11 +316,8 @@ class OPTForCausalLM(nn.Module):
316
cache_dir=os.path.join(path, "cache"))
317
bin_files = glob.glob(os.path.join(folder, "*.bin"))
318
319
- if "/" in model_name:
320
- model_name = model_name.split("/")[1].lower()
321
-
319
for bin_file in tqdm(bin_files, desc="Convert format"):
323
- state = torch.load(bin_file)
320
+ state = torch.load(bin_file, map_location="cpu")
321
for name, param in tqdm(state.items(), leave=False):
322
if name.startswith("decoder."):
323
name = "model." + name
cacheflow/models/sample.py
+1
-1
@@ -39,7 +39,7 @@ class Sampler(nn.Module):
39
# Compute the probabilities.
40
probs = torch.softmax(logits, dim=-1, dtype=torch.float)
41
# Compute the log probabilities (before applying top-p).
42
- logprobs = torch.log(probs)
42
+ logprobs = torch.log(probs, out=logits)
43
44
# Apply top-p truncation.
45
top_ps = _get_top_ps(input_metadata)