A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
New weight loader without np copy (#52)
Zhuohan Li committed
May 3, 2023 at 15:32 UTC
27f1410d065ceca53a07abd2518082eb25228e4f
12 files changed
+289
-357
benchmark/benchmark_latency.py
+4
-42
@@ -6,53 +6,15 @@ from tqdm import tqdm
6
import numpy as np
7
import torch
8
9
-from cacheflow.master.simple_frontend import SimpleFrontend
10
-from cacheflow.master.server import (Server, add_server_arguments,
11
- process_server_arguments,
12
- initialize_cluster)
9
+from cacheflow.master.server import (
10
+ add_server_arguments, process_server_arguments,
11
+ init_local_server_and_frontend_with_arguments)
12
from cacheflow.sampling_params import SamplingParams
14
-from cacheflow.utils import get_gpu_memory, get_cpu_memory
13
14
15
def main(args: argparse.Namespace):
18
- # TODO(zhuohan): Support pipeline parallelism.
19
- assert args.pipeline_parallel_size == 1, (
20
- 'Pipeline parallelism is not supported yet.')
16
+ server, frontend = init_local_server_and_frontend_with_arguments(args)
17
22
- (num_nodes, num_devices_per_node, distributed_init_method,
23
- all_stage_devices) = (
24
- initialize_cluster(
25
- use_ray=args.use_ray,
26
- pipeline_parallel_size=args.pipeline_parallel_size,
27
- tensor_parallel_size=args.tensor_parallel_size))
28
-
29
- # Create a server.
30
- server = Server(
31
- model=args.model,
32
- model_path=args.model_path,
33
- use_dummy_weights=args.use_dummy_weights,
34
- pipeline_parallel_size=args.pipeline_parallel_size,
35
- tensor_parallel_size=args.tensor_parallel_size,
36
- block_size=args.block_size,
37
- dtype=args.dtype,
38
- seed=args.seed,
39
- swap_space=args.swap_space,
40
- max_num_batched_tokens=args.max_num_batched_tokens,
41
- max_num_sequences=args.max_num_sequences,
42
- num_nodes=num_nodes,
43
- num_devices_per_node=num_devices_per_node,
44
- distributed_init_method=distributed_init_method,
45
- all_stage_devices=all_stage_devices,
46
- gpu_memory=get_gpu_memory(),
47
- cpu_memory=get_cpu_memory(),
48
- use_ray=args.use_ray,
49
- )
50
-
51
- # Create a frontend.
52
- frontend = SimpleFrontend(
53
- model_name=args.model,
54
- block_size=args.block_size,
55
- )
18
sampling_params_dict = {
19
'n': args.n,
20
'temperature': 0.0 if args.use_beam_search else 1.0,
benchmark/benchmark_text_completion.py
+4
-43
@@ -9,57 +9,18 @@ from tqdm import tqdm
9
from transformers import AutoConfig
10
11
from benchmark.trace import generate_text_completion_requests
12
-from cacheflow.master.simple_frontend import SimpleFrontend
13
-from cacheflow.master.server import (Server, add_server_arguments,
14
- process_server_arguments,
15
- initialize_cluster)
12
+from cacheflow.master.server import (
13
+ add_server_arguments, process_server_arguments,
14
+ init_local_server_and_frontend_with_arguments)
15
from cacheflow.sampling_params import SamplingParams
17
-from cacheflow.utils import get_gpu_memory, get_cpu_memory
16
17
18
logger = logging.getLogger(__name__)
19
20
21
def main(args: argparse.Namespace):
24
- assert args.pipeline_parallel_size == 1, (
25
- 'Pipeline parallelism is not supported yet.')
22
+ server, frontend = init_local_server_and_frontend_with_arguments(args)
23
27
- (num_nodes, num_devices_per_node, distributed_init_method,
28
- all_stage_devices) = (
29
- initialize_cluster(
30
- use_ray=args.use_ray,
31
- pipeline_parallel_size=args.pipeline_parallel_size,
32
- tensor_parallel_size=args.tensor_parallel_size))
33
-
34
- # Create a server.
35
- server = Server(
36
- model=args.model,
37
- model_path=args.model_path,
38
- use_dummy_weights=args.use_dummy_weights,
39
- pipeline_parallel_size=args.pipeline_parallel_size,
40
- tensor_parallel_size=args.tensor_parallel_size,
41
- block_size=args.block_size,
42
- dtype=args.dtype,
43
- seed=args.seed,
44
- swap_space=args.swap_space,
45
- max_num_batched_tokens=args.max_num_batched_tokens,
46
- max_num_sequences=args.max_num_sequences,
47
- num_nodes=num_nodes,
48
- num_devices_per_node=num_devices_per_node,
49
- distributed_init_method=distributed_init_method,
50
- all_stage_devices=all_stage_devices,
51
- gpu_memory=get_gpu_memory(),
52
- cpu_memory=get_cpu_memory(),
53
- use_ray=args.use_ray,
54
- collect_stats=True,
55
- do_memory_analysis=args.do_memory_analysis,
56
- )
57
-
58
- # Create a frontend.
59
- frontend = SimpleFrontend(
60
- model_name=args.model,
61
- block_size=args.block_size,
62
- )
24
# Generate requests.
25
requests = generate_text_completion_requests(
26
args.dataset,
cacheflow/http_frontend/fastapi_frontend.py
+10
-7
@@ -1,7 +1,7 @@
1
import argparse
2
import asyncio
3
import time
4
-from typing import List, Dict
4
+from typing import List, Dict, Optional
5
import json
6
7
import ray
@@ -22,11 +22,12 @@ TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
22
app = FastAPI()
23
24
25
-class FastAPIFrontend:
25
+class FastAPIServer:
26
def __init__(
27
self,
28
model: str,
29
- model_path: str,
29
+ cache_dir: Optional[str],
30
+ use_np_cache: bool,
31
pipeline_parallel_size: int,
32
tensor_parallel_size: int,
33
block_size: int,
@@ -52,8 +53,9 @@ class FastAPIFrontend:
53
remote_server_class = ray.remote(num_gpus=1)(Server)
54
self.server = remote_server_class.remote(
55
model=model,
55
- model_path=model_path,
56
+ cache_dir=cache_dir,
57
use_dummy_weights=False,
58
+ use_np_cache=use_np_cache,
59
pipeline_parallel_size=pipeline_parallel_size,
60
tensor_parallel_size=tensor_parallel_size,
61
block_size=block_size,
@@ -148,7 +150,7 @@ class FastAPIFrontend:
150
@app.post("/generate")
151
async def generate_stream(request: Request):
152
request_dict = await request.json()
151
- return StreamingResponse(frontend.generate(request_dict))
153
+ return StreamingResponse(server.generate(request_dict))
154
155
156
if __name__ == "__main__":
@@ -170,9 +172,10 @@ if __name__ == "__main__":
172
pipeline_parallel_size=args.pipeline_parallel_size,
173
tensor_parallel_size=args.tensor_parallel_size))
174
173
- frontend = FastAPIFrontend(
175
+ server = FastAPIServer(
176
model=args.model,
175
- model_path=args.model_path,
177
+ cache_dir=args.cache_dir,
178
+ use_np_cache=args.use_np_cache,
179
pipeline_parallel_size=args.pipeline_parallel_size,
180
tensor_parallel_size=args.tensor_parallel_size,
181
block_size=args.block_size,
cacheflow/master/server.py
+58
-7
@@ -9,18 +9,21 @@ except ImportError:
9
ray = None
10
11
from cacheflow.master.scheduler import Scheduler
12
+from cacheflow.master.simple_frontend import SimpleFrontend
13
from cacheflow.models import get_memory_analyzer
14
from cacheflow.worker.controller import Controller, DeviceID
15
from cacheflow.sequence import SequenceGroup
16
from cacheflow.sampling_params import SamplingParams
17
+from cacheflow.utils import get_gpu_memory, get_cpu_memory
18
19
20
class Server:
21
def __init__(
22
self,
23
model: str,
22
- model_path: str,
24
+ cache_dir: Optional[str],
25
use_dummy_weights: bool,
26
+ use_np_cache: bool,
27
pipeline_parallel_size: int,
28
tensor_parallel_size: int,
29
block_size: int,
@@ -78,8 +81,9 @@ class Server:
81
num_cpu_blocks=self.num_cpu_blocks,
82
dtype=dtype,
83
seed=seed,
81
- model_path=model_path,
84
+ cache_dir=cache_dir,
85
use_dummy_weights=use_dummy_weights,
86
+ use_np_cache=use_np_cache,
87
max_num_batched_tokens=max_num_batched_tokens,
88
use_ray=use_ray,
89
)
@@ -203,25 +207,72 @@ def initialize_cluster(
207
def add_server_arguments(parser: argparse.ArgumentParser):
208
# Model arguments
209
parser.add_argument('--model', type=str, default='facebook/opt-125m', help='model name')
206
- parser.add_argument('--model-path', type=str, default='~/.cacheflow/model_weights',
207
- help='model path to download and load the weights')
210
+ parser.add_argument('--cache-dir', type=str, default=None,
211
+ help='cache dir to download and load the weights, '
212
+ 'default to the default cache dir of huggingface')
213
+ parser.add_argument('--use-np-cache', action='store_true',
214
+ help='save a numpy copy of model weights for faster loading')
215
+ parser.add_argument('--use-dummy-weights', action='store_true', help='use dummy values for model weights')
216
+ # NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
217
+ parser.add_argument('--dtype', type=str, default='half', choices=['half'], help='data type')
218
# Parallel arguments
219
parser.add_argument('--use-ray', action='store_true', help='use Ray for distributed serving, will be automatically set when using more than 1 GPU')
220
parser.add_argument('--pipeline-parallel-size', '-pp', type=int, default=1, help='number of pipeline stages')
221
parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1, help='number of tensor parallel replicas')
222
# KV cache arguments
223
parser.add_argument('--block-size', type=int, default=16, choices=[1, 2, 4, 8, 16, 32, 64, 128, 256], help='token block size')
214
- # NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
215
- parser.add_argument('--dtype', type=str, default='half', choices=['half'], help='data type')
224
# TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
225
parser.add_argument('--seed', type=int, default=0, help='random seed')
226
parser.add_argument('--swap-space', type=int, default=20, help='CPU swap space size (GiB) per GPU')
227
parser.add_argument('--max-num-batched-tokens', type=int, default=2560, help='maximum number of batched tokens per iteration')
228
parser.add_argument('--max-num-sequences', type=int, default=256, help='maximum number of sequences per iteration')
221
- parser.add_argument('--use-dummy-weights', action='store_true', help='use dummy values for model weights')
229
return parser
230
231
+
232
def process_server_arguments(args: argparse.Namespace):
233
if args.pipeline_parallel_size * args.tensor_parallel_size > 1:
234
args.use_ray = True
235
return args
236
+
237
+
238
+def init_local_server_and_frontend_with_arguments(args: argparse.Namespace):
239
+ # TODO(zhuohan): Support pipeline parallelism.
240
+ assert args.pipeline_parallel_size == 1, (
241
+ 'Pipeline parallelism is not supported yet.')
242
+
243
+ (num_nodes, num_devices_per_node, distributed_init_method,
244
+ all_stage_devices) = (
245
+ initialize_cluster(
246
+ use_ray=args.use_ray,
247
+ pipeline_parallel_size=args.pipeline_parallel_size,
248
+ tensor_parallel_size=args.tensor_parallel_size))
249
+
250
+ # Create a server.
251
+ server = Server(
252
+ model=args.model,
253
+ cache_dir=args.cache_dir,
254
+ use_dummy_weights=args.use_dummy_weights,
255
+ use_np_cache=args.use_np_cache,
256
+ pipeline_parallel_size=args.pipeline_parallel_size,
257
+ tensor_parallel_size=args.tensor_parallel_size,
258
+ block_size=args.block_size,
259
+ dtype=args.dtype,
260
+ seed=args.seed,
261
+ swap_space=args.swap_space,
262
+ max_num_batched_tokens=args.max_num_batched_tokens,
263
+ max_num_sequences=args.max_num_sequences,
264
+ num_nodes=num_nodes,
265
+ num_devices_per_node=num_devices_per_node,
266
+ distributed_init_method=distributed_init_method,
267
+ all_stage_devices=all_stage_devices,
268
+ gpu_memory=get_gpu_memory(),
269
+ cpu_memory=get_cpu_memory(),
270
+ use_ray=args.use_ray,
271
+ )
272
+
273
+ # Create a frontend.
274
+ frontend = SimpleFrontend(
275
+ model_name=args.model,
276
+ block_size=args.block_size,
277
+ )
278
+ return server, frontend
cacheflow/models/gpt_neox.py
+15
-59
@@ -1,18 +1,14 @@
1
"""1D GPT-NeoX model compatible with HuggingFace weights."""
2
-import os
3
-import glob
4
-import filelock
5
-from tqdm import tqdm
2
from typing import Dict, List, Optional, Tuple
3
8
-import numpy as np
4
import torch
5
from torch import nn
11
-from huggingface_hub import snapshot_download
6
7
from cacheflow.models import InputMetadata
8
from cacheflow.models.attention import GPTNeoXCacheFlowAttention
9
from cacheflow.models.sample import Sampler
10
+from cacheflow.models.utils import (hf_model_weights_iterator,
11
+ load_tensor_parallel_weights)
12
from cacheflow.parallel_utils.parallel_state import (
13
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
14
from cacheflow.parallel_utils.tensor_parallel import (VocabParallelEmbedding,
@@ -196,17 +192,22 @@ class GPTNeoXForCausalLM(nn.Module):
192
_column_parallel_weights = ["embed_in.weight", "embed_out.weight", "dense_h_to_4h.weight", "dense_h_to_4h.bias"]
193
_row_parallel_weights = ["dense.weight", "dense_4h_to_h.weight"]
194
199
- def load_weights(self, weights_path: str):
195
+ def load_weights(self, model_name_or_path: str,
196
+ cache_dir: Optional[str] = None,
197
+ use_np_cache: bool = False):
198
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
199
state_dict = self.state_dict()
202
- for name, param in state_dict.items():
200
+ for name, loaded_weight in hf_model_weights_iterator(
201
+ model_name_or_path, cache_dir, use_np_cache):
202
+ if ("attention.bias" in name or "attention.masked_bias" in name
203
+ or "rotary_emb.inv_freq" in name):
204
+ continue
205
+ param = state_dict[name]
206
if "query_key_value" in name:
207
# NOTE(woosuk): GPT-NeoX's fused QKV has the shape of
208
# [num_heads * 3 * head_size, num_heads * head_size], while the
209
# required shape is [3 * num_heads * head_size, num_heads * head_size].
210
# Thus, we need weight conversion.
208
- loaded_weight = torch.from_numpy(
209
- np.load(os.path.join(weights_path, name)))
211
shard_size = param.shape[0]
212
loaded_weight = loaded_weight[shard_size * tensor_model_parallel_rank
213
:shard_size * (tensor_model_parallel_rank + 1)]
@@ -223,55 +224,10 @@ class GPTNeoXForCausalLM(nn.Module):
224
loaded_weight = loaded_weight.transpose(0, 1)
225
loaded_weight = loaded_weight.reshape(-1).contiguous()
226
else:
226
- assert False
227
- else:
228
- loaded_weight = torch.from_numpy(
229
- np.load(os.path.join(weights_path, name)))
230
- for p in self._column_parallel_weights:
231
- if p in name:
232
- shard_size = param.shape[0]
233
- loaded_weight = loaded_weight[
234
- shard_size * tensor_model_parallel_rank
235
- :shard_size * (tensor_model_parallel_rank + 1)]
236
- break
237
- for p in self._row_parallel_weights:
238
- if p in name:
239
- shard_size = param.shape[1]
240
- loaded_weight = loaded_weight[
241
- :,
242
- shard_size * tensor_model_parallel_rank
243
- :shard_size * (tensor_model_parallel_rank + 1)]
244
- break
245
-
246
- assert param.shape == loaded_weight.shape
247
- param.data.copy_(loaded_weight)
248
-
249
- @staticmethod
250
- def get_weights(model_name: str, path: str):
251
- path = os.path.join(path, f"{model_name}-np")
252
- path = os.path.abspath(os.path.expanduser(path))
253
- os.makedirs(path, exist_ok=True)
254
- lock_path = os.path.join(path, "file_lock")
255
- lock = filelock.FileLock(lock_path)
256
-
257
- with lock:
258
- test_weight_path = os.path.join(
259
- path, "gpt_neox.embed_in.weight")
260
- if os.path.exists(test_weight_path):
261
- return path
262
-
263
- folder = snapshot_download(model_name, allow_patterns="*.bin",
264
- cache_dir=os.path.join(path, "cache"))
265
- bin_files = glob.glob(os.path.join(folder, "*.bin"))
266
-
267
- for bin_file in tqdm(bin_files, desc="Convert format"):
268
- state = torch.load(bin_file, map_location="cpu")
269
- for name, param in tqdm(state.items(), leave=False):
270
- param_path = os.path.join(path, name)
271
- with open(param_path, "wb") as f:
272
- np.save(f, param.cpu().detach().numpy())
273
-
274
- return path
227
+ raise ValueError(f"Unexpected weight name: {name}")
228
+ load_tensor_parallel_weights(param, loaded_weight, name,
229
+ self._column_parallel_weights,
230
+ self._row_parallel_weights)
231
232
def initialize_dummy_weights(self) -> None:
233
for param in self.state_dict().values():
cacheflow/models/llama.py
+51
-73
@@ -1,11 +1,6 @@
1
"""1D LLaMA model compatible with HuggingFace weights."""
2
-import os
3
-import glob
4
-import filelock
5
-from tqdm import tqdm
2
from typing import Dict, List, Optional, Tuple
3
8
-import numpy as np
4
import torch
5
from torch import nn
6
from transformers import LlamaConfig
@@ -15,6 +10,8 @@ from cacheflow.models.activation import SiluAndMul
10
from cacheflow.models.attention import LlamaCacheFlowAttention
11
from cacheflow.models.layernorm import RMSNorm
12
from cacheflow.models.sample import Sampler
13
+from cacheflow.models.utils import (hf_model_weights_iterator,
14
+ load_tensor_parallel_weights)
15
from cacheflow.parallel_utils.parallel_state import (
16
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
17
from cacheflow.parallel_utils.tensor_parallel import (VocabParallelEmbedding,
@@ -216,76 +213,57 @@ class LlamaForCausalLM(nn.Module):
213
"up_proj.weight"]
214
_row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
215
219
- def load_weights(self, weights_path: str):
216
+ def load_weights(self, model_name_or_path: str,
217
+ cache_dir: Optional[str] = None,
218
+ use_np_cache: bool = False):
219
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
220
state_dict = self.state_dict()
222
- for name, param in state_dict.items():
223
- if "qkv_proj" in name or "gate_up_proj" in name:
224
- if "qkv_proj" in name:
225
- original_name = "qkv_proj"
226
- weight_names = ["q_proj", "k_proj", "v_proj"]
227
- shard_size = param.shape[0] // 3
228
- else:
229
- original_name = "gate_up_proj"
230
- weight_names = ["gate_proj", "up_proj"]
231
- shard_size = param.shape[0] // 2
232
- weights_to_concat = []
233
- for weight_name in weight_names:
234
- weight = np.load(os.path.join(
235
- weights_path, name.replace(original_name, weight_name)))
236
- weights_to_concat.append(weight[
237
- shard_size * tensor_model_parallel_rank
238
- :shard_size * (tensor_model_parallel_rank + 1)])
239
- loaded_weight = torch.from_numpy(
240
- np.concatenate(weights_to_concat, axis=0))
241
- else:
242
- loaded_weight = torch.from_numpy(
243
- np.load(os.path.join(weights_path, name)))
244
- for p in self._column_parallel_weights:
245
- if p in name:
246
- shard_size = param.shape[0]
247
- loaded_weight = loaded_weight[
248
- shard_size * tensor_model_parallel_rank
249
- :shard_size * (tensor_model_parallel_rank + 1)]
250
- break
251
- for p in self._row_parallel_weights:
252
- if p in name:
253
- shard_size = param.shape[1]
254
- loaded_weight = loaded_weight[
255
- :,
256
- shard_size * tensor_model_parallel_rank
257
- :shard_size * (tensor_model_parallel_rank + 1)]
258
- break
259
-
260
- assert param.shape == loaded_weight.shape
261
- param.data.copy_(loaded_weight)
262
-
263
- @staticmethod
264
- def get_weights(model_name: str, path: str):
265
- if not os.path.isfile(os.path.join(model_name, "config.json")):
266
- raise ValueError("LLaMA model's model_name has to be a path"
267
- "to the huggingface model's directory.")
268
- path = os.path.join(model_name, f"np")
269
- path = os.path.abspath(os.path.expanduser(path))
270
- os.makedirs(path, exist_ok=True)
271
- lock_path = os.path.join(path, "file_lock")
272
- lock = filelock.FileLock(lock_path)
273
-
274
- with lock:
275
- test_weight_path = os.path.join(path, "model.embed_tokens.weight")
276
- if os.path.exists(test_weight_path):
277
- return path
278
-
279
- bin_files = glob.glob(os.path.join(model_name, "*.bin"))
280
-
281
- for bin_file in tqdm(bin_files, desc="Convert format"):
282
- state = torch.load(bin_file, map_location="cpu")
283
- for name, param in tqdm(state.items(), leave=False):
284
- param_path = os.path.join(path, name)
285
- with open(param_path, "wb") as f:
286
- np.save(f, param.cpu().detach().numpy())
287
-
288
- return path
221
+
222
+ for name, loaded_weight in hf_model_weights_iterator(
223
+ model_name_or_path, cache_dir, use_np_cache):
224
+ if "rotary_emb.inv_freq" in name:
225
+ continue
226
+
227
+ is_attention_weight = False
228
+ for stride_id, att_weight_name in enumerate(["q_proj", "k_proj", "v_proj"]):
229
+ if att_weight_name not in name:
230
+ continue
231
+ param = state_dict[name.replace(att_weight_name, "qkv_proj")]
232
+ shard_size = param.shape[0] // 3
233
+ loaded_weight = loaded_weight[
234
+ shard_size * tensor_model_parallel_rank
235
+ :shard_size * (tensor_model_parallel_rank + 1)]
236
+ param_slice = param.data[shard_size * stride_id
237
+ :shard_size * (stride_id + 1)]
238
+ assert param_slice.shape == loaded_weight.shape
239
+ param_slice.copy_(loaded_weight)
240
+ is_attention_weight = True
241
+ break
242
+ if is_attention_weight:
243
+ continue
244
+
245
+ is_gate_up_weight = False
246
+ for stride_id, weight_name in enumerate(["gate_proj", "up_proj"]):
247
+ if weight_name not in name:
248
+ continue
249
+ param = state_dict[name.replace(weight_name, "gate_up_proj")]
250
+ shard_size = param.shape[0] // 2
251
+ loaded_weight = loaded_weight[
252
+ shard_size * tensor_model_parallel_rank
253
+ :shard_size * (tensor_model_parallel_rank + 1)]
254
+ param_slice = param.data[shard_size * stride_id
255
+ :shard_size * (stride_id + 1)]
256
+ assert param_slice.shape == loaded_weight.shape
257
+ param_slice.copy_(loaded_weight)
258
+ is_gate_up_weight = True
259
+ break
260
+ if is_gate_up_weight:
261
+ continue
262
+
263
+ param = state_dict[name]
264
+ load_tensor_parallel_weights(param, loaded_weight, name,
265
+ self._column_parallel_weights,
266
+ self._row_parallel_weights)
267
268
def initialize_dummy_weights(self) -> None:
269
for param in self.state_dict().values():
cacheflow/models/model_utils.py
+5
-6
@@ -1,4 +1,4 @@
1
-from typing import Union
1
+from typing import Union, Optional
2
3
import torch
4
import torch.nn as nn
@@ -32,8 +32,9 @@ _MEMORY_ANALYZERS = {
32
def get_model(
33
model_name: str,
34
dtype: Union[torch.dtype, str],
35
- path: str,
35
+ cache_dir: Optional[str],
36
use_dummy_weights: bool,
37
+ use_np_cache: bool,
38
) -> nn.Module:
39
torch_dtype = get_torch_dtype(dtype)
40
torch.set_default_dtype(torch_dtype)
@@ -46,15 +47,13 @@ def get_model(
47
model = model_class(config)
48
model = model.cuda()
49
# NOTE(woosuk): For precise performance evaluation, we assign
49
- # random values to the weights.
50
+ # random values to the weights.
51
model.initialize_dummy_weights()
52
else:
52
- # Download model weights if it's not cached.
53
- weights_dir = model_class.get_weights(model_name, path=path)
53
# Create a model instance.
54
model = model_class(config)
55
# Load the weights from the cached or downloaded files.
57
- model.load_weights(weights_dir)
56
+ model.load_weights(model_name, cache_dir, use_np_cache)
57
model = model.cuda()
58
return model.eval(), torch_dtype
59
raise ValueError(f'Unsupported model name: {model_name}')
cacheflow/models/opt.py
+34
-69
@@ -1,19 +1,15 @@
1
"""1D OPT model compatible with HuggingFace weights."""
2
-import os
3
-import glob
4
-import filelock
5
-from tqdm import tqdm
2
from typing import Dict, List, Optional, Tuple
3
8
-import numpy as np
4
import torch
5
from torch import nn
6
from transformers import OPTConfig
12
-from huggingface_hub import snapshot_download
7
8
from cacheflow.models import InputMetadata
9
from cacheflow.models.attention import OPTCacheFlowAttention
10
from cacheflow.models.sample import Sampler
11
+from cacheflow.models.utils import (hf_model_weights_iterator,
12
+ load_tensor_parallel_weights)
13
from cacheflow.parallel_utils.parallel_state import (
14
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
15
from cacheflow.parallel_utils.tensor_parallel import (VocabParallelEmbedding,
@@ -257,73 +253,42 @@ class OPTForCausalLM(nn.Module):
253
_column_parallel_weights = ["embed_tokens.weight", "fc1.weight", "fc1.bias"]
254
_row_parallel_weights = ["out_proj.weight", "fc2.weight"]
255
260
- def load_weights(self, weights_path: str):
256
+ def load_weights(self, model_name_or_path: str,
257
+ cache_dir: Optional[str] = None,
258
+ use_np_cache: bool = False):
259
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
260
state_dict = self.state_dict()
263
- for name, param in state_dict.items():
264
- if "lm_head_weight" in name:
261
+
262
+ for name, loaded_weight in hf_model_weights_iterator(
263
+ model_name_or_path, cache_dir, use_np_cache):
264
+ if "lm_head.weight" in name:
265
continue
266
- if "qkv_proj" in name:
266
+
267
+ if name.startswith("decoder."):
268
+ name = "model." + name
269
+
270
+ is_attention_weight = False
271
+ for stride_id, att_weight_name in enumerate(["q_proj", "k_proj", "v_proj"]):
272
+ if att_weight_name not in name:
273
+ continue
274
+ param = state_dict[name.replace(att_weight_name, "qkv_proj")]
275
shard_size = param.shape[0] // 3
268
- weights_to_concat = []
269
- for weight_name in ["q_proj", "k_proj", "v_proj"]:
270
- weight = np.load(os.path.join(
271
- weights_path, name.replace("qkv_proj", weight_name)))
272
- weights_to_concat.append(weight[
273
- shard_size * tensor_model_parallel_rank
274
- :shard_size * (tensor_model_parallel_rank + 1)])
275
- loaded_weight = torch.from_numpy(
276
- np.concatenate(weights_to_concat, axis=0))
277
- else:
278
- loaded_weight = torch.from_numpy(
279
- np.load(os.path.join(weights_path, name)))
280
- for p in self._column_parallel_weights:
281
- if p in name:
282
- shard_size = param.shape[0]
283
- loaded_weight = loaded_weight[
284
- shard_size * tensor_model_parallel_rank
285
- :shard_size * (tensor_model_parallel_rank + 1)]
286
- break
287
- for p in self._row_parallel_weights:
288
- if p in name:
289
- shard_size = param.shape[1]
290
- loaded_weight = loaded_weight[
291
- :,
292
- shard_size * tensor_model_parallel_rank
293
- :shard_size * (tensor_model_parallel_rank + 1)]
294
- break
295
-
296
- assert param.shape == loaded_weight.shape
297
- param.data.copy_(loaded_weight)
298
-
299
- @staticmethod
300
- def get_weights(model_name: str, path: str):
301
- path = os.path.join(path, f"{model_name}-np")
302
- path = os.path.abspath(os.path.expanduser(path))
303
- os.makedirs(path, exist_ok=True)
304
- lock_path = os.path.join(path, "file_lock")
305
- lock = filelock.FileLock(lock_path)
306
-
307
- with lock:
308
- test_weight_path = os.path.join(
309
- path, "model.decoder.embed_positions.weight")
310
- if os.path.exists(test_weight_path):
311
- return path
312
-
313
- folder = snapshot_download(model_name, allow_patterns="*.bin",
314
- cache_dir=os.path.join(path, "cache"))
315
- bin_files = glob.glob(os.path.join(folder, "*.bin"))
316
-
317
- for bin_file in tqdm(bin_files, desc="Convert format"):
318
- state = torch.load(bin_file, map_location="cpu")
319
- for name, param in tqdm(state.items(), leave=False):
320
- if name.startswith("decoder."):
321
- name = "model." + name
322
- param_path = os.path.join(path, name)
323
- with open(param_path, "wb") as f:
324
- np.save(f, param.cpu().detach().numpy())
325
-
326
- return path
276
+ loaded_weight = loaded_weight[
277
+ shard_size * tensor_model_parallel_rank
278
+ :shard_size * (tensor_model_parallel_rank + 1)]
279
+ param_slice = param.data[shard_size * stride_id
280
+ :shard_size * (stride_id + 1)]
281
+ assert param_slice.shape == loaded_weight.shape
282
+ param_slice.copy_(loaded_weight)
283
+ is_attention_weight = True
284
+ break
285
+ if is_attention_weight:
286
+ continue
287
+
288
+ param = state_dict[name]
289
+ load_tensor_parallel_weights(param, loaded_weight, name,
290
+ self._column_parallel_weights,
291
+ self._row_parallel_weights)
292
293
def initialize_dummy_weights(self) -> None:
294
for param in self.state_dict().values():
cacheflow/models/utils.py
+94
-1
@@ -1,6 +1,16 @@
1
-from typing import Union
1
+import os
2
+import glob
3
+import json
4
+import filelock
5
+from typing import Union, Optional
6
7
+import numpy as np
8
import torch
9
+from tqdm.auto import tqdm
10
+from huggingface_hub import snapshot_download
11
+from cacheflow.parallel_utils.parallel_state import (
12
+ get_tensor_model_parallel_rank)
13
+
14
15
_STR_DTYPE_TO_TORCH_DTYPE = {
16
'half': torch.half,
@@ -22,3 +32,86 @@ def get_dtype_size(dtype: Union[torch.dtype, str]) -> int:
32
torch_dtype = get_torch_dtype(dtype)
33
return torch.tensor([], dtype=torch_dtype).element_size()
34
35
+
36
+class Disabledtqdm(tqdm):
37
+ def __init__(self, *args, **kwargs):
38
+ super().__init__(*args, **kwargs, disable=True)
39
+
40
+
41
+def hf_model_weights_iterator(model_name_or_path: str,
42
+ cache_dir: Optional[str] = None,
43
+ use_np_cache: bool = False):
44
+ # Prepare file lock directory to prevent multiple processes from
45
+ # downloading the same model weights at the same time.
46
+ lock_dir = cache_dir if cache_dir is not None else "/tmp"
47
+ lock_file_name = model_name_or_path.replace("/", "-") + ".lock"
48
+ lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name))
49
+
50
+ # Download model weights from huggingface.
51
+ is_local = os.path.isdir(model_name_or_path)
52
+ if not is_local:
53
+ with lock:
54
+ hf_folder = snapshot_download(model_name_or_path,
55
+ allow_patterns="*.bin",
56
+ cache_dir=cache_dir,
57
+ tqdm_class=Disabledtqdm)
58
+ else:
59
+ hf_folder = model_name_or_path
60
+
61
+ hf_bin_files = glob.glob(os.path.join(hf_folder, "*.bin"))
62
+
63
+ if use_np_cache:
64
+ # Convert the model weights from torch tensors to numpy arrays for
65
+ # faster loading.
66
+ np_folder = os.path.join(hf_folder, 'np')
67
+ os.makedirs(np_folder, exist_ok=True)
68
+ weight_names_file = os.path.join(np_folder, 'weight_names.json')
69
+ with lock:
70
+ if not os.path.exists(weight_names_file):
71
+ weight_names = []
72
+ for bin_file in hf_bin_files:
73
+ state = torch.load(bin_file, map_location="cpu")
74
+ for name, param in state.items():
75
+ param_path = os.path.join(np_folder, name)
76
+ with open(param_path, "wb") as f:
77
+ np.save(f, param.cpu().detach().numpy())
78
+ weight_names.append(name)
79
+ with open(weight_names_file, 'w') as f:
80
+ json.dump(weight_names, f)
81
+
82
+ with open(weight_names_file, 'r') as f:
83
+ weight_names = json.load(f)
84
+
85
+ for name in weight_names:
86
+ param_path = os.path.join(np_folder, name)
87
+ with open(param_path, "rb") as f:
88
+ param = np.load(f)
89
+ yield name, torch.from_numpy(param)
90
+ else:
91
+ for bin_file in hf_bin_files:
92
+ state = torch.load(bin_file, map_location="cpu")
93
+ for name, param in state.items():
94
+ yield name, param
95
+
96
+
97
+def load_tensor_parallel_weights(param, loaded_weight, param_name,
98
+ column_parallel_weight_names,
99
+ row_parallel_weight_names):
100
+ tensor_model_parallel_rank = get_tensor_model_parallel_rank()
101
+ for p in column_parallel_weight_names:
102
+ if p in param_name:
103
+ shard_size = param.shape[0]
104
+ loaded_weight = loaded_weight[
105
+ shard_size * tensor_model_parallel_rank
106
+ :shard_size * (tensor_model_parallel_rank + 1)]
107
+ break
108
+ for p in row_parallel_weight_names:
109
+ if p in param_name:
110
+ shard_size = param.shape[1]
111
+ loaded_weight = loaded_weight[
112
+ :,
113
+ shard_size * tensor_model_parallel_rank
114
+ :shard_size * (tensor_model_parallel_rank + 1)]
115
+ break
116
+ assert param.shape == loaded_weight.shape
117
+ param.data.copy_(loaded_weight)
cacheflow/worker/controller.py
+5
-3
@@ -1,4 +1,4 @@
1
-from typing import Dict, List, Union, Tuple
1
+from typing import Dict, List, Union, Tuple, Optional
2
3
try:
4
import ray
@@ -29,8 +29,9 @@ class Controller:
29
num_cpu_blocks: int,
30
dtype: str,
31
seed: int,
32
- model_path: str,
32
+ cache_dir: Optional[str],
33
use_dummy_weights: bool,
34
+ use_np_cache: bool,
35
max_num_batched_tokens: int,
36
use_ray: bool,
37
) -> None:
@@ -66,8 +67,9 @@ class Controller:
67
world_size=world_size,
68
tensor_parallel_size=tensor_parallel_size,
69
pipeline_parallel_size=pipeline_parallel_size,
69
- model_path=model_path,
70
+ cache_dir=cache_dir,
71
use_dummy_weights=use_dummy_weights,
72
+ use_np_cache=use_np_cache,
73
max_num_batched_tokens=max_num_batched_tokens,
74
)
75
self.workers.append(worker)
cacheflow/worker/worker.py
+5
-3
@@ -1,4 +1,4 @@
1
-from typing import Dict, List, Tuple
1
+from typing import Dict, List, Tuple, Optional
2
3
import torch
4
@@ -28,8 +28,9 @@ class Worker:
28
distributed_init_method: str,
29
rank: int,
30
world_size: int,
31
- model_path: str,
31
+ cache_dir: Optional[str],
32
use_dummy_weights: bool,
33
+ use_np_cache: bool,
34
max_num_batched_tokens: int,
35
tensor_parallel_size: int = 1,
36
pipeline_parallel_size: int = 1,
@@ -45,7 +46,8 @@ class Worker:
46
47
# Initialize the model.
48
self.model, self.dtype = get_model(
48
- model_name, dtype=dtype, path=model_path, use_dummy_weights=use_dummy_weights)
49
+ model_name, dtype=dtype, cache_dir=cache_dir,
50
+ use_dummy_weights=use_dummy_weights, use_np_cache=use_np_cache)
51
tensor_model_parallel_world_size = (
52
get_tensor_model_parallel_world_size())
53
initialize_all_reduce_launcher(
simple_server.py
+4
-44
@@ -1,53 +1,13 @@
1
import argparse
2
from typing import List
3
4
-from cacheflow.master.simple_frontend import SimpleFrontend
5
-from cacheflow.master.server import (Server, add_server_arguments,
6
- process_server_arguments,
7
- initialize_cluster)
4
+from cacheflow.master.server import (
5
+ add_server_arguments, process_server_arguments,
6
+ init_local_server_and_frontend_with_arguments)
7
from cacheflow.sampling_params import SamplingParams
9
-from cacheflow.utils import get_gpu_memory, get_cpu_memory
8
9
def main(args: argparse.Namespace):
12
- # TODO(zhuohan): Support pipeline parallelism.
13
- assert args.pipeline_parallel_size == 1, (
14
- 'Pipeline parallelism is not supported yet.')
15
-
16
- (num_nodes, num_devices_per_node, distributed_init_method,
17
- all_stage_devices) = (
18
- initialize_cluster(
19
- use_ray=args.use_ray,
20
- pipeline_parallel_size=args.pipeline_parallel_size,
21
- tensor_parallel_size=args.tensor_parallel_size))
22
-
23
- # Create a server.
24
- server = Server(
25
- model=args.model,
26
- model_path=args.model_path,
27
- use_dummy_weights=args.use_dummy_weights,
28
- pipeline_parallel_size=args.pipeline_parallel_size,
29
- tensor_parallel_size=args.tensor_parallel_size,
30
- block_size=args.block_size,
31
- dtype=args.dtype,
32
- seed=args.seed,
33
- swap_space=args.swap_space,
34
- max_num_batched_tokens=args.max_num_batched_tokens,
35
- max_num_sequences=args.max_num_sequences,
36
- num_nodes=num_nodes,
37
- num_devices_per_node=num_devices_per_node,
38
- distributed_init_method=distributed_init_method,
39
- all_stage_devices=all_stage_devices,
40
- gpu_memory=get_gpu_memory(),
41
- cpu_memory=get_cpu_memory(),
42
- use_ray=args.use_ray,
43
- )
44
-
45
- # Create a frontend.
46
- frontend = SimpleFrontend(
47
- model_name=args.model,
48
- block_size=args.block_size,
49
- )
50
-
10
+ server, frontend = init_local_server_and_frontend_with_arguments(args)
11
# Test the following inputs.
12
test_inputs = [
13
('Ion Stoica is a', {'n': 4, 'use_beam_search': True, 'temperature': 0.0}),