A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add an option to use dummy model weights (#33)
Woosuk Kwon committed
Apr 8, 2023 at 23:36 UTC
ee88a7e5f3acc1f81c52dfc45d2bdd542b4cd9ed
9 files changed
+36
-8
benchmark/benchmark_latency.py
+1
@@ -29,6 +29,7 @@ def main(args: argparse.Namespace):
29
server = Server(
30
model=args.model,
31
model_path=args.model_path,
32
+ use_dummy_weights=args.use_dummy_weights,
33
pipeline_parallel_size=args.pipeline_parallel_size,
34
tensor_parallel_size=args.tensor_parallel_size,
35
block_size=args.block_size,
cacheflow/http_frontend/fastapi_frontend.py
+1
@@ -47,6 +47,7 @@ class FastAPIFrontend:
47
self.server = remote_server_class.remote(
48
model=model,
49
model_path=model_path,
50
+ use_dummy_weights=False,
51
pipeline_parallel_size=pipeline_parallel_size,
52
tensor_parallel_size=tensor_parallel_size,
53
block_size=block_size,
cacheflow/master/server.py
+3
@@ -16,6 +16,7 @@ class Server:
16
self,
17
model: str,
18
model_path: str,
19
+ use_dummy_weights: bool,
20
pipeline_parallel_size: int,
21
tensor_parallel_size: int,
22
block_size: int,
@@ -66,6 +67,7 @@ class Server:
67
dtype=dtype,
68
seed=seed,
69
model_path=model_path,
70
+ use_dummy_weights=use_dummy_weights,
71
max_num_batched_tokens=max_num_batched_tokens,
72
)
73
self.controllers.append(controller)
@@ -179,4 +181,5 @@ def add_server_arguments(parser: argparse.ArgumentParser):
181
parser.add_argument('--seed', type=int, default=0, help='random seed')
182
parser.add_argument('--swap-space', type=int, default=20, help='CPU swap space size (GiB) per GPU')
183
parser.add_argument('--max-num-batched-tokens', type=int, default=2560, help='maximum number of batched tokens')
184
+ parser.add_argument('--use-dummy-weights', action='store_true', help='use dummy values for model weights')
185
return parser
cacheflow/models/llama.py
+4
@@ -286,3 +286,7 @@ class LlamaForCausalLM(nn.Module):
286
np.save(f, param.cpu().detach().numpy())
287
288
return path
289
+
290
+ def initialize_dummy_weights(self) -> None:
291
+ for param in self.state_dict().values():
292
+ param.data.uniform_(-0.1, 0.1)
cacheflow/models/model_utils.py
+17
-6
@@ -28,18 +28,29 @@ def get_model(
28
model_name: str,
29
dtype: Union[torch.dtype, str],
30
path: str,
31
+ use_dummy_weights: bool,
32
) -> nn.Module:
33
torch_dtype = get_torch_dtype(dtype)
34
torch.set_default_dtype(torch_dtype)
35
config = AutoConfig.from_pretrained(model_name)
36
for model_class_name, model_class in _MODELS.items():
37
if model_class_name in model_name:
37
- # Download model weights if it's not cached.
38
- weights_dir = model_class.get_weights(model_name, path=path)
39
- # Create a model instance.
40
- model = model_class(config)
41
- # Load the weights from the cached or downloaded files.
42
- model.load_weights(weights_dir)
38
+ if use_dummy_weights:
39
+ # Create a model instance.
40
+ # The weights will be initialized as empty tensors.
41
+ model = model_class(config)
42
+ model = model.cuda()
43
+ # NOTE(woosuk): For precise performance evaluation, we assign
44
+ # random values to the weights.
45
+ model.initialize_dummy_weights()
46
+ else:
47
+ # Download model weights if it's not cached.
48
+ weights_dir = model_class.get_weights(model_name, path=path)
49
+ # Create a model instance.
50
+ model = model_class(config)
51
+ # Load the weights from the cached or downloaded files.
52
+ model.load_weights(weights_dir)
53
+ model = model.cuda()
54
return model.eval(), torch_dtype
55
raise ValueError(f'Unsupported model name: {model_name}')
56
cacheflow/models/opt.py
+4
@@ -324,3 +324,7 @@ class OPTForCausalLM(nn.Module):
324
np.save(f, param.cpu().detach().numpy())
325
326
return path
327
+
328
+ def initialize_dummy_weights(self) -> None:
329
+ for param in self.state_dict().values():
330
+ param.data.uniform_(-0.1, 0.1)
cacheflow/worker/controller.py
+2
@@ -27,6 +27,7 @@ class Controller:
27
dtype: str,
28
seed: int,
29
model_path: str,
30
+ use_dummy_weights: bool,
31
max_num_batched_tokens: int,
32
) -> None:
33
self.stage_id = stage_id
@@ -58,6 +59,7 @@ class Controller:
59
tensor_parallel_size=tensor_parallel_size,
60
pipeline_parallel_size=pipeline_parallel_size,
61
model_path=model_path,
62
+ use_dummy_weights=use_dummy_weights,
63
max_num_batched_tokens=max_num_batched_tokens,
64
)
65
self.workers.append(worker)
cacheflow/worker/worker.py
+3
-2
@@ -29,6 +29,7 @@ class Worker:
29
rank: int,
30
world_size: int,
31
model_path: str,
32
+ use_dummy_weights: bool,
33
max_num_batched_tokens: int,
34
tensor_parallel_size: int = 1,
35
pipeline_parallel_size: int = 1,
@@ -43,8 +44,8 @@ class Worker:
44
set_random_seed(seed)
45
46
# Initialize the model.
46
- self.model, self.dtype = get_model(model_name, dtype=dtype, path=model_path)
47
- self.model = self.model.cuda()
47
+ self.model, self.dtype = get_model(
48
+ model_name, dtype=dtype, path=model_path, use_dummy_weights=use_dummy_weights)
49
tensor_model_parallel_world_size = (
50
get_tensor_model_parallel_world_size())
51
initialize_all_reduce_launcher(
simple_server.py
+1
@@ -22,6 +22,7 @@ def main(args: argparse.Namespace):
22
server = Server(
23
model=args.model,
24
model_path=args.model_path,
25
+ use_dummy_weights=args.use_dummy_weights,
26
pipeline_parallel_size=args.pipeline_parallel_size,
27
tensor_parallel_size=args.tensor_parallel_size,
28
block_size=args.block_size,