@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ee88a7e5f3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add an option to use dummy model weights (#33)

Woosuk Kwon committed Apr 8, 2023 at 23:36 UTC ee88a7e5f3acc1f81c52dfc45d2bdd542b4cd9ed
9 files changed +36 -8
benchmark/benchmark_latency.py
+1
@@ -29,6 +29,7 @@ def main(args: argparse.Namespace):
29 server = Server(
30 model=args.model,
31 model_path=args.model_path,
32 + use_dummy_weights=args.use_dummy_weights,
33 pipeline_parallel_size=args.pipeline_parallel_size,
34 tensor_parallel_size=args.tensor_parallel_size,
35 block_size=args.block_size,
cacheflow/http_frontend/fastapi_frontend.py
+1
@@ -47,6 +47,7 @@ class FastAPIFrontend:
47 self.server = remote_server_class.remote(
48 model=model,
49 model_path=model_path,
50 + use_dummy_weights=False,
51 pipeline_parallel_size=pipeline_parallel_size,
52 tensor_parallel_size=tensor_parallel_size,
53 block_size=block_size,
cacheflow/master/server.py
+3
@@ -16,6 +16,7 @@ class Server:
16 self,
17 model: str,
18 model_path: str,
19 + use_dummy_weights: bool,
20 pipeline_parallel_size: int,
21 tensor_parallel_size: int,
22 block_size: int,
@@ -66,6 +67,7 @@ class Server:
67 dtype=dtype,
68 seed=seed,
69 model_path=model_path,
70 + use_dummy_weights=use_dummy_weights,
71 max_num_batched_tokens=max_num_batched_tokens,
72 )
73 self.controllers.append(controller)
@@ -179,4 +181,5 @@ def add_server_arguments(parser: argparse.ArgumentParser):
181 parser.add_argument('--seed', type=int, default=0, help='random seed')
182 parser.add_argument('--swap-space', type=int, default=20, help='CPU swap space size (GiB) per GPU')
183 parser.add_argument('--max-num-batched-tokens', type=int, default=2560, help='maximum number of batched tokens')
184 + parser.add_argument('--use-dummy-weights', action='store_true', help='use dummy values for model weights')
185 return parser
cacheflow/models/llama.py
+4
@@ -286,3 +286,7 @@ class LlamaForCausalLM(nn.Module):
286 np.save(f, param.cpu().detach().numpy())
287
288 return path
289 +
290 + def initialize_dummy_weights(self) -> None:
291 + for param in self.state_dict().values():
292 + param.data.uniform_(-0.1, 0.1)
cacheflow/models/model_utils.py
+17 -6
@@ -28,18 +28,29 @@ def get_model(
28 model_name: str,
29 dtype: Union[torch.dtype, str],
30 path: str,
31 + use_dummy_weights: bool,
32 ) -> nn.Module:
33 torch_dtype = get_torch_dtype(dtype)
34 torch.set_default_dtype(torch_dtype)
35 config = AutoConfig.from_pretrained(model_name)
36 for model_class_name, model_class in _MODELS.items():
37 if model_class_name in model_name:
37 - # Download model weights if it's not cached.
38 - weights_dir = model_class.get_weights(model_name, path=path)
39 - # Create a model instance.
40 - model = model_class(config)
41 - # Load the weights from the cached or downloaded files.
42 - model.load_weights(weights_dir)
38 + if use_dummy_weights:
39 + # Create a model instance.
40 + # The weights will be initialized as empty tensors.
41 + model = model_class(config)
42 + model = model.cuda()
43 + # NOTE(woosuk): For precise performance evaluation, we assign
44 + # random values to the weights.
45 + model.initialize_dummy_weights()
46 + else:
47 + # Download model weights if it's not cached.
48 + weights_dir = model_class.get_weights(model_name, path=path)
49 + # Create a model instance.
50 + model = model_class(config)
51 + # Load the weights from the cached or downloaded files.
52 + model.load_weights(weights_dir)
53 + model = model.cuda()
54 return model.eval(), torch_dtype
55 raise ValueError(f'Unsupported model name: {model_name}')
56
cacheflow/models/opt.py
+4
@@ -324,3 +324,7 @@ class OPTForCausalLM(nn.Module):
324 np.save(f, param.cpu().detach().numpy())
325
326 return path
327 +
328 + def initialize_dummy_weights(self) -> None:
329 + for param in self.state_dict().values():
330 + param.data.uniform_(-0.1, 0.1)
cacheflow/worker/controller.py
+2
@@ -27,6 +27,7 @@ class Controller:
27 dtype: str,
28 seed: int,
29 model_path: str,
30 + use_dummy_weights: bool,
31 max_num_batched_tokens: int,
32 ) -> None:
33 self.stage_id = stage_id
@@ -58,6 +59,7 @@ class Controller:
59 tensor_parallel_size=tensor_parallel_size,
60 pipeline_parallel_size=pipeline_parallel_size,
61 model_path=model_path,
62 + use_dummy_weights=use_dummy_weights,
63 max_num_batched_tokens=max_num_batched_tokens,
64 )
65 self.workers.append(worker)
cacheflow/worker/worker.py
+3 -2
@@ -29,6 +29,7 @@ class Worker:
29 rank: int,
30 world_size: int,
31 model_path: str,
32 + use_dummy_weights: bool,
33 max_num_batched_tokens: int,
34 tensor_parallel_size: int = 1,
35 pipeline_parallel_size: int = 1,
@@ -43,8 +44,8 @@ class Worker:
44 set_random_seed(seed)
45
46 # Initialize the model.
46 - self.model, self.dtype = get_model(model_name, dtype=dtype, path=model_path)
47 - self.model = self.model.cuda()
47 + self.model, self.dtype = get_model(
48 + model_name, dtype=dtype, path=model_path, use_dummy_weights=use_dummy_weights)
49 tensor_model_parallel_world_size = (
50 get_tensor_model_parallel_world_size())
51 initialize_all_reduce_launcher(
simple_server.py
+1
@@ -22,6 +22,7 @@ def main(args: argparse.Namespace):
22 server = Server(
23 model=args.model,
24 model_path=args.model_path,
25 + use_dummy_weights=args.use_dummy_weights,
26 pipeline_parallel_size=args.pipeline_parallel_size,
27 tensor_parallel_size=args.tensor_parallel_size,
28 block_size=args.block_size,