@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 27f1410d06
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

New weight loader without np copy (#52)

Zhuohan Li committed May 3, 2023 at 15:32 UTC 27f1410d065ceca53a07abd2518082eb25228e4f
12 files changed +289 -357
benchmark/benchmark_latency.py
+4 -42
@@ -6,53 +6,15 @@ from tqdm import tqdm
6 import numpy as np
7 import torch
8
9 -from cacheflow.master.simple_frontend import SimpleFrontend
10 -from cacheflow.master.server import (Server, add_server_arguments,
11 - process_server_arguments,
12 - initialize_cluster)
9 +from cacheflow.master.server import (
10 + add_server_arguments, process_server_arguments,
11 + init_local_server_and_frontend_with_arguments)
12 from cacheflow.sampling_params import SamplingParams
14 -from cacheflow.utils import get_gpu_memory, get_cpu_memory
13
14
15 def main(args: argparse.Namespace):
18 - # TODO(zhuohan): Support pipeline parallelism.
19 - assert args.pipeline_parallel_size == 1, (
20 - 'Pipeline parallelism is not supported yet.')
16 + server, frontend = init_local_server_and_frontend_with_arguments(args)
17
22 - (num_nodes, num_devices_per_node, distributed_init_method,
23 - all_stage_devices) = (
24 - initialize_cluster(
25 - use_ray=args.use_ray,
26 - pipeline_parallel_size=args.pipeline_parallel_size,
27 - tensor_parallel_size=args.tensor_parallel_size))
28 -
29 - # Create a server.
30 - server = Server(
31 - model=args.model,
32 - model_path=args.model_path,
33 - use_dummy_weights=args.use_dummy_weights,
34 - pipeline_parallel_size=args.pipeline_parallel_size,
35 - tensor_parallel_size=args.tensor_parallel_size,
36 - block_size=args.block_size,
37 - dtype=args.dtype,
38 - seed=args.seed,
39 - swap_space=args.swap_space,
40 - max_num_batched_tokens=args.max_num_batched_tokens,
41 - max_num_sequences=args.max_num_sequences,
42 - num_nodes=num_nodes,
43 - num_devices_per_node=num_devices_per_node,
44 - distributed_init_method=distributed_init_method,
45 - all_stage_devices=all_stage_devices,
46 - gpu_memory=get_gpu_memory(),
47 - cpu_memory=get_cpu_memory(),
48 - use_ray=args.use_ray,
49 - )
50 -
51 - # Create a frontend.
52 - frontend = SimpleFrontend(
53 - model_name=args.model,
54 - block_size=args.block_size,
55 - )
18 sampling_params_dict = {
19 'n': args.n,
20 'temperature': 0.0 if args.use_beam_search else 1.0,
benchmark/benchmark_text_completion.py
+4 -43
@@ -9,57 +9,18 @@ from tqdm import tqdm
9 from transformers import AutoConfig
10
11 from benchmark.trace import generate_text_completion_requests
12 -from cacheflow.master.simple_frontend import SimpleFrontend
13 -from cacheflow.master.server import (Server, add_server_arguments,
14 - process_server_arguments,
15 - initialize_cluster)
12 +from cacheflow.master.server import (
13 + add_server_arguments, process_server_arguments,
14 + init_local_server_and_frontend_with_arguments)
15 from cacheflow.sampling_params import SamplingParams
17 -from cacheflow.utils import get_gpu_memory, get_cpu_memory
16
17
18 logger = logging.getLogger(__name__)
19
20
21 def main(args: argparse.Namespace):
24 - assert args.pipeline_parallel_size == 1, (
25 - 'Pipeline parallelism is not supported yet.')
22 + server, frontend = init_local_server_and_frontend_with_arguments(args)
23
27 - (num_nodes, num_devices_per_node, distributed_init_method,
28 - all_stage_devices) = (
29 - initialize_cluster(
30 - use_ray=args.use_ray,
31 - pipeline_parallel_size=args.pipeline_parallel_size,
32 - tensor_parallel_size=args.tensor_parallel_size))
33 -
34 - # Create a server.
35 - server = Server(
36 - model=args.model,
37 - model_path=args.model_path,
38 - use_dummy_weights=args.use_dummy_weights,
39 - pipeline_parallel_size=args.pipeline_parallel_size,
40 - tensor_parallel_size=args.tensor_parallel_size,
41 - block_size=args.block_size,
42 - dtype=args.dtype,
43 - seed=args.seed,
44 - swap_space=args.swap_space,
45 - max_num_batched_tokens=args.max_num_batched_tokens,
46 - max_num_sequences=args.max_num_sequences,
47 - num_nodes=num_nodes,
48 - num_devices_per_node=num_devices_per_node,
49 - distributed_init_method=distributed_init_method,
50 - all_stage_devices=all_stage_devices,
51 - gpu_memory=get_gpu_memory(),
52 - cpu_memory=get_cpu_memory(),
53 - use_ray=args.use_ray,
54 - collect_stats=True,
55 - do_memory_analysis=args.do_memory_analysis,
56 - )
57 -
58 - # Create a frontend.
59 - frontend = SimpleFrontend(
60 - model_name=args.model,
61 - block_size=args.block_size,
62 - )
24 # Generate requests.
25 requests = generate_text_completion_requests(
26 args.dataset,
cacheflow/http_frontend/fastapi_frontend.py
+10 -7
@@ -1,7 +1,7 @@
1 import argparse
2 import asyncio
3 import time
4 -from typing import List, Dict
4 +from typing import List, Dict, Optional
5 import json
6
7 import ray
@@ -22,11 +22,12 @@ TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
22 app = FastAPI()
23
24
25 -class FastAPIFrontend:
25 +class FastAPIServer:
26 def __init__(
27 self,
28 model: str,
29 - model_path: str,
29 + cache_dir: Optional[str],
30 + use_np_cache: bool,
31 pipeline_parallel_size: int,
32 tensor_parallel_size: int,
33 block_size: int,
@@ -52,8 +53,9 @@ class FastAPIFrontend:
53 remote_server_class = ray.remote(num_gpus=1)(Server)
54 self.server = remote_server_class.remote(
55 model=model,
55 - model_path=model_path,
56 + cache_dir=cache_dir,
57 use_dummy_weights=False,
58 + use_np_cache=use_np_cache,
59 pipeline_parallel_size=pipeline_parallel_size,
60 tensor_parallel_size=tensor_parallel_size,
61 block_size=block_size,
@@ -148,7 +150,7 @@ class FastAPIFrontend:
150 @app.post("/generate")
151 async def generate_stream(request: Request):
152 request_dict = await request.json()
151 - return StreamingResponse(frontend.generate(request_dict))
153 + return StreamingResponse(server.generate(request_dict))
154
155
156 if __name__ == "__main__":
@@ -170,9 +172,10 @@ if __name__ == "__main__":
172 pipeline_parallel_size=args.pipeline_parallel_size,
173 tensor_parallel_size=args.tensor_parallel_size))
174
173 - frontend = FastAPIFrontend(
175 + server = FastAPIServer(
176 model=args.model,
175 - model_path=args.model_path,
177 + cache_dir=args.cache_dir,
178 + use_np_cache=args.use_np_cache,
179 pipeline_parallel_size=args.pipeline_parallel_size,
180 tensor_parallel_size=args.tensor_parallel_size,
181 block_size=args.block_size,
cacheflow/master/server.py
+58 -7
@@ -9,18 +9,21 @@ except ImportError:
9 ray = None
10
11 from cacheflow.master.scheduler import Scheduler
12 +from cacheflow.master.simple_frontend import SimpleFrontend
13 from cacheflow.models import get_memory_analyzer
14 from cacheflow.worker.controller import Controller, DeviceID
15 from cacheflow.sequence import SequenceGroup
16 from cacheflow.sampling_params import SamplingParams
17 +from cacheflow.utils import get_gpu_memory, get_cpu_memory
18
19
20 class Server:
21 def __init__(
22 self,
23 model: str,
22 - model_path: str,
24 + cache_dir: Optional[str],
25 use_dummy_weights: bool,
26 + use_np_cache: bool,
27 pipeline_parallel_size: int,
28 tensor_parallel_size: int,
29 block_size: int,
@@ -78,8 +81,9 @@ class Server:
81 num_cpu_blocks=self.num_cpu_blocks,
82 dtype=dtype,
83 seed=seed,
81 - model_path=model_path,
84 + cache_dir=cache_dir,
85 use_dummy_weights=use_dummy_weights,
86 + use_np_cache=use_np_cache,
87 max_num_batched_tokens=max_num_batched_tokens,
88 use_ray=use_ray,
89 )
@@ -203,25 +207,72 @@ def initialize_cluster(
207 def add_server_arguments(parser: argparse.ArgumentParser):
208 # Model arguments
209 parser.add_argument('--model', type=str, default='facebook/opt-125m', help='model name')
206 - parser.add_argument('--model-path', type=str, default='~/.cacheflow/model_weights',
207 - help='model path to download and load the weights')
210 + parser.add_argument('--cache-dir', type=str, default=None,
211 + help='cache dir to download and load the weights, '
212 + 'default to the default cache dir of huggingface')
213 + parser.add_argument('--use-np-cache', action='store_true',
214 + help='save a numpy copy of model weights for faster loading')
215 + parser.add_argument('--use-dummy-weights', action='store_true', help='use dummy values for model weights')
216 + # NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
217 + parser.add_argument('--dtype', type=str, default='half', choices=['half'], help='data type')
218 # Parallel arguments
219 parser.add_argument('--use-ray', action='store_true', help='use Ray for distributed serving, will be automatically set when using more than 1 GPU')
220 parser.add_argument('--pipeline-parallel-size', '-pp', type=int, default=1, help='number of pipeline stages')
221 parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1, help='number of tensor parallel replicas')
222 # KV cache arguments
223 parser.add_argument('--block-size', type=int, default=16, choices=[1, 2, 4, 8, 16, 32, 64, 128, 256], help='token block size')
214 - # NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
215 - parser.add_argument('--dtype', type=str, default='half', choices=['half'], help='data type')
224 # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
225 parser.add_argument('--seed', type=int, default=0, help='random seed')
226 parser.add_argument('--swap-space', type=int, default=20, help='CPU swap space size (GiB) per GPU')
227 parser.add_argument('--max-num-batched-tokens', type=int, default=2560, help='maximum number of batched tokens per iteration')
228 parser.add_argument('--max-num-sequences', type=int, default=256, help='maximum number of sequences per iteration')
221 - parser.add_argument('--use-dummy-weights', action='store_true', help='use dummy values for model weights')
229 return parser
230
231 +
232 def process_server_arguments(args: argparse.Namespace):
233 if args.pipeline_parallel_size * args.tensor_parallel_size > 1:
234 args.use_ray = True
235 return args
236 +
237 +
238 +def init_local_server_and_frontend_with_arguments(args: argparse.Namespace):
239 + # TODO(zhuohan): Support pipeline parallelism.
240 + assert args.pipeline_parallel_size == 1, (
241 + 'Pipeline parallelism is not supported yet.')
242 +
243 + (num_nodes, num_devices_per_node, distributed_init_method,
244 + all_stage_devices) = (
245 + initialize_cluster(
246 + use_ray=args.use_ray,
247 + pipeline_parallel_size=args.pipeline_parallel_size,
248 + tensor_parallel_size=args.tensor_parallel_size))
249 +
250 + # Create a server.
251 + server = Server(
252 + model=args.model,
253 + cache_dir=args.cache_dir,
254 + use_dummy_weights=args.use_dummy_weights,
255 + use_np_cache=args.use_np_cache,
256 + pipeline_parallel_size=args.pipeline_parallel_size,
257 + tensor_parallel_size=args.tensor_parallel_size,
258 + block_size=args.block_size,
259 + dtype=args.dtype,
260 + seed=args.seed,
261 + swap_space=args.swap_space,
262 + max_num_batched_tokens=args.max_num_batched_tokens,
263 + max_num_sequences=args.max_num_sequences,
264 + num_nodes=num_nodes,
265 + num_devices_per_node=num_devices_per_node,
266 + distributed_init_method=distributed_init_method,
267 + all_stage_devices=all_stage_devices,
268 + gpu_memory=get_gpu_memory(),
269 + cpu_memory=get_cpu_memory(),
270 + use_ray=args.use_ray,
271 + )
272 +
273 + # Create a frontend.
274 + frontend = SimpleFrontend(
275 + model_name=args.model,
276 + block_size=args.block_size,
277 + )
278 + return server, frontend
cacheflow/models/gpt_neox.py
+15 -59
@@ -1,18 +1,14 @@
1 """1D GPT-NeoX model compatible with HuggingFace weights."""
2 -import os
3 -import glob
4 -import filelock
5 -from tqdm import tqdm
2 from typing import Dict, List, Optional, Tuple
3
8 -import numpy as np
4 import torch
5 from torch import nn
11 -from huggingface_hub import snapshot_download
6
7 from cacheflow.models import InputMetadata
8 from cacheflow.models.attention import GPTNeoXCacheFlowAttention
9 from cacheflow.models.sample import Sampler
10 +from cacheflow.models.utils import (hf_model_weights_iterator,
11 + load_tensor_parallel_weights)
12 from cacheflow.parallel_utils.parallel_state import (
13 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
14 from cacheflow.parallel_utils.tensor_parallel import (VocabParallelEmbedding,
@@ -196,17 +192,22 @@ class GPTNeoXForCausalLM(nn.Module):
192 _column_parallel_weights = ["embed_in.weight", "embed_out.weight", "dense_h_to_4h.weight", "dense_h_to_4h.bias"]
193 _row_parallel_weights = ["dense.weight", "dense_4h_to_h.weight"]
194
199 - def load_weights(self, weights_path: str):
195 + def load_weights(self, model_name_or_path: str,
196 + cache_dir: Optional[str] = None,
197 + use_np_cache: bool = False):
198 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
199 state_dict = self.state_dict()
202 - for name, param in state_dict.items():
200 + for name, loaded_weight in hf_model_weights_iterator(
201 + model_name_or_path, cache_dir, use_np_cache):
202 + if ("attention.bias" in name or "attention.masked_bias" in name
203 + or "rotary_emb.inv_freq" in name):
204 + continue
205 + param = state_dict[name]
206 if "query_key_value" in name:
207 # NOTE(woosuk): GPT-NeoX's fused QKV has the shape of
208 # [num_heads * 3 * head_size, num_heads * head_size], while the
209 # required shape is [3 * num_heads * head_size, num_heads * head_size].
210 # Thus, we need weight conversion.
208 - loaded_weight = torch.from_numpy(
209 - np.load(os.path.join(weights_path, name)))
211 shard_size = param.shape[0]
212 loaded_weight = loaded_weight[shard_size * tensor_model_parallel_rank
213 :shard_size * (tensor_model_parallel_rank + 1)]
@@ -223,55 +224,10 @@ class GPTNeoXForCausalLM(nn.Module):
224 loaded_weight = loaded_weight.transpose(0, 1)
225 loaded_weight = loaded_weight.reshape(-1).contiguous()
226 else:
226 - assert False
227 - else:
228 - loaded_weight = torch.from_numpy(
229 - np.load(os.path.join(weights_path, name)))
230 - for p in self._column_parallel_weights:
231 - if p in name:
232 - shard_size = param.shape[0]
233 - loaded_weight = loaded_weight[
234 - shard_size * tensor_model_parallel_rank
235 - :shard_size * (tensor_model_parallel_rank + 1)]
236 - break
237 - for p in self._row_parallel_weights:
238 - if p in name:
239 - shard_size = param.shape[1]
240 - loaded_weight = loaded_weight[
241 - :,
242 - shard_size * tensor_model_parallel_rank
243 - :shard_size * (tensor_model_parallel_rank + 1)]
244 - break
245 -
246 - assert param.shape == loaded_weight.shape
247 - param.data.copy_(loaded_weight)
248 -
249 - @staticmethod
250 - def get_weights(model_name: str, path: str):
251 - path = os.path.join(path, f"{model_name}-np")
252 - path = os.path.abspath(os.path.expanduser(path))
253 - os.makedirs(path, exist_ok=True)
254 - lock_path = os.path.join(path, "file_lock")
255 - lock = filelock.FileLock(lock_path)
256 -
257 - with lock:
258 - test_weight_path = os.path.join(
259 - path, "gpt_neox.embed_in.weight")
260 - if os.path.exists(test_weight_path):
261 - return path
262 -
263 - folder = snapshot_download(model_name, allow_patterns="*.bin",
264 - cache_dir=os.path.join(path, "cache"))
265 - bin_files = glob.glob(os.path.join(folder, "*.bin"))
266 -
267 - for bin_file in tqdm(bin_files, desc="Convert format"):
268 - state = torch.load(bin_file, map_location="cpu")
269 - for name, param in tqdm(state.items(), leave=False):
270 - param_path = os.path.join(path, name)
271 - with open(param_path, "wb") as f:
272 - np.save(f, param.cpu().detach().numpy())
273 -
274 - return path
227 + raise ValueError(f"Unexpected weight name: {name}")
228 + load_tensor_parallel_weights(param, loaded_weight, name,
229 + self._column_parallel_weights,
230 + self._row_parallel_weights)
231
232 def initialize_dummy_weights(self) -> None:
233 for param in self.state_dict().values():
cacheflow/models/llama.py
+51 -73
@@ -1,11 +1,6 @@
1 """1D LLaMA model compatible with HuggingFace weights."""
2 -import os
3 -import glob
4 -import filelock
5 -from tqdm import tqdm
2 from typing import Dict, List, Optional, Tuple
3
8 -import numpy as np
4 import torch
5 from torch import nn
6 from transformers import LlamaConfig
@@ -15,6 +10,8 @@ from cacheflow.models.activation import SiluAndMul
10 from cacheflow.models.attention import LlamaCacheFlowAttention
11 from cacheflow.models.layernorm import RMSNorm
12 from cacheflow.models.sample import Sampler
13 +from cacheflow.models.utils import (hf_model_weights_iterator,
14 + load_tensor_parallel_weights)
15 from cacheflow.parallel_utils.parallel_state import (
16 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
17 from cacheflow.parallel_utils.tensor_parallel import (VocabParallelEmbedding,
@@ -216,76 +213,57 @@ class LlamaForCausalLM(nn.Module):
213 "up_proj.weight"]
214 _row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
215
219 - def load_weights(self, weights_path: str):
216 + def load_weights(self, model_name_or_path: str,
217 + cache_dir: Optional[str] = None,
218 + use_np_cache: bool = False):
219 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
220 state_dict = self.state_dict()
222 - for name, param in state_dict.items():
223 - if "qkv_proj" in name or "gate_up_proj" in name:
224 - if "qkv_proj" in name:
225 - original_name = "qkv_proj"
226 - weight_names = ["q_proj", "k_proj", "v_proj"]
227 - shard_size = param.shape[0] // 3
228 - else:
229 - original_name = "gate_up_proj"
230 - weight_names = ["gate_proj", "up_proj"]
231 - shard_size = param.shape[0] // 2
232 - weights_to_concat = []
233 - for weight_name in weight_names:
234 - weight = np.load(os.path.join(
235 - weights_path, name.replace(original_name, weight_name)))
236 - weights_to_concat.append(weight[
237 - shard_size * tensor_model_parallel_rank
238 - :shard_size * (tensor_model_parallel_rank + 1)])
239 - loaded_weight = torch.from_numpy(
240 - np.concatenate(weights_to_concat, axis=0))
241 - else:
242 - loaded_weight = torch.from_numpy(
243 - np.load(os.path.join(weights_path, name)))
244 - for p in self._column_parallel_weights:
245 - if p in name:
246 - shard_size = param.shape[0]
247 - loaded_weight = loaded_weight[
248 - shard_size * tensor_model_parallel_rank
249 - :shard_size * (tensor_model_parallel_rank + 1)]
250 - break
251 - for p in self._row_parallel_weights:
252 - if p in name:
253 - shard_size = param.shape[1]
254 - loaded_weight = loaded_weight[
255 - :,
256 - shard_size * tensor_model_parallel_rank
257 - :shard_size * (tensor_model_parallel_rank + 1)]
258 - break
259 -
260 - assert param.shape == loaded_weight.shape
261 - param.data.copy_(loaded_weight)
262 -
263 - @staticmethod
264 - def get_weights(model_name: str, path: str):
265 - if not os.path.isfile(os.path.join(model_name, "config.json")):
266 - raise ValueError("LLaMA model's model_name has to be a path"
267 - "to the huggingface model's directory.")
268 - path = os.path.join(model_name, f"np")
269 - path = os.path.abspath(os.path.expanduser(path))
270 - os.makedirs(path, exist_ok=True)
271 - lock_path = os.path.join(path, "file_lock")
272 - lock = filelock.FileLock(lock_path)
273 -
274 - with lock:
275 - test_weight_path = os.path.join(path, "model.embed_tokens.weight")
276 - if os.path.exists(test_weight_path):
277 - return path
278 -
279 - bin_files = glob.glob(os.path.join(model_name, "*.bin"))
280 -
281 - for bin_file in tqdm(bin_files, desc="Convert format"):
282 - state = torch.load(bin_file, map_location="cpu")
283 - for name, param in tqdm(state.items(), leave=False):
284 - param_path = os.path.join(path, name)
285 - with open(param_path, "wb") as f:
286 - np.save(f, param.cpu().detach().numpy())
287 -
288 - return path
221 +
222 + for name, loaded_weight in hf_model_weights_iterator(
223 + model_name_or_path, cache_dir, use_np_cache):
224 + if "rotary_emb.inv_freq" in name:
225 + continue
226 +
227 + is_attention_weight = False
228 + for stride_id, att_weight_name in enumerate(["q_proj", "k_proj", "v_proj"]):
229 + if att_weight_name not in name:
230 + continue
231 + param = state_dict[name.replace(att_weight_name, "qkv_proj")]
232 + shard_size = param.shape[0] // 3
233 + loaded_weight = loaded_weight[
234 + shard_size * tensor_model_parallel_rank
235 + :shard_size * (tensor_model_parallel_rank + 1)]
236 + param_slice = param.data[shard_size * stride_id
237 + :shard_size * (stride_id + 1)]
238 + assert param_slice.shape == loaded_weight.shape
239 + param_slice.copy_(loaded_weight)
240 + is_attention_weight = True
241 + break
242 + if is_attention_weight:
243 + continue
244 +
245 + is_gate_up_weight = False
246 + for stride_id, weight_name in enumerate(["gate_proj", "up_proj"]):
247 + if weight_name not in name:
248 + continue
249 + param = state_dict[name.replace(weight_name, "gate_up_proj")]
250 + shard_size = param.shape[0] // 2
251 + loaded_weight = loaded_weight[
252 + shard_size * tensor_model_parallel_rank
253 + :shard_size * (tensor_model_parallel_rank + 1)]
254 + param_slice = param.data[shard_size * stride_id
255 + :shard_size * (stride_id + 1)]
256 + assert param_slice.shape == loaded_weight.shape
257 + param_slice.copy_(loaded_weight)
258 + is_gate_up_weight = True
259 + break
260 + if is_gate_up_weight:
261 + continue
262 +
263 + param = state_dict[name]
264 + load_tensor_parallel_weights(param, loaded_weight, name,
265 + self._column_parallel_weights,
266 + self._row_parallel_weights)
267
268 def initialize_dummy_weights(self) -> None:
269 for param in self.state_dict().values():
cacheflow/models/model_utils.py
+5 -6
@@ -1,4 +1,4 @@
1 -from typing import Union
1 +from typing import Union, Optional
2
3 import torch
4 import torch.nn as nn
@@ -32,8 +32,9 @@ _MEMORY_ANALYZERS = {
32 def get_model(
33 model_name: str,
34 dtype: Union[torch.dtype, str],
35 - path: str,
35 + cache_dir: Optional[str],
36 use_dummy_weights: bool,
37 + use_np_cache: bool,
38 ) -> nn.Module:
39 torch_dtype = get_torch_dtype(dtype)
40 torch.set_default_dtype(torch_dtype)
@@ -46,15 +47,13 @@ def get_model(
47 model = model_class(config)
48 model = model.cuda()
49 # NOTE(woosuk): For precise performance evaluation, we assign
49 - # random values to the weights.
50 + # random values to the weights.
51 model.initialize_dummy_weights()
52 else:
52 - # Download model weights if it's not cached.
53 - weights_dir = model_class.get_weights(model_name, path=path)
53 # Create a model instance.
54 model = model_class(config)
55 # Load the weights from the cached or downloaded files.
57 - model.load_weights(weights_dir)
56 + model.load_weights(model_name, cache_dir, use_np_cache)
57 model = model.cuda()
58 return model.eval(), torch_dtype
59 raise ValueError(f'Unsupported model name: {model_name}')
cacheflow/models/opt.py
+34 -69
@@ -1,19 +1,15 @@
1 """1D OPT model compatible with HuggingFace weights."""
2 -import os
3 -import glob
4 -import filelock
5 -from tqdm import tqdm
2 from typing import Dict, List, Optional, Tuple
3
8 -import numpy as np
4 import torch
5 from torch import nn
6 from transformers import OPTConfig
12 -from huggingface_hub import snapshot_download
7
8 from cacheflow.models import InputMetadata
9 from cacheflow.models.attention import OPTCacheFlowAttention
10 from cacheflow.models.sample import Sampler
11 +from cacheflow.models.utils import (hf_model_weights_iterator,
12 + load_tensor_parallel_weights)
13 from cacheflow.parallel_utils.parallel_state import (
14 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
15 from cacheflow.parallel_utils.tensor_parallel import (VocabParallelEmbedding,
@@ -257,73 +253,42 @@ class OPTForCausalLM(nn.Module):
253 _column_parallel_weights = ["embed_tokens.weight", "fc1.weight", "fc1.bias"]
254 _row_parallel_weights = ["out_proj.weight", "fc2.weight"]
255
260 - def load_weights(self, weights_path: str):
256 + def load_weights(self, model_name_or_path: str,
257 + cache_dir: Optional[str] = None,
258 + use_np_cache: bool = False):
259 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
260 state_dict = self.state_dict()
263 - for name, param in state_dict.items():
264 - if "lm_head_weight" in name:
261 +
262 + for name, loaded_weight in hf_model_weights_iterator(
263 + model_name_or_path, cache_dir, use_np_cache):
264 + if "lm_head.weight" in name:
265 continue
266 - if "qkv_proj" in name:
266 +
267 + if name.startswith("decoder."):
268 + name = "model." + name
269 +
270 + is_attention_weight = False
271 + for stride_id, att_weight_name in enumerate(["q_proj", "k_proj", "v_proj"]):
272 + if att_weight_name not in name:
273 + continue
274 + param = state_dict[name.replace(att_weight_name, "qkv_proj")]
275 shard_size = param.shape[0] // 3
268 - weights_to_concat = []
269 - for weight_name in ["q_proj", "k_proj", "v_proj"]:
270 - weight = np.load(os.path.join(
271 - weights_path, name.replace("qkv_proj", weight_name)))
272 - weights_to_concat.append(weight[
273 - shard_size * tensor_model_parallel_rank
274 - :shard_size * (tensor_model_parallel_rank + 1)])
275 - loaded_weight = torch.from_numpy(
276 - np.concatenate(weights_to_concat, axis=0))
277 - else:
278 - loaded_weight = torch.from_numpy(
279 - np.load(os.path.join(weights_path, name)))
280 - for p in self._column_parallel_weights:
281 - if p in name:
282 - shard_size = param.shape[0]
283 - loaded_weight = loaded_weight[
284 - shard_size * tensor_model_parallel_rank
285 - :shard_size * (tensor_model_parallel_rank + 1)]
286 - break
287 - for p in self._row_parallel_weights:
288 - if p in name:
289 - shard_size = param.shape[1]
290 - loaded_weight = loaded_weight[
291 - :,
292 - shard_size * tensor_model_parallel_rank
293 - :shard_size * (tensor_model_parallel_rank + 1)]
294 - break
295 -
296 - assert param.shape == loaded_weight.shape
297 - param.data.copy_(loaded_weight)
298 -
299 - @staticmethod
300 - def get_weights(model_name: str, path: str):
301 - path = os.path.join(path, f"{model_name}-np")
302 - path = os.path.abspath(os.path.expanduser(path))
303 - os.makedirs(path, exist_ok=True)
304 - lock_path = os.path.join(path, "file_lock")
305 - lock = filelock.FileLock(lock_path)
306 -
307 - with lock:
308 - test_weight_path = os.path.join(
309 - path, "model.decoder.embed_positions.weight")
310 - if os.path.exists(test_weight_path):
311 - return path
312 -
313 - folder = snapshot_download(model_name, allow_patterns="*.bin",
314 - cache_dir=os.path.join(path, "cache"))
315 - bin_files = glob.glob(os.path.join(folder, "*.bin"))
316 -
317 - for bin_file in tqdm(bin_files, desc="Convert format"):
318 - state = torch.load(bin_file, map_location="cpu")
319 - for name, param in tqdm(state.items(), leave=False):
320 - if name.startswith("decoder."):
321 - name = "model." + name
322 - param_path = os.path.join(path, name)
323 - with open(param_path, "wb") as f:
324 - np.save(f, param.cpu().detach().numpy())
325 -
326 - return path
276 + loaded_weight = loaded_weight[
277 + shard_size * tensor_model_parallel_rank
278 + :shard_size * (tensor_model_parallel_rank + 1)]
279 + param_slice = param.data[shard_size * stride_id
280 + :shard_size * (stride_id + 1)]
281 + assert param_slice.shape == loaded_weight.shape
282 + param_slice.copy_(loaded_weight)
283 + is_attention_weight = True
284 + break
285 + if is_attention_weight:
286 + continue
287 +
288 + param = state_dict[name]
289 + load_tensor_parallel_weights(param, loaded_weight, name,
290 + self._column_parallel_weights,
291 + self._row_parallel_weights)
292
293 def initialize_dummy_weights(self) -> None:
294 for param in self.state_dict().values():
cacheflow/models/utils.py
+94 -1
@@ -1,6 +1,16 @@
1 -from typing import Union
1 +import os
2 +import glob
3 +import json
4 +import filelock
5 +from typing import Union, Optional
6
7 +import numpy as np
8 import torch
9 +from tqdm.auto import tqdm
10 +from huggingface_hub import snapshot_download
11 +from cacheflow.parallel_utils.parallel_state import (
12 + get_tensor_model_parallel_rank)
13 +
14
15 _STR_DTYPE_TO_TORCH_DTYPE = {
16 'half': torch.half,
@@ -22,3 +32,86 @@ def get_dtype_size(dtype: Union[torch.dtype, str]) -> int:
32 torch_dtype = get_torch_dtype(dtype)
33 return torch.tensor([], dtype=torch_dtype).element_size()
34
35 +
36 +class Disabledtqdm(tqdm):
37 + def __init__(self, *args, **kwargs):
38 + super().__init__(*args, **kwargs, disable=True)
39 +
40 +
41 +def hf_model_weights_iterator(model_name_or_path: str,
42 + cache_dir: Optional[str] = None,
43 + use_np_cache: bool = False):
44 + # Prepare file lock directory to prevent multiple processes from
45 + # downloading the same model weights at the same time.
46 + lock_dir = cache_dir if cache_dir is not None else "/tmp"
47 + lock_file_name = model_name_or_path.replace("/", "-") + ".lock"
48 + lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name))
49 +
50 + # Download model weights from huggingface.
51 + is_local = os.path.isdir(model_name_or_path)
52 + if not is_local:
53 + with lock:
54 + hf_folder = snapshot_download(model_name_or_path,
55 + allow_patterns="*.bin",
56 + cache_dir=cache_dir,
57 + tqdm_class=Disabledtqdm)
58 + else:
59 + hf_folder = model_name_or_path
60 +
61 + hf_bin_files = glob.glob(os.path.join(hf_folder, "*.bin"))
62 +
63 + if use_np_cache:
64 + # Convert the model weights from torch tensors to numpy arrays for
65 + # faster loading.
66 + np_folder = os.path.join(hf_folder, 'np')
67 + os.makedirs(np_folder, exist_ok=True)
68 + weight_names_file = os.path.join(np_folder, 'weight_names.json')
69 + with lock:
70 + if not os.path.exists(weight_names_file):
71 + weight_names = []
72 + for bin_file in hf_bin_files:
73 + state = torch.load(bin_file, map_location="cpu")
74 + for name, param in state.items():
75 + param_path = os.path.join(np_folder, name)
76 + with open(param_path, "wb") as f:
77 + np.save(f, param.cpu().detach().numpy())
78 + weight_names.append(name)
79 + with open(weight_names_file, 'w') as f:
80 + json.dump(weight_names, f)
81 +
82 + with open(weight_names_file, 'r') as f:
83 + weight_names = json.load(f)
84 +
85 + for name in weight_names:
86 + param_path = os.path.join(np_folder, name)
87 + with open(param_path, "rb") as f:
88 + param = np.load(f)
89 + yield name, torch.from_numpy(param)
90 + else:
91 + for bin_file in hf_bin_files:
92 + state = torch.load(bin_file, map_location="cpu")
93 + for name, param in state.items():
94 + yield name, param
95 +
96 +
97 +def load_tensor_parallel_weights(param, loaded_weight, param_name,
98 + column_parallel_weight_names,
99 + row_parallel_weight_names):
100 + tensor_model_parallel_rank = get_tensor_model_parallel_rank()
101 + for p in column_parallel_weight_names:
102 + if p in param_name:
103 + shard_size = param.shape[0]
104 + loaded_weight = loaded_weight[
105 + shard_size * tensor_model_parallel_rank
106 + :shard_size * (tensor_model_parallel_rank + 1)]
107 + break
108 + for p in row_parallel_weight_names:
109 + if p in param_name:
110 + shard_size = param.shape[1]
111 + loaded_weight = loaded_weight[
112 + :,
113 + shard_size * tensor_model_parallel_rank
114 + :shard_size * (tensor_model_parallel_rank + 1)]
115 + break
116 + assert param.shape == loaded_weight.shape
117 + param.data.copy_(loaded_weight)
cacheflow/worker/controller.py
+5 -3
@@ -1,4 +1,4 @@
1 -from typing import Dict, List, Union, Tuple
1 +from typing import Dict, List, Union, Tuple, Optional
2
3 try:
4 import ray
@@ -29,8 +29,9 @@ class Controller:
29 num_cpu_blocks: int,
30 dtype: str,
31 seed: int,
32 - model_path: str,
32 + cache_dir: Optional[str],
33 use_dummy_weights: bool,
34 + use_np_cache: bool,
35 max_num_batched_tokens: int,
36 use_ray: bool,
37 ) -> None:
@@ -66,8 +67,9 @@ class Controller:
67 world_size=world_size,
68 tensor_parallel_size=tensor_parallel_size,
69 pipeline_parallel_size=pipeline_parallel_size,
69 - model_path=model_path,
70 + cache_dir=cache_dir,
71 use_dummy_weights=use_dummy_weights,
72 + use_np_cache=use_np_cache,
73 max_num_batched_tokens=max_num_batched_tokens,
74 )
75 self.workers.append(worker)
cacheflow/worker/worker.py
+5 -3
@@ -1,4 +1,4 @@
1 -from typing import Dict, List, Tuple
1 +from typing import Dict, List, Tuple, Optional
2
3 import torch
4
@@ -28,8 +28,9 @@ class Worker:
28 distributed_init_method: str,
29 rank: int,
30 world_size: int,
31 - model_path: str,
31 + cache_dir: Optional[str],
32 use_dummy_weights: bool,
33 + use_np_cache: bool,
34 max_num_batched_tokens: int,
35 tensor_parallel_size: int = 1,
36 pipeline_parallel_size: int = 1,
@@ -45,7 +46,8 @@ class Worker:
46
47 # Initialize the model.
48 self.model, self.dtype = get_model(
48 - model_name, dtype=dtype, path=model_path, use_dummy_weights=use_dummy_weights)
49 + model_name, dtype=dtype, cache_dir=cache_dir,
50 + use_dummy_weights=use_dummy_weights, use_np_cache=use_np_cache)
51 tensor_model_parallel_world_size = (
52 get_tensor_model_parallel_world_size())
53 initialize_all_reduce_launcher(
simple_server.py
+4 -44
@@ -1,53 +1,13 @@
1 import argparse
2 from typing import List
3
4 -from cacheflow.master.simple_frontend import SimpleFrontend
5 -from cacheflow.master.server import (Server, add_server_arguments,
6 - process_server_arguments,
7 - initialize_cluster)
4 +from cacheflow.master.server import (
5 + add_server_arguments, process_server_arguments,
6 + init_local_server_and_frontend_with_arguments)
7 from cacheflow.sampling_params import SamplingParams
9 -from cacheflow.utils import get_gpu_memory, get_cpu_memory
8
9 def main(args: argparse.Namespace):
12 - # TODO(zhuohan): Support pipeline parallelism.
13 - assert args.pipeline_parallel_size == 1, (
14 - 'Pipeline parallelism is not supported yet.')
15 -
16 - (num_nodes, num_devices_per_node, distributed_init_method,
17 - all_stage_devices) = (
18 - initialize_cluster(
19 - use_ray=args.use_ray,
20 - pipeline_parallel_size=args.pipeline_parallel_size,
21 - tensor_parallel_size=args.tensor_parallel_size))
22 -
23 - # Create a server.
24 - server = Server(
25 - model=args.model,
26 - model_path=args.model_path,
27 - use_dummy_weights=args.use_dummy_weights,
28 - pipeline_parallel_size=args.pipeline_parallel_size,
29 - tensor_parallel_size=args.tensor_parallel_size,
30 - block_size=args.block_size,
31 - dtype=args.dtype,
32 - seed=args.seed,
33 - swap_space=args.swap_space,
34 - max_num_batched_tokens=args.max_num_batched_tokens,
35 - max_num_sequences=args.max_num_sequences,
36 - num_nodes=num_nodes,
37 - num_devices_per_node=num_devices_per_node,
38 - distributed_init_method=distributed_init_method,
39 - all_stage_devices=all_stage_devices,
40 - gpu_memory=get_gpu_memory(),
41 - cpu_memory=get_cpu_memory(),
42 - use_ray=args.use_ray,
43 - )
44 -
45 - # Create a frontend.
46 - frontend = SimpleFrontend(
47 - model_name=args.model,
48 - block_size=args.block_size,
49 - )
50 -
10 + server, frontend = init_local_server_and_frontend_with_arguments(args)
11 # Test the following inputs.
12 test_inputs = [
13 ('Ion Stoica is a', {'n': 4, 'use_beam_search': True, 'temperature': 0.0}),