@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c957c741d9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Enable safetensors loading for all models (#974)

Zhuohan Li committed Sep 7, 2023 at 15:49 UTC c957c741d9da4bb07a303663e82d826fb5f8c6c4
18 files changed +143 -83
vllm/config.py
+23 -7
@@ -24,9 +24,16 @@ class ModelConfig:
24 downloading the model and tokenizer.
25 download_dir: Directory to download and load the weights, default to the
26 default cache directory of huggingface.
27 - use_np_weights: Save a numpy copy of model weights for faster loading.
28 - This can increase the disk usage by up to 2x.
29 - use_dummy_weights: Use dummy values for model weights (for profiling).
27 + load_format: The format of the model weights to load:
28 + "auto" will try to load the weights in the safetensors format and
29 + fall back to the pytorch bin format if safetensors format is
30 + not available.
31 + "pt" will load the weights in the pytorch bin format.
32 + "safetensors" will load the weights in the safetensors format.
33 + "npcache" will load the weights in pytorch format and store
34 + a numpy cache to speed up the loading.
35 + "dummy" will initialize the weights with random values, which is
36 + mainly for profiling.
37 dtype: Data type for model weights and activations. The "auto" option
38 will use FP16 precision for FP32 and FP16 models, and BF16 precision
39 for BF16 models.
@@ -40,8 +47,7 @@ class ModelConfig:
47 tokenizer_mode: str,
48 trust_remote_code: bool,
49 download_dir: Optional[str],
43 - use_np_weights: bool,
44 - use_dummy_weights: bool,
50 + load_format: str,
51 dtype: str,
52 seed: int,
53 ) -> None:
@@ -50,14 +56,24 @@ class ModelConfig:
56 self.tokenizer_mode = tokenizer_mode
57 self.trust_remote_code = trust_remote_code
58 self.download_dir = download_dir
53 - self.use_np_weights = use_np_weights
54 - self.use_dummy_weights = use_dummy_weights
59 + self.load_format = load_format
60 self.seed = seed
61
62 self.hf_config = get_config(model, trust_remote_code)
63 self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
64 + self._verify_load_format()
65 self._verify_tokenizer_mode()
66
67 + def _verify_load_format(self) -> None:
68 + load_format = self.load_format.lower()
69 + if load_format not in [
70 + "auto", "pt", "safetensors", "npcache", "dummy"
71 + ]:
72 + raise ValueError(
73 + f"Unknown load format: {self.load_format}. Must be one of "
74 + "'auto', 'pt', 'safetensors', 'npcache', or 'dummy'.")
75 + self.load_format = load_format
76 +
77 def _verify_tokenizer_mode(self) -> None:
78 tokenizer_mode = self.tokenizer_mode.lower()
79 if tokenizer_mode not in ["auto", "slow"]:
vllm/engine/arg_utils.py
+18 -13
@@ -15,8 +15,7 @@ class EngineArgs:
15 tokenizer_mode: str = 'auto'
16 trust_remote_code: bool = False
17 download_dir: Optional[str] = None
18 - use_np_weights: bool = False
19 - use_dummy_weights: bool = False
18 + load_format: str = 'auto'
19 dtype: str = 'auto'
20 seed: int = 0
21 worker_use_ray: bool = False
@@ -65,14 +64,21 @@ class EngineArgs:
64 help='directory to download and load the weights, '
65 'default to the default cache dir of '
66 'huggingface')
68 - parser.add_argument('--use-np-weights',
69 - action='store_true',
70 - help='save a numpy copy of model weights for '
71 - 'faster loading. This can increase the disk '
72 - 'usage by up to 2x.')
73 - parser.add_argument('--use-dummy-weights',
74 - action='store_true',
75 - help='use dummy values for model weights')
67 + parser.add_argument(
68 + '--load-format',
69 + type=str,
70 + default=EngineArgs.load_format,
71 + choices=['auto', 'pt', 'safetensors', 'npcache', 'dummy'],
72 + help='The format of the model weights to load. '
73 + '"auto" will try to load the weights in the safetensors format '
74 + 'and fall back to the pytorch bin format if safetensors format '
75 + 'is not available. '
76 + '"pt" will load the weights in the pytorch bin format. '
77 + '"safetensors" will load the weights in the safetensors format. '
78 + '"npcache" will load the weights in pytorch format and store '
79 + 'a numpy cache to speed up the loading. '
80 + '"dummy" will initialize the weights with random values, '
81 + 'which is mainly for profiling.')
82 # TODO(woosuk): Support FP32.
83 parser.add_argument(
84 '--dtype',
@@ -146,9 +152,8 @@ class EngineArgs:
152 # Initialize the configs.
153 model_config = ModelConfig(self.model, self.tokenizer,
154 self.tokenizer_mode, self.trust_remote_code,
149 - self.download_dir, self.use_np_weights,
150 - self.use_dummy_weights, self.dtype,
151 - self.seed)
155 + self.download_dir, self.load_format,
156 + self.dtype, self.seed)
157 cache_config = CacheConfig(self.block_size,
158 self.gpu_memory_utilization,
159 self.swap_space)
vllm/engine/llm_engine.py
+1 -2
@@ -76,9 +76,8 @@ class LLMEngine:
76 f"tokenizer_mode={model_config.tokenizer_mode}, "
77 f"trust_remote_code={model_config.trust_remote_code}, "
78 f"dtype={model_config.dtype}, "
79 - f"use_dummy_weights={model_config.use_dummy_weights}, "
79 f"download_dir={model_config.download_dir!r}, "
81 - f"use_np_weights={model_config.use_np_weights}, "
80 + f"load_format={model_config.load_format}, "
81 f"tensor_parallel_size={parallel_config.tensor_parallel_size}, "
82 f"seed={model_config.seed})")
83 # TODO(woosuk): Print more configs in debug mode.
vllm/model_executor/model_loader.py
+2 -2
@@ -56,7 +56,7 @@ def get_model(model_config: ModelConfig) -> nn.Module:
56 # Create a model instance.
57 # The weights will be initialized as empty tensors.
58 model = model_class(model_config.hf_config)
59 - if model_config.use_dummy_weights:
59 + if model_config.load_format == "dummy":
60 model = model.cuda()
61 # NOTE(woosuk): For accurate performance evaluation, we assign
62 # random values to the weights.
@@ -64,6 +64,6 @@ def get_model(model_config: ModelConfig) -> nn.Module:
64 else:
65 # Load the weights from the cached or downloaded files.
66 model.load_weights(model_config.model, model_config.download_dir,
67 - model_config.use_np_weights)
67 + model_config.load_format)
68 model = model.cuda()
69 return model.eval()
vllm/model_executor/models/aquila.py
+2 -2
@@ -288,7 +288,7 @@ class AquilaForCausalLM(nn.Module):
288 def load_weights(self,
289 model_name_or_path: str,
290 cache_dir: Optional[str] = None,
291 - use_np_cache: bool = False):
291 + load_format: str = "auto"):
292 tp_size = get_tensor_model_parallel_world_size()
293 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
294 q_proj_shard_size = (self.config.hidden_size // tp_size)
@@ -305,7 +305,7 @@ class AquilaForCausalLM(nn.Module):
305 state_dict = self.state_dict()
306
307 for name, loaded_weight in hf_model_weights_iterator(
308 - model_name_or_path, cache_dir, use_np_cache):
308 + model_name_or_path, cache_dir, load_format):
309 if "rotary_emb.inv_freq" in name:
310 continue
311
vllm/model_executor/models/baichuan.py
+6 -4
@@ -35,8 +35,8 @@ from vllm.model_executor.layers.attention import (PagedAttentionWithRoPE,
35 PagedAttentionWithALiBi)
36 from vllm.model_executor.layers.sampler import Sampler
37 from vllm.model_executor.weight_utils import (
38 - hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
39 - load_tensor_parallel_weights)
38 + convert_pyslice_to_tensor, hf_model_weights_iterator,
39 + load_padded_tensor_parallel_vocab, load_tensor_parallel_weights)
40 from vllm.model_executor.parallel_utils.parallel_state import (
41 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42 from vllm.model_executor.parallel_utils.tensor_parallel import (
@@ -303,16 +303,18 @@ class BaiChuanBaseForCausalLM(nn.Module):
303 def load_weights(self,
304 model_name_or_path: str,
305 cache_dir: Optional[str] = None,
306 - use_np_cache: bool = False):
306 + load_format: str = "auto"):
307 tp_world_size = get_tensor_model_parallel_world_size()
308 tp_rank = get_tensor_model_parallel_rank()
309 state_dict = self.state_dict()
310
311 for name, loaded_weight in hf_model_weights_iterator(
312 - model_name_or_path, cache_dir, use_np_cache):
312 + model_name_or_path, cache_dir, load_format):
313 if "rotary_emb.inv_freq" in name:
314 continue
315
316 + loaded_weight = convert_pyslice_to_tensor(loaded_weight)
317 +
318 if "W_pack" in name:
319 total_num_heads = self.config.num_attention_heads
320 hidden_size = self.config.hidden_size
vllm/model_executor/models/bloom.py
+2 -2
@@ -279,11 +279,11 @@ class BloomForCausalLM(nn.Module):
279 def load_weights(self,
280 model_name_or_path: str,
281 cache_dir: Optional[str] = None,
282 - use_np_cache: bool = False):
282 + load_format: str = "auto"):
283 tp_rank = get_tensor_model_parallel_rank()
284 state_dict = self.state_dict()
285 for name, loaded_weight in hf_model_weights_iterator(
286 - model_name_or_path, cache_dir, use_np_cache):
286 + model_name_or_path, cache_dir, load_format):
287 if name == "lm_head.weight":
288 # Since hidden_states are parallelized, we need to
289 # load lm_head.weight in parallel.
vllm/model_executor/models/falcon.py
+5 -3
@@ -31,7 +31,8 @@ from vllm.model_executor.layers.attention import (PagedAttention,
31 PagedAttentionWithALiBi,
32 PagedAttentionWithRoPE)
33 from vllm.model_executor.layers.sampler import Sampler
34 -from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
34 +from vllm.model_executor.weight_utils import (convert_pyslice_to_tensor,
35 + hf_model_weights_iterator,
36 load_tensor_parallel_weights)
37 from vllm.model_executor.parallel_utils.parallel_state import (
38 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
@@ -419,7 +420,7 @@ class FalconForCausalLM(nn.Module):
420 def load_weights(self,
421 model_name_or_path: str,
422 cache_dir: Optional[str] = None,
422 - use_np_cache: bool = False):
423 + load_format: str = "auto"):
424 tp_size = (get_tensor_model_parallel_world_size())
425 tp_rank = get_tensor_model_parallel_rank()
426
@@ -451,8 +452,9 @@ class FalconForCausalLM(nn.Module):
452 state_dict = self.state_dict()
453
454 for name, loaded_weight in hf_model_weights_iterator(
454 - model_name_or_path, cache_dir, use_np_cache):
455 + model_name_or_path, cache_dir, load_format):
456 if "query_key_value" in name:
457 + loaded_weight = convert_pyslice_to_tensor(loaded_weight)
458 loaded_weight_size = loaded_weight.size()
459 loaded_weight = loaded_weight.view(
460 total_num_kv_heads, num_query_heads_per_kv_head + 2,
vllm/model_executor/models/gpt2.py
+6 -4
@@ -32,8 +32,8 @@ from vllm.model_executor.layers.activation import get_act_fn
32 from vllm.model_executor.layers.attention import PagedAttention
33 from vllm.model_executor.layers.sampler import Sampler
34 from vllm.model_executor.weight_utils import (
35 - hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
36 - load_tensor_parallel_weights)
35 + convert_pyslice_to_tensor, hf_model_weights_iterator,
36 + load_padded_tensor_parallel_vocab, load_tensor_parallel_weights)
37 from vllm.model_executor.parallel_utils.parallel_state import (
38 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
39 from vllm.model_executor.parallel_utils.tensor_parallel import (
@@ -231,14 +231,14 @@ class GPT2LMHeadModel(nn.Module):
231 def load_weights(self,
232 model_name_or_path: str,
233 cache_dir: Optional[str] = None,
234 - use_np_cache: bool = False):
234 + load_format: str = "auto"):
235 tensor_model_parallel_world_size = (
236 get_tensor_model_parallel_world_size())
237 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
238 state_dict = self.state_dict()
239
240 for name, loaded_weight in hf_model_weights_iterator(
241 - model_name_or_path, cache_dir, use_np_cache):
241 + model_name_or_path, cache_dir, load_format):
242 if "lm_head.weight" in name:
243 # GPT-2 ties the weights of the embedding layer and the final
244 # linear layer.
@@ -251,6 +251,8 @@ class GPT2LMHeadModel(nn.Module):
251 if not name.startswith("transformer."):
252 name = "transformer." + name
253
254 + loaded_weight = convert_pyslice_to_tensor(loaded_weight)
255 +
256 # The HF's GPT-2 implementation uses Conv1D instead of Linear.
257 # Because of this, we need to transpose the weights.
258 for conv1d_weight_name in ["c_attn", "c_proj", "c_fc"]:
vllm/model_executor/models/gpt_bigcode.py
+5 -4
@@ -33,8 +33,8 @@ from vllm.model_executor.layers.activation import get_act_fn
33 from vllm.model_executor.layers.attention import PagedAttention
34 from vllm.model_executor.layers.sampler import Sampler
35 from vllm.model_executor.weight_utils import (
36 - hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
37 - load_tensor_parallel_weights)
36 + convert_pyslice_to_tensor, hf_model_weights_iterator,
37 + load_padded_tensor_parallel_vocab, load_tensor_parallel_weights)
38 from vllm.model_executor.parallel_utils.parallel_state import (
39 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
40 from vllm.model_executor.parallel_utils.tensor_parallel import (
@@ -259,14 +259,14 @@ class GPTBigCodeForCausalLM(nn.Module):
259 def load_weights(self,
260 model_name_or_path: str,
261 cache_dir: Optional[str] = None,
262 - use_np_cache: bool = False):
262 + load_format: str = "auto"):
263 tensor_model_parallel_world_size = (
264 get_tensor_model_parallel_world_size())
265 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
266 state_dict = self.state_dict()
267
268 for name, loaded_weight in hf_model_weights_iterator(
269 - model_name_or_path, cache_dir, use_np_cache):
269 + model_name_or_path, cache_dir, load_format):
270 if "lm_head.weight" in name:
271 # GPT-2 ties the weights of the embedding layer and the final
272 # linear layer.
@@ -295,6 +295,7 @@ class GPTBigCodeForCausalLM(nn.Module):
295 head_start = tensor_model_parallel_rank * num_heads
296 head_end = (tensor_model_parallel_rank + 1) * num_heads
297
298 + loaded_weight = convert_pyslice_to_tensor(loaded_weight)
299 wq, wk, wv = torch.split(
300 loaded_weight, [hidden_size, total_kv_size, total_kv_size],
301 dim=0)
vllm/model_executor/models/gpt_j.py
+2 -2
@@ -222,11 +222,11 @@ class GPTJForCausalLM(nn.Module):
222 def load_weights(self,
223 model_name_or_path: str,
224 cache_dir: Optional[str] = None,
225 - use_np_cache: bool = False):
225 + load_format: str = "auto"):
226 tp_rank = get_tensor_model_parallel_rank()
227 state_dict = self.state_dict()
228 for name, loaded_weight in hf_model_weights_iterator(
229 - model_name_or_path, cache_dir, use_np_cache):
229 + model_name_or_path, cache_dir, load_format):
230 if "attn.bias" in name or "attn.masked_bias" in name:
231 continue
232
vllm/model_executor/models/gpt_neox.py
+2 -2
@@ -231,11 +231,11 @@ class GPTNeoXForCausalLM(nn.Module):
231 def load_weights(self,
232 model_name_or_path: str,
233 cache_dir: Optional[str] = None,
234 - use_np_cache: bool = False):
234 + load_format: str = "auto"):
235 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
236 state_dict = self.state_dict()
237 for name, loaded_weight in hf_model_weights_iterator(
238 - model_name_or_path, cache_dir, use_np_cache):
238 + model_name_or_path, cache_dir, load_format):
239 if ("attention.bias" in name or "attention.masked_bias" in name
240 or "rotary_emb.inv_freq" in name):
241 continue
vllm/model_executor/models/internlm.py
+2 -2
@@ -233,12 +233,12 @@ class InternLMForCausalLM(nn.Module):
233 def load_weights(self,
234 model_name_or_path: str,
235 cache_dir: Optional[str] = None,
236 - use_np_cache: bool = False):
236 + load_format: str = "auto"):
237 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
238 state_dict = self.state_dict()
239
240 for name, loaded_weight in hf_model_weights_iterator(
241 - model_name_or_path, cache_dir, use_np_cache):
241 + model_name_or_path, cache_dir, load_format):
242 if "rotary_emb.inv_freq" in name:
243 continue
244
vllm/model_executor/models/llama.py
+2 -3
@@ -271,8 +271,7 @@ class LlamaForCausalLM(nn.Module):
271 def load_weights(self,
272 model_name_or_path: str,
273 cache_dir: Optional[str] = None,
274 - use_np_cache: bool = False,
275 - use_safetensor: bool = True):
274 + load_format: str = "auto"):
275 tp_size = get_tensor_model_parallel_world_size()
276 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
277 q_proj_shard_size = (self.config.hidden_size // tp_size)
@@ -289,7 +288,7 @@ class LlamaForCausalLM(nn.Module):
288 state_dict = self.state_dict()
289
290 for name, loaded_weight in hf_model_weights_iterator(
292 - model_name_or_path, cache_dir, use_np_cache, use_safetensor):
291 + model_name_or_path, cache_dir, load_format):
292 if "rotary_emb.inv_freq" in name:
293 continue
294
vllm/model_executor/models/mpt.py
+5 -4
@@ -10,7 +10,8 @@ from vllm.model_executor.input_metadata import InputMetadata
10 from vllm.model_executor.layers.activation import get_act_fn
11 from vllm.model_executor.layers.attention import PagedAttentionWithALiBi
12 from vllm.model_executor.layers.sampler import Sampler
13 -from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
13 +from vllm.model_executor.weight_utils import (convert_pyslice_to_tensor,
14 + hf_model_weights_iterator,
15 load_tensor_parallel_weights)
16 from vllm.model_executor.parallel_utils.parallel_state import (
17 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
@@ -243,12 +244,12 @@ class MPTForCausalLM(nn.Module):
244 def load_weights(self,
245 model_name_or_path: str,
246 cache_dir: Optional[str] = None,
246 - use_np_cache: bool = False):
247 + load_format: str = "auto"):
248 tp_world_size = get_tensor_model_parallel_world_size()
249 tp_rank = get_tensor_model_parallel_rank()
250 state_dict = self.state_dict()
251 for name, loaded_weight in hf_model_weights_iterator(
251 - model_name_or_path, cache_dir, use_np_cache):
252 + model_name_or_path, cache_dir, load_format):
253 if "Wqkv" in name:
254 # NOTE(woosuk): MPT's fused QKV has the shape of
255 # [3 * num_heads * head_size, hidden_size].
@@ -260,7 +261,7 @@ class MPTForCausalLM(nn.Module):
261 num_heads = total_num_heads // tp_world_size
262 head_start = tp_rank * num_heads
263 head_end = (tp_rank + 1) * num_heads
263 -
264 + loaded_weight = convert_pyslice_to_tensor(loaded_weight)
265 if name.endswith(".weight"):
266 loaded_weight = loaded_weight.view(3, total_num_heads,
267 head_size, hidden_size)
vllm/model_executor/models/opt.py
+2 -2
@@ -297,12 +297,12 @@ class OPTForCausalLM(nn.Module):
297 def load_weights(self,
298 model_name_or_path: str,
299 cache_dir: Optional[str] = None,
300 - use_np_cache: bool = False):
300 + load_format: str = "auto"):
301 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
302 state_dict = self.state_dict()
303
304 for name, loaded_weight in hf_model_weights_iterator(
305 - model_name_or_path, cache_dir, use_np_cache):
305 + model_name_or_path, cache_dir, load_format):
306 if "lm_head.weight" in name:
307 continue
308
vllm/model_executor/models/qwen.py
+5 -2
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.layernorm import RMSNorm
19 from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
20 from vllm.model_executor.layers.sampler import Sampler
21 from vllm.model_executor.weight_utils import (
22 + convert_pyslice_to_tensor,
23 hf_model_weights_iterator,
24 load_padded_tensor_parallel_vocab,
25 load_tensor_parallel_weights,
@@ -249,17 +250,19 @@ class QWenLMHeadModel(nn.Module):
250 self,
251 model_name_or_path: str,
252 cache_dir: Optional[str] = None,
252 - use_np_cache: bool = False,
253 + load_format: str = "auto",
254 ):
255 tp_world_size = get_tensor_model_parallel_world_size()
256 tp_rank = get_tensor_model_parallel_rank()
257 state_dict = self.state_dict()
258
259 for name, loaded_weight in hf_model_weights_iterator(
259 - model_name_or_path, cache_dir, use_np_cache):
260 + model_name_or_path, cache_dir, load_format):
261 if "rotary_emb.inv_freq" in name:
262 continue
263
264 + loaded_weight = convert_pyslice_to_tensor(loaded_weight)
265 +
266 if "c_attn" in name:
267 total_num_heads = self.config.num_attention_heads
268 hidden_size = self.config.hidden_size
vllm/model_executor/weight_utils.py
+53 -23
@@ -81,11 +81,12 @@ def convert_bin_to_safetensor_file(
81 def prepare_hf_model_weights(
82 model_name_or_path: str,
83 cache_dir: Optional[str] = None,
84 - use_safetensor: bool = False,
84 + use_safetensors: bool = False,
85 + fall_back_to_pt: bool = True,
86 ):
87 # Download model weights from huggingface.
88 is_local = os.path.isdir(model_name_or_path)
88 - allow_patterns = "*.safetensors" if use_safetensor else "*.bin"
89 + allow_patterns = "*.safetensors" if use_safetensors else "*.bin"
90 if not is_local:
91 # Use file lock to prevent multiple processes from
92 # downloading the same model weights at the same time.
@@ -97,32 +98,53 @@ def prepare_hf_model_weights(
98 else:
99 hf_folder = model_name_or_path
100 hf_weights_files = glob.glob(os.path.join(hf_folder, allow_patterns))
100 - if not use_safetensor:
101 + if not use_safetensors:
102 hf_weights_files = [
103 x for x in hf_weights_files if not x.endswith("training_args.bin")
104 ]
105
105 - if len(hf_weights_files) == 0 and use_safetensor:
106 - logger.warning("No *.safetensors files found, "
107 - "fall back to *.bin files")
106 + if len(hf_weights_files) == 0 and use_safetensors and fall_back_to_pt:
107 return prepare_hf_model_weights(model_name_or_path,
108 cache_dir=cache_dir,
110 - use_safetensor=False)
111 - return hf_folder, hf_weights_files, use_safetensor
109 + use_safetensors=False,
110 + fall_back_to_pt=False)
111 +
112 + if len(hf_weights_files) == 0:
113 + raise RuntimeError(
114 + f"Cannot find any model weights with `{model_name_or_path}`")
115 +
116 + return hf_folder, hf_weights_files, use_safetensors
117
118
119 def hf_model_weights_iterator(
120 model_name_or_path: str,
121 cache_dir: Optional[str] = None,
117 - use_np_cache: bool = False,
118 - use_safetensor: bool = False,
122 + load_format: str = "auto",
123 ) -> Iterator[Tuple[str, torch.Tensor]]:
120 - hf_folder, hf_weights_files, use_safetensor = prepare_hf_model_weights(
121 - model_name_or_path, cache_dir=cache_dir, use_safetensor=use_safetensor)
124 + use_safetensors = False
125 + use_np_cache = False
126 + fall_back_to_pt = False
127 + if load_format == "auto":
128 + use_safetensors = True
129 + fall_back_to_pt = True
130 + elif load_format == "safetensors":
131 + use_safetensors = True
132 + elif load_format == "pt":
133 + pass
134 + elif load_format == "npcache":
135 + use_np_cache = True
136 + else:
137 + raise ValueError(f"Unknown load_format: {load_format}")
138 +
139 + hf_folder, hf_weights_files, use_safetensors = prepare_hf_model_weights(
140 + model_name_or_path,
141 + cache_dir=cache_dir,
142 + use_safetensors=use_safetensors,
143 + fall_back_to_pt=fall_back_to_pt)
144
145 if use_np_cache:
146 # Currently np_cache only support *.bin checkpoints
125 - assert use_safetensor is False
147 + assert use_safetensors is False
148
149 # Convert the model weights from torch tensors to numpy arrays for
150 # faster loading.
@@ -152,7 +174,7 @@ def hf_model_weights_iterator(
174 with open(param_path, "rb") as f:
175 param = np.load(f)
176 yield name, torch.from_numpy(param)
155 - elif use_safetensor:
177 + elif use_safetensors:
178 for st_file in hf_weights_files:
179 with safe_open(st_file, framework="pt") as f:
180 for name in f.keys():
@@ -167,6 +189,21 @@ def hf_model_weights_iterator(
189 torch.cuda.empty_cache()
190
191
192 +def convert_pyslice_to_tensor(x: Any) -> torch.Tensor:
193 + """convert PySafeSlice object from safetensors to torch.Tensor
194 +
195 + PySafeSlice object supports indexing, which is done before loading the
196 + actual tensor and can reduce the amount of memory being read into the
197 + memory. However, it does not support more advanced functionalities
198 + like `.view()` or `.t()`. Therefore, if we need to modify the loaded
199 + tensor with these more complicated operators, we need to convert to
200 + tensor first.
201 + """
202 + if not isinstance(x, torch.Tensor):
203 + x = x[:]
204 + return x
205 +
206 +
207 def load_padded_tensor_parallel_vocab(
208 param: torch.Tensor,
209 loaded_weight: Any, # `torch.Tensor` or `PySafeSlice`
@@ -176,11 +213,7 @@ def load_padded_tensor_parallel_vocab(
213 start_idx = tensor_model_parallel_rank * shard_size
214 end_idx = (tensor_model_parallel_rank + 1) * shard_size
215 loaded_weight = loaded_weight[start_idx:end_idx]
179 -
180 - # convert PySafeSlice object to torch.Tensor
181 - if not isinstance(loaded_weight, torch.Tensor):
182 - loaded_weight = loaded_weight[:]
183 -
216 + loaded_weight = convert_pyslice_to_tensor(loaded_weight)
217 param[:loaded_weight.shape[0]].copy_(loaded_weight)
218
219
@@ -207,10 +240,7 @@ def load_tensor_parallel_weights(
240 loaded_weight = loaded_weight[:, start_idx:end_idx]
241 break
242
210 - # convert PySafeSlice object to torch.Tensor
211 - if not isinstance(loaded_weight, torch.Tensor):
212 - loaded_weight = loaded_weight[:]
213 -
243 + loaded_weight = convert_pyslice_to_tensor(loaded_weight)
244 assert param.shape == loaded_weight.shape, (
245 f"{param_name} shape mismatch between model and checkpoint: "
246 f"{param.shape} != {loaded_weight.shape}")