A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Enable safetensors loading for all models (#974)
Zhuohan Li committed
Sep 7, 2023 at 15:49 UTC
c957c741d9da4bb07a303663e82d826fb5f8c6c4
18 files changed
+143
-83
vllm/config.py
+23
-7
@@ -24,9 +24,16 @@ class ModelConfig:
24
downloading the model and tokenizer.
25
download_dir: Directory to download and load the weights, default to the
26
default cache directory of huggingface.
27
- use_np_weights: Save a numpy copy of model weights for faster loading.
28
- This can increase the disk usage by up to 2x.
29
- use_dummy_weights: Use dummy values for model weights (for profiling).
27
+ load_format: The format of the model weights to load:
28
+ "auto" will try to load the weights in the safetensors format and
29
+ fall back to the pytorch bin format if safetensors format is
30
+ not available.
31
+ "pt" will load the weights in the pytorch bin format.
32
+ "safetensors" will load the weights in the safetensors format.
33
+ "npcache" will load the weights in pytorch format and store
34
+ a numpy cache to speed up the loading.
35
+ "dummy" will initialize the weights with random values, which is
36
+ mainly for profiling.
37
dtype: Data type for model weights and activations. The "auto" option
38
will use FP16 precision for FP32 and FP16 models, and BF16 precision
39
for BF16 models.
@@ -40,8 +47,7 @@ class ModelConfig:
47
tokenizer_mode: str,
48
trust_remote_code: bool,
49
download_dir: Optional[str],
43
- use_np_weights: bool,
44
- use_dummy_weights: bool,
50
+ load_format: str,
51
dtype: str,
52
seed: int,
53
) -> None:
@@ -50,14 +56,24 @@ class ModelConfig:
56
self.tokenizer_mode = tokenizer_mode
57
self.trust_remote_code = trust_remote_code
58
self.download_dir = download_dir
53
- self.use_np_weights = use_np_weights
54
- self.use_dummy_weights = use_dummy_weights
59
+ self.load_format = load_format
60
self.seed = seed
61
62
self.hf_config = get_config(model, trust_remote_code)
63
self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
64
+ self._verify_load_format()
65
self._verify_tokenizer_mode()
66
67
+ def _verify_load_format(self) -> None:
68
+ load_format = self.load_format.lower()
69
+ if load_format not in [
70
+ "auto", "pt", "safetensors", "npcache", "dummy"
71
+ ]:
72
+ raise ValueError(
73
+ f"Unknown load format: {self.load_format}. Must be one of "
74
+ "'auto', 'pt', 'safetensors', 'npcache', or 'dummy'.")
75
+ self.load_format = load_format
76
+
77
def _verify_tokenizer_mode(self) -> None:
78
tokenizer_mode = self.tokenizer_mode.lower()
79
if tokenizer_mode not in ["auto", "slow"]:
vllm/engine/arg_utils.py
+18
-13
@@ -15,8 +15,7 @@ class EngineArgs:
15
tokenizer_mode: str = 'auto'
16
trust_remote_code: bool = False
17
download_dir: Optional[str] = None
18
- use_np_weights: bool = False
19
- use_dummy_weights: bool = False
18
+ load_format: str = 'auto'
19
dtype: str = 'auto'
20
seed: int = 0
21
worker_use_ray: bool = False
@@ -65,14 +64,21 @@ class EngineArgs:
64
help='directory to download and load the weights, '
65
'default to the default cache dir of '
66
'huggingface')
68
- parser.add_argument('--use-np-weights',
69
- action='store_true',
70
- help='save a numpy copy of model weights for '
71
- 'faster loading. This can increase the disk '
72
- 'usage by up to 2x.')
73
- parser.add_argument('--use-dummy-weights',
74
- action='store_true',
75
- help='use dummy values for model weights')
67
+ parser.add_argument(
68
+ '--load-format',
69
+ type=str,
70
+ default=EngineArgs.load_format,
71
+ choices=['auto', 'pt', 'safetensors', 'npcache', 'dummy'],
72
+ help='The format of the model weights to load. '
73
+ '"auto" will try to load the weights in the safetensors format '
74
+ 'and fall back to the pytorch bin format if safetensors format '
75
+ 'is not available. '
76
+ '"pt" will load the weights in the pytorch bin format. '
77
+ '"safetensors" will load the weights in the safetensors format. '
78
+ '"npcache" will load the weights in pytorch format and store '
79
+ 'a numpy cache to speed up the loading. '
80
+ '"dummy" will initialize the weights with random values, '
81
+ 'which is mainly for profiling.')
82
# TODO(woosuk): Support FP32.
83
parser.add_argument(
84
'--dtype',
@@ -146,9 +152,8 @@ class EngineArgs:
152
# Initialize the configs.
153
model_config = ModelConfig(self.model, self.tokenizer,
154
self.tokenizer_mode, self.trust_remote_code,
149
- self.download_dir, self.use_np_weights,
150
- self.use_dummy_weights, self.dtype,
151
- self.seed)
155
+ self.download_dir, self.load_format,
156
+ self.dtype, self.seed)
157
cache_config = CacheConfig(self.block_size,
158
self.gpu_memory_utilization,
159
self.swap_space)
vllm/engine/llm_engine.py
+1
-2
@@ -76,9 +76,8 @@ class LLMEngine:
76
f"tokenizer_mode={model_config.tokenizer_mode}, "
77
f"trust_remote_code={model_config.trust_remote_code}, "
78
f"dtype={model_config.dtype}, "
79
- f"use_dummy_weights={model_config.use_dummy_weights}, "
79
f"download_dir={model_config.download_dir!r}, "
81
- f"use_np_weights={model_config.use_np_weights}, "
80
+ f"load_format={model_config.load_format}, "
81
f"tensor_parallel_size={parallel_config.tensor_parallel_size}, "
82
f"seed={model_config.seed})")
83
# TODO(woosuk): Print more configs in debug mode.
vllm/model_executor/model_loader.py
+2
-2
@@ -56,7 +56,7 @@ def get_model(model_config: ModelConfig) -> nn.Module:
56
# Create a model instance.
57
# The weights will be initialized as empty tensors.
58
model = model_class(model_config.hf_config)
59
- if model_config.use_dummy_weights:
59
+ if model_config.load_format == "dummy":
60
model = model.cuda()
61
# NOTE(woosuk): For accurate performance evaluation, we assign
62
# random values to the weights.
@@ -64,6 +64,6 @@ def get_model(model_config: ModelConfig) -> nn.Module:
64
else:
65
# Load the weights from the cached or downloaded files.
66
model.load_weights(model_config.model, model_config.download_dir,
67
- model_config.use_np_weights)
67
+ model_config.load_format)
68
model = model.cuda()
69
return model.eval()
vllm/model_executor/models/aquila.py
+2
-2
@@ -288,7 +288,7 @@ class AquilaForCausalLM(nn.Module):
288
def load_weights(self,
289
model_name_or_path: str,
290
cache_dir: Optional[str] = None,
291
- use_np_cache: bool = False):
291
+ load_format: str = "auto"):
292
tp_size = get_tensor_model_parallel_world_size()
293
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
294
q_proj_shard_size = (self.config.hidden_size // tp_size)
@@ -305,7 +305,7 @@ class AquilaForCausalLM(nn.Module):
305
state_dict = self.state_dict()
306
307
for name, loaded_weight in hf_model_weights_iterator(
308
- model_name_or_path, cache_dir, use_np_cache):
308
+ model_name_or_path, cache_dir, load_format):
309
if "rotary_emb.inv_freq" in name:
310
continue
311
vllm/model_executor/models/baichuan.py
+6
-4
@@ -35,8 +35,8 @@ from vllm.model_executor.layers.attention import (PagedAttentionWithRoPE,
35
PagedAttentionWithALiBi)
36
from vllm.model_executor.layers.sampler import Sampler
37
from vllm.model_executor.weight_utils import (
38
- hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
39
- load_tensor_parallel_weights)
38
+ convert_pyslice_to_tensor, hf_model_weights_iterator,
39
+ load_padded_tensor_parallel_vocab, load_tensor_parallel_weights)
40
from vllm.model_executor.parallel_utils.parallel_state import (
41
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42
from vllm.model_executor.parallel_utils.tensor_parallel import (
@@ -303,16 +303,18 @@ class BaiChuanBaseForCausalLM(nn.Module):
303
def load_weights(self,
304
model_name_or_path: str,
305
cache_dir: Optional[str] = None,
306
- use_np_cache: bool = False):
306
+ load_format: str = "auto"):
307
tp_world_size = get_tensor_model_parallel_world_size()
308
tp_rank = get_tensor_model_parallel_rank()
309
state_dict = self.state_dict()
310
311
for name, loaded_weight in hf_model_weights_iterator(
312
- model_name_or_path, cache_dir, use_np_cache):
312
+ model_name_or_path, cache_dir, load_format):
313
if "rotary_emb.inv_freq" in name:
314
continue
315
316
+ loaded_weight = convert_pyslice_to_tensor(loaded_weight)
317
+
318
if "W_pack" in name:
319
total_num_heads = self.config.num_attention_heads
320
hidden_size = self.config.hidden_size
vllm/model_executor/models/bloom.py
+2
-2
@@ -279,11 +279,11 @@ class BloomForCausalLM(nn.Module):
279
def load_weights(self,
280
model_name_or_path: str,
281
cache_dir: Optional[str] = None,
282
- use_np_cache: bool = False):
282
+ load_format: str = "auto"):
283
tp_rank = get_tensor_model_parallel_rank()
284
state_dict = self.state_dict()
285
for name, loaded_weight in hf_model_weights_iterator(
286
- model_name_or_path, cache_dir, use_np_cache):
286
+ model_name_or_path, cache_dir, load_format):
287
if name == "lm_head.weight":
288
# Since hidden_states are parallelized, we need to
289
# load lm_head.weight in parallel.
vllm/model_executor/models/falcon.py
+5
-3
@@ -31,7 +31,8 @@ from vllm.model_executor.layers.attention import (PagedAttention,
31
PagedAttentionWithALiBi,
32
PagedAttentionWithRoPE)
33
from vllm.model_executor.layers.sampler import Sampler
34
-from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
34
+from vllm.model_executor.weight_utils import (convert_pyslice_to_tensor,
35
+ hf_model_weights_iterator,
36
load_tensor_parallel_weights)
37
from vllm.model_executor.parallel_utils.parallel_state import (
38
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
@@ -419,7 +420,7 @@ class FalconForCausalLM(nn.Module):
420
def load_weights(self,
421
model_name_or_path: str,
422
cache_dir: Optional[str] = None,
422
- use_np_cache: bool = False):
423
+ load_format: str = "auto"):
424
tp_size = (get_tensor_model_parallel_world_size())
425
tp_rank = get_tensor_model_parallel_rank()
426
@@ -451,8 +452,9 @@ class FalconForCausalLM(nn.Module):
452
state_dict = self.state_dict()
453
454
for name, loaded_weight in hf_model_weights_iterator(
454
- model_name_or_path, cache_dir, use_np_cache):
455
+ model_name_or_path, cache_dir, load_format):
456
if "query_key_value" in name:
457
+ loaded_weight = convert_pyslice_to_tensor(loaded_weight)
458
loaded_weight_size = loaded_weight.size()
459
loaded_weight = loaded_weight.view(
460
total_num_kv_heads, num_query_heads_per_kv_head + 2,
vllm/model_executor/models/gpt2.py
+6
-4
@@ -32,8 +32,8 @@ from vllm.model_executor.layers.activation import get_act_fn
32
from vllm.model_executor.layers.attention import PagedAttention
33
from vllm.model_executor.layers.sampler import Sampler
34
from vllm.model_executor.weight_utils import (
35
- hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
36
- load_tensor_parallel_weights)
35
+ convert_pyslice_to_tensor, hf_model_weights_iterator,
36
+ load_padded_tensor_parallel_vocab, load_tensor_parallel_weights)
37
from vllm.model_executor.parallel_utils.parallel_state import (
38
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
39
from vllm.model_executor.parallel_utils.tensor_parallel import (
@@ -231,14 +231,14 @@ class GPT2LMHeadModel(nn.Module):
231
def load_weights(self,
232
model_name_or_path: str,
233
cache_dir: Optional[str] = None,
234
- use_np_cache: bool = False):
234
+ load_format: str = "auto"):
235
tensor_model_parallel_world_size = (
236
get_tensor_model_parallel_world_size())
237
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
238
state_dict = self.state_dict()
239
240
for name, loaded_weight in hf_model_weights_iterator(
241
- model_name_or_path, cache_dir, use_np_cache):
241
+ model_name_or_path, cache_dir, load_format):
242
if "lm_head.weight" in name:
243
# GPT-2 ties the weights of the embedding layer and the final
244
# linear layer.
@@ -251,6 +251,8 @@ class GPT2LMHeadModel(nn.Module):
251
if not name.startswith("transformer."):
252
name = "transformer." + name
253
254
+ loaded_weight = convert_pyslice_to_tensor(loaded_weight)
255
+
256
# The HF's GPT-2 implementation uses Conv1D instead of Linear.
257
# Because of this, we need to transpose the weights.
258
for conv1d_weight_name in ["c_attn", "c_proj", "c_fc"]:
vllm/model_executor/models/gpt_bigcode.py
+5
-4
@@ -33,8 +33,8 @@ from vllm.model_executor.layers.activation import get_act_fn
33
from vllm.model_executor.layers.attention import PagedAttention
34
from vllm.model_executor.layers.sampler import Sampler
35
from vllm.model_executor.weight_utils import (
36
- hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
37
- load_tensor_parallel_weights)
36
+ convert_pyslice_to_tensor, hf_model_weights_iterator,
37
+ load_padded_tensor_parallel_vocab, load_tensor_parallel_weights)
38
from vllm.model_executor.parallel_utils.parallel_state import (
39
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
40
from vllm.model_executor.parallel_utils.tensor_parallel import (
@@ -259,14 +259,14 @@ class GPTBigCodeForCausalLM(nn.Module):
259
def load_weights(self,
260
model_name_or_path: str,
261
cache_dir: Optional[str] = None,
262
- use_np_cache: bool = False):
262
+ load_format: str = "auto"):
263
tensor_model_parallel_world_size = (
264
get_tensor_model_parallel_world_size())
265
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
266
state_dict = self.state_dict()
267
268
for name, loaded_weight in hf_model_weights_iterator(
269
- model_name_or_path, cache_dir, use_np_cache):
269
+ model_name_or_path, cache_dir, load_format):
270
if "lm_head.weight" in name:
271
# GPT-2 ties the weights of the embedding layer and the final
272
# linear layer.
@@ -295,6 +295,7 @@ class GPTBigCodeForCausalLM(nn.Module):
295
head_start = tensor_model_parallel_rank * num_heads
296
head_end = (tensor_model_parallel_rank + 1) * num_heads
297
298
+ loaded_weight = convert_pyslice_to_tensor(loaded_weight)
299
wq, wk, wv = torch.split(
300
loaded_weight, [hidden_size, total_kv_size, total_kv_size],
301
dim=0)
vllm/model_executor/models/gpt_j.py
+2
-2
@@ -222,11 +222,11 @@ class GPTJForCausalLM(nn.Module):
222
def load_weights(self,
223
model_name_or_path: str,
224
cache_dir: Optional[str] = None,
225
- use_np_cache: bool = False):
225
+ load_format: str = "auto"):
226
tp_rank = get_tensor_model_parallel_rank()
227
state_dict = self.state_dict()
228
for name, loaded_weight in hf_model_weights_iterator(
229
- model_name_or_path, cache_dir, use_np_cache):
229
+ model_name_or_path, cache_dir, load_format):
230
if "attn.bias" in name or "attn.masked_bias" in name:
231
continue
232
vllm/model_executor/models/gpt_neox.py
+2
-2
@@ -231,11 +231,11 @@ class GPTNeoXForCausalLM(nn.Module):
231
def load_weights(self,
232
model_name_or_path: str,
233
cache_dir: Optional[str] = None,
234
- use_np_cache: bool = False):
234
+ load_format: str = "auto"):
235
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
236
state_dict = self.state_dict()
237
for name, loaded_weight in hf_model_weights_iterator(
238
- model_name_or_path, cache_dir, use_np_cache):
238
+ model_name_or_path, cache_dir, load_format):
239
if ("attention.bias" in name or "attention.masked_bias" in name
240
or "rotary_emb.inv_freq" in name):
241
continue
vllm/model_executor/models/internlm.py
+2
-2
@@ -233,12 +233,12 @@ class InternLMForCausalLM(nn.Module):
233
def load_weights(self,
234
model_name_or_path: str,
235
cache_dir: Optional[str] = None,
236
- use_np_cache: bool = False):
236
+ load_format: str = "auto"):
237
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
238
state_dict = self.state_dict()
239
240
for name, loaded_weight in hf_model_weights_iterator(
241
- model_name_or_path, cache_dir, use_np_cache):
241
+ model_name_or_path, cache_dir, load_format):
242
if "rotary_emb.inv_freq" in name:
243
continue
244
vllm/model_executor/models/llama.py
+2
-3
@@ -271,8 +271,7 @@ class LlamaForCausalLM(nn.Module):
271
def load_weights(self,
272
model_name_or_path: str,
273
cache_dir: Optional[str] = None,
274
- use_np_cache: bool = False,
275
- use_safetensor: bool = True):
274
+ load_format: str = "auto"):
275
tp_size = get_tensor_model_parallel_world_size()
276
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
277
q_proj_shard_size = (self.config.hidden_size // tp_size)
@@ -289,7 +288,7 @@ class LlamaForCausalLM(nn.Module):
288
state_dict = self.state_dict()
289
290
for name, loaded_weight in hf_model_weights_iterator(
292
- model_name_or_path, cache_dir, use_np_cache, use_safetensor):
291
+ model_name_or_path, cache_dir, load_format):
292
if "rotary_emb.inv_freq" in name:
293
continue
294
vllm/model_executor/models/mpt.py
+5
-4
@@ -10,7 +10,8 @@ from vllm.model_executor.input_metadata import InputMetadata
10
from vllm.model_executor.layers.activation import get_act_fn
11
from vllm.model_executor.layers.attention import PagedAttentionWithALiBi
12
from vllm.model_executor.layers.sampler import Sampler
13
-from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
13
+from vllm.model_executor.weight_utils import (convert_pyslice_to_tensor,
14
+ hf_model_weights_iterator,
15
load_tensor_parallel_weights)
16
from vllm.model_executor.parallel_utils.parallel_state import (
17
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
@@ -243,12 +244,12 @@ class MPTForCausalLM(nn.Module):
244
def load_weights(self,
245
model_name_or_path: str,
246
cache_dir: Optional[str] = None,
246
- use_np_cache: bool = False):
247
+ load_format: str = "auto"):
248
tp_world_size = get_tensor_model_parallel_world_size()
249
tp_rank = get_tensor_model_parallel_rank()
250
state_dict = self.state_dict()
251
for name, loaded_weight in hf_model_weights_iterator(
251
- model_name_or_path, cache_dir, use_np_cache):
252
+ model_name_or_path, cache_dir, load_format):
253
if "Wqkv" in name:
254
# NOTE(woosuk): MPT's fused QKV has the shape of
255
# [3 * num_heads * head_size, hidden_size].
@@ -260,7 +261,7 @@ class MPTForCausalLM(nn.Module):
261
num_heads = total_num_heads // tp_world_size
262
head_start = tp_rank * num_heads
263
head_end = (tp_rank + 1) * num_heads
263
-
264
+ loaded_weight = convert_pyslice_to_tensor(loaded_weight)
265
if name.endswith(".weight"):
266
loaded_weight = loaded_weight.view(3, total_num_heads,
267
head_size, hidden_size)
vllm/model_executor/models/opt.py
+2
-2
@@ -297,12 +297,12 @@ class OPTForCausalLM(nn.Module):
297
def load_weights(self,
298
model_name_or_path: str,
299
cache_dir: Optional[str] = None,
300
- use_np_cache: bool = False):
300
+ load_format: str = "auto"):
301
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
302
state_dict = self.state_dict()
303
304
for name, loaded_weight in hf_model_weights_iterator(
305
- model_name_or_path, cache_dir, use_np_cache):
305
+ model_name_or_path, cache_dir, load_format):
306
if "lm_head.weight" in name:
307
continue
308
vllm/model_executor/models/qwen.py
+5
-2
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.layernorm import RMSNorm
19
from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
20
from vllm.model_executor.layers.sampler import Sampler
21
from vllm.model_executor.weight_utils import (
22
+ convert_pyslice_to_tensor,
23
hf_model_weights_iterator,
24
load_padded_tensor_parallel_vocab,
25
load_tensor_parallel_weights,
@@ -249,17 +250,19 @@ class QWenLMHeadModel(nn.Module):
250
self,
251
model_name_or_path: str,
252
cache_dir: Optional[str] = None,
252
- use_np_cache: bool = False,
253
+ load_format: str = "auto",
254
):
255
tp_world_size = get_tensor_model_parallel_world_size()
256
tp_rank = get_tensor_model_parallel_rank()
257
state_dict = self.state_dict()
258
259
for name, loaded_weight in hf_model_weights_iterator(
259
- model_name_or_path, cache_dir, use_np_cache):
260
+ model_name_or_path, cache_dir, load_format):
261
if "rotary_emb.inv_freq" in name:
262
continue
263
264
+ loaded_weight = convert_pyslice_to_tensor(loaded_weight)
265
+
266
if "c_attn" in name:
267
total_num_heads = self.config.num_attention_heads
268
hidden_size = self.config.hidden_size
vllm/model_executor/weight_utils.py
+53
-23
@@ -81,11 +81,12 @@ def convert_bin_to_safetensor_file(
81
def prepare_hf_model_weights(
82
model_name_or_path: str,
83
cache_dir: Optional[str] = None,
84
- use_safetensor: bool = False,
84
+ use_safetensors: bool = False,
85
+ fall_back_to_pt: bool = True,
86
):
87
# Download model weights from huggingface.
88
is_local = os.path.isdir(model_name_or_path)
88
- allow_patterns = "*.safetensors" if use_safetensor else "*.bin"
89
+ allow_patterns = "*.safetensors" if use_safetensors else "*.bin"
90
if not is_local:
91
# Use file lock to prevent multiple processes from
92
# downloading the same model weights at the same time.
@@ -97,32 +98,53 @@ def prepare_hf_model_weights(
98
else:
99
hf_folder = model_name_or_path
100
hf_weights_files = glob.glob(os.path.join(hf_folder, allow_patterns))
100
- if not use_safetensor:
101
+ if not use_safetensors:
102
hf_weights_files = [
103
x for x in hf_weights_files if not x.endswith("training_args.bin")
104
]
105
105
- if len(hf_weights_files) == 0 and use_safetensor:
106
- logger.warning("No *.safetensors files found, "
107
- "fall back to *.bin files")
106
+ if len(hf_weights_files) == 0 and use_safetensors and fall_back_to_pt:
107
return prepare_hf_model_weights(model_name_or_path,
108
cache_dir=cache_dir,
110
- use_safetensor=False)
111
- return hf_folder, hf_weights_files, use_safetensor
109
+ use_safetensors=False,
110
+ fall_back_to_pt=False)
111
+
112
+ if len(hf_weights_files) == 0:
113
+ raise RuntimeError(
114
+ f"Cannot find any model weights with `{model_name_or_path}`")
115
+
116
+ return hf_folder, hf_weights_files, use_safetensors
117
118
119
def hf_model_weights_iterator(
120
model_name_or_path: str,
121
cache_dir: Optional[str] = None,
117
- use_np_cache: bool = False,
118
- use_safetensor: bool = False,
122
+ load_format: str = "auto",
123
) -> Iterator[Tuple[str, torch.Tensor]]:
120
- hf_folder, hf_weights_files, use_safetensor = prepare_hf_model_weights(
121
- model_name_or_path, cache_dir=cache_dir, use_safetensor=use_safetensor)
124
+ use_safetensors = False
125
+ use_np_cache = False
126
+ fall_back_to_pt = False
127
+ if load_format == "auto":
128
+ use_safetensors = True
129
+ fall_back_to_pt = True
130
+ elif load_format == "safetensors":
131
+ use_safetensors = True
132
+ elif load_format == "pt":
133
+ pass
134
+ elif load_format == "npcache":
135
+ use_np_cache = True
136
+ else:
137
+ raise ValueError(f"Unknown load_format: {load_format}")
138
+
139
+ hf_folder, hf_weights_files, use_safetensors = prepare_hf_model_weights(
140
+ model_name_or_path,
141
+ cache_dir=cache_dir,
142
+ use_safetensors=use_safetensors,
143
+ fall_back_to_pt=fall_back_to_pt)
144
145
if use_np_cache:
146
# Currently np_cache only support *.bin checkpoints
125
- assert use_safetensor is False
147
+ assert use_safetensors is False
148
149
# Convert the model weights from torch tensors to numpy arrays for
150
# faster loading.
@@ -152,7 +174,7 @@ def hf_model_weights_iterator(
174
with open(param_path, "rb") as f:
175
param = np.load(f)
176
yield name, torch.from_numpy(param)
155
- elif use_safetensor:
177
+ elif use_safetensors:
178
for st_file in hf_weights_files:
179
with safe_open(st_file, framework="pt") as f:
180
for name in f.keys():
@@ -167,6 +189,21 @@ def hf_model_weights_iterator(
189
torch.cuda.empty_cache()
190
191
192
+def convert_pyslice_to_tensor(x: Any) -> torch.Tensor:
193
+ """convert PySafeSlice object from safetensors to torch.Tensor
194
+
195
+ PySafeSlice object supports indexing, which is done before loading the
196
+ actual tensor and can reduce the amount of memory being read into the
197
+ memory. However, it does not support more advanced functionalities
198
+ like `.view()` or `.t()`. Therefore, if we need to modify the loaded
199
+ tensor with these more complicated operators, we need to convert to
200
+ tensor first.
201
+ """
202
+ if not isinstance(x, torch.Tensor):
203
+ x = x[:]
204
+ return x
205
+
206
+
207
def load_padded_tensor_parallel_vocab(
208
param: torch.Tensor,
209
loaded_weight: Any, # `torch.Tensor` or `PySafeSlice`
@@ -176,11 +213,7 @@ def load_padded_tensor_parallel_vocab(
213
start_idx = tensor_model_parallel_rank * shard_size
214
end_idx = (tensor_model_parallel_rank + 1) * shard_size
215
loaded_weight = loaded_weight[start_idx:end_idx]
179
-
180
- # convert PySafeSlice object to torch.Tensor
181
- if not isinstance(loaded_weight, torch.Tensor):
182
- loaded_weight = loaded_weight[:]
183
-
216
+ loaded_weight = convert_pyslice_to_tensor(loaded_weight)
217
param[:loaded_weight.shape[0]].copy_(loaded_weight)
218
219
@@ -207,10 +240,7 @@ def load_tensor_parallel_weights(
240
loaded_weight = loaded_weight[:, start_idx:end_idx]
241
break
242
210
- # convert PySafeSlice object to torch.Tensor
211
- if not isinstance(loaded_weight, torch.Tensor):
212
- loaded_weight = loaded_weight[:]
213
-
243
+ loaded_weight = convert_pyslice_to_tensor(loaded_weight)
244
assert param.shape == loaded_weight.shape, (
245
f"{param_name} shape mismatch between model and checkpoint: "
246
f"{param.shape} != {loaded_weight.shape}")