A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Simplify weight loading logic (#2133)
Roy committed
Dec 17, 2023 at 04:41 UTC
eed74a558ffacc9a456d440b5d2ec1ca869e80b5
3 files changed
+33
-37
vllm/config.py
+4
-9
@@ -122,15 +122,10 @@ class ModelConfig:
122
123
# TODO: Remove this check once HF updates the pt weights of Mixtral.
124
architectures = getattr(self.hf_config, "architectures", [])
125
- if "MixtralForCausalLM" in architectures:
126
- if load_format == "pt":
127
- raise ValueError(
128
- "Currently, the 'pt' format is not supported for Mixtral. "
129
- "Please use the 'safetensors' format instead. ")
130
- elif load_format == "auto":
131
- # Do not fall back to pt weights.
132
- load_format = "safetensors"
133
-
125
+ if "MixtralForCausalLM" in architectures and load_format == "pt":
126
+ raise ValueError(
127
+ "Currently, the 'pt' format is not supported for Mixtral. "
128
+ "Please use the 'safetensors' format instead. ")
129
self.load_format = load_format
130
131
def _verify_tokenizer_mode(self) -> None:
vllm/model_executor/models/mixtral.py
+5
-1
@@ -412,7 +412,11 @@ class MixtralForCausalLM(nn.Module):
412
413
params_dict = dict(self.named_parameters())
414
for name, loaded_weight in hf_model_weights_iterator(
415
- model_name_or_path, cache_dir, load_format, revision):
415
+ model_name_or_path,
416
+ cache_dir,
417
+ load_format,
418
+ revision,
419
+ fall_back_to_pt=False):
420
if "rotary_emb.inv_freq" in name:
421
continue
422
for (param_name, weight_name, shard_id) in stacked_params_mapping:
vllm/model_executor/weight_utils.py
+24
-27
@@ -125,15 +125,29 @@ def get_quant_config(
125
def prepare_hf_model_weights(
126
model_name_or_path: str,
127
cache_dir: Optional[str] = None,
128
- use_safetensors: bool = False,
128
+ load_format: str = "auto",
129
fall_back_to_pt: bool = True,
130
revision: Optional[str] = None,
131
) -> Tuple[str, List[str], bool]:
132
# Download model weights from huggingface.
133
is_local = os.path.isdir(model_name_or_path)
134
+ use_safetensors = False
135
# Some quantized models use .pt files for storing the weights.
135
- allow_patterns = ["*.safetensors"
136
- ] if use_safetensors else ["*.bin", "*.pt"]
136
+ if load_format == "auto":
137
+ allow_patterns = ["*.safetensors", "*.bin"]
138
+ elif load_format == "safetensors":
139
+ use_safetensors = True
140
+ allow_patterns = ["*.safetensors"]
141
+ elif load_format == "pt":
142
+ allow_patterns = ["*.pt"]
143
+ elif load_format == "npcache":
144
+ allow_patterns = ["*.bin"]
145
+ else:
146
+ raise ValueError(f"Unknown load_format: {load_format}")
147
+
148
+ if fall_back_to_pt:
149
+ allow_patterns += [".pt"]
150
+
151
if not is_local:
152
# Use file lock to prevent multiple processes from
153
# downloading the same model weights at the same time.
@@ -148,6 +162,10 @@ def prepare_hf_model_weights(
162
hf_weights_files: List[str] = []
163
for pattern in allow_patterns:
164
hf_weights_files += glob.glob(os.path.join(hf_folder, pattern))
165
+ if len(hf_weights_files) > 0:
166
+ if pattern == "*.safetensors":
167
+ use_safetensors = True
168
+ break
169
if not use_safetensors:
170
# Exclude files that are not needed for inference.
171
# https://github.com/huggingface/transformers/blob/v4.34.0/src/transformers/trainer.py#L227-L233
@@ -163,13 +181,6 @@ def prepare_hf_model_weights(
181
if not any(f.endswith(x) for x in blacklist)
182
]
183
166
- if len(hf_weights_files) == 0 and use_safetensors and fall_back_to_pt:
167
- return prepare_hf_model_weights(model_name_or_path,
168
- cache_dir=cache_dir,
169
- use_safetensors=False,
170
- fall_back_to_pt=False,
171
- revision=revision)
172
-
184
if len(hf_weights_files) == 0:
185
raise RuntimeError(
186
f"Cannot find any model weights with `{model_name_or_path}`")
@@ -182,30 +193,16 @@ def hf_model_weights_iterator(
193
cache_dir: Optional[str] = None,
194
load_format: str = "auto",
195
revision: Optional[str] = None,
196
+ fall_back_to_pt: Optional[bool] = True,
197
) -> Iterator[Tuple[str, torch.Tensor]]:
186
- use_safetensors = False
187
- use_np_cache = False
188
- fall_back_to_pt = False
189
- if load_format == "auto":
190
- use_safetensors = True
191
- fall_back_to_pt = True
192
- elif load_format == "safetensors":
193
- use_safetensors = True
194
- elif load_format == "pt":
195
- pass
196
- elif load_format == "npcache":
197
- use_np_cache = True
198
- else:
199
- raise ValueError(f"Unknown load_format: {load_format}")
200
-
198
hf_folder, hf_weights_files, use_safetensors = prepare_hf_model_weights(
199
model_name_or_path,
200
cache_dir=cache_dir,
204
- use_safetensors=use_safetensors,
201
+ load_format=load_format,
202
fall_back_to_pt=fall_back_to_pt,
203
revision=revision)
204
208
- if use_np_cache:
205
+ if load_format == "npcache":
206
# Currently np_cache only support *.bin checkpoints
207
assert use_safetensors is False
208