A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Bugfix] Fix Mistral v0.3 Weight Loading (#5005)
Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>
Robert Shaw committed
May 24, 2024 at 14:28 UTC
919770957f26d71a5a6eda7a1a7443dfeb5ba0ee
3 files changed
+79
-3
tests/models/test_mistral.py
+1
@@ -8,6 +8,7 @@ from .utils import check_logprobs_close
8
9
MODELS = [
10
"mistralai/Mistral-7B-Instruct-v0.1",
11
+ "mistralai/Mistral-7B-Instruct-v0.3",
12
]
13
14
vllm/model_executor/model_loader/loader.py
+15
-2
@@ -23,7 +23,8 @@ from vllm.model_executor.model_loader.tensorizer import (
23
from vllm.model_executor.model_loader.utils import (get_model_architecture,
24
set_default_torch_dtype)
25
from vllm.model_executor.model_loader.weight_utils import (
26
- download_weights_from_hf, filter_files_not_needed_for_inference,
26
+ download_safetensors_index_file_from_hf, download_weights_from_hf,
27
+ filter_duplicate_safetensors_files, filter_files_not_needed_for_inference,
28
get_quant_config, initialize_dummy_weights, np_cache_weights_iterator,
29
pt_weights_iterator, safetensors_weights_iterator)
30
from vllm.model_executor.models.vlm_base import VisionLanguageModelBase
@@ -188,7 +189,19 @@ class DefaultModelLoader(BaseModelLoader):
189
use_safetensors = True
190
break
191
191
- if not use_safetensors:
192
+ if use_safetensors:
193
+ # For models like Mistral-7B-Instruct-v0.3
194
+ # there are both sharded safetensors files and a consolidated
195
+ # safetensors file. Using both breaks.
196
+ # Here, we download the `model.safetensors.index.json` and filter
197
+ # any files not found in the index.
198
+ if not is_local:
199
+ download_safetensors_index_file_from_hf(
200
+ model_name_or_path, self.load_config.download_dir,
201
+ revision)
202
+ hf_weights_files = filter_duplicate_safetensors_files(
203
+ hf_weights_files, hf_folder)
204
+ else:
205
hf_weights_files = filter_files_not_needed_for_inference(
206
hf_weights_files)
207
vllm/model_executor/model_loader/weight_utils.py
+63
-1
@@ -12,9 +12,10 @@ import filelock
12
import huggingface_hub.constants
13
import numpy as np
14
import torch
15
-from huggingface_hub import HfFileSystem, snapshot_download
15
+from huggingface_hub import HfFileSystem, hf_hub_download, snapshot_download
16
from safetensors.torch import load_file, safe_open, save_file
17
from tqdm.auto import tqdm
18
+from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
19
20
from vllm.config import LoadConfig, ModelConfig
21
from vllm.logger import init_logger
@@ -218,6 +219,67 @@ def download_weights_from_hf(
219
return hf_folder
220
221
222
+def download_safetensors_index_file_from_hf(
223
+ model_name_or_path: str,
224
+ cache_dir: Optional[str],
225
+ revision: Optional[str] = None,
226
+) -> None:
227
+ """Download hf safetensors index file from Hugging Face Hub.
228
+
229
+ Args:
230
+ model_name_or_path (str): The model name or path.
231
+ cache_dir (Optional[str]): The cache directory to store the model
232
+ weights. If None, will use HF defaults.
233
+ revision (Optional[str]): The revision of the model.
234
+ """
235
+ # Use file lock to prevent multiple processes from
236
+ # downloading the same model weights at the same time.
237
+ with get_lock(model_name_or_path, cache_dir):
238
+ try:
239
+ # Download the safetensors index file.
240
+ hf_hub_download(
241
+ repo_id=model_name_or_path,
242
+ filename=SAFE_WEIGHTS_INDEX_NAME,
243
+ cache_dir=cache_dir,
244
+ revision=revision,
245
+ local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
246
+ )
247
+ # If file not found on remote or locally, we should not fail since
248
+ # only some models will have SAFE_WEIGHTS_INDEX_NAME.
249
+ except huggingface_hub.utils.EntryNotFoundError:
250
+ logger.info("No %s found in remote.", SAFE_WEIGHTS_INDEX_NAME)
251
+ except huggingface_hub.utils.LocalEntryNotFoundError:
252
+ logger.info("No %s found in local cache.", SAFE_WEIGHTS_INDEX_NAME)
253
+
254
+
255
+# For models like Mistral-7B-v0.3, there are both sharded
256
+# safetensors files and a consolidated safetensors file.
257
+# Passing both of these to the weight loader functionality breaks.
258
+# So, we use the SAFE_WEIGHTS_INDEX_NAME to
259
+# look up which safetensors files should be used.
260
+def filter_duplicate_safetensors_files(hf_weights_files: List[str],
261
+ hf_folder: str) -> List[str]:
262
+ # model.safetensors.index.json is a mapping from keys in the
263
+ # torch state_dict to safetensors file holding that weight.
264
+ index_file_name = os.path.join(hf_folder, SAFE_WEIGHTS_INDEX_NAME)
265
+ if not os.path.isfile(index_file_name):
266
+ return hf_weights_files
267
+
268
+ # Iterate through the weight_map (weight_name: safetensors files)
269
+ # to identify weights that we should use.
270
+ with open(index_file_name) as index_file:
271
+ weight_map = json.load(index_file)["weight_map"]
272
+ weight_files_in_index = set()
273
+ for weight_name in weight_map:
274
+ weight_files_in_index.add(
275
+ os.path.join(hf_folder, weight_map[weight_name]))
276
+ # Filter out any fields that are not found in the index file.
277
+ hf_weights_files = [
278
+ f for f in hf_weights_files if f in weight_files_in_index
279
+ ]
280
+ return hf_weights_files
281
+
282
+
283
def filter_files_not_needed_for_inference(
284
hf_weights_files: List[str]) -> List[str]:
285
"""