@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 919770957f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Bugfix] Fix Mistral v0.3 Weight Loading (#5005)

Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>

Robert Shaw committed May 24, 2024 at 14:28 UTC 919770957f26d71a5a6eda7a1a7443dfeb5ba0ee
3 files changed +79 -3
tests/models/test_mistral.py
+1
@@ -8,6 +8,7 @@ from .utils import check_logprobs_close
8
9 MODELS = [
10 "mistralai/Mistral-7B-Instruct-v0.1",
11 + "mistralai/Mistral-7B-Instruct-v0.3",
12 ]
13
14
vllm/model_executor/model_loader/loader.py
+15 -2
@@ -23,7 +23,8 @@ from vllm.model_executor.model_loader.tensorizer import (
23 from vllm.model_executor.model_loader.utils import (get_model_architecture,
24 set_default_torch_dtype)
25 from vllm.model_executor.model_loader.weight_utils import (
26 - download_weights_from_hf, filter_files_not_needed_for_inference,
26 + download_safetensors_index_file_from_hf, download_weights_from_hf,
27 + filter_duplicate_safetensors_files, filter_files_not_needed_for_inference,
28 get_quant_config, initialize_dummy_weights, np_cache_weights_iterator,
29 pt_weights_iterator, safetensors_weights_iterator)
30 from vllm.model_executor.models.vlm_base import VisionLanguageModelBase
@@ -188,7 +189,19 @@ class DefaultModelLoader(BaseModelLoader):
189 use_safetensors = True
190 break
191
191 - if not use_safetensors:
192 + if use_safetensors:
193 + # For models like Mistral-7B-Instruct-v0.3
194 + # there are both sharded safetensors files and a consolidated
195 + # safetensors file. Using both breaks.
196 + # Here, we download the `model.safetensors.index.json` and filter
197 + # any files not found in the index.
198 + if not is_local:
199 + download_safetensors_index_file_from_hf(
200 + model_name_or_path, self.load_config.download_dir,
201 + revision)
202 + hf_weights_files = filter_duplicate_safetensors_files(
203 + hf_weights_files, hf_folder)
204 + else:
205 hf_weights_files = filter_files_not_needed_for_inference(
206 hf_weights_files)
207
vllm/model_executor/model_loader/weight_utils.py
+63 -1
@@ -12,9 +12,10 @@ import filelock
12 import huggingface_hub.constants
13 import numpy as np
14 import torch
15 -from huggingface_hub import HfFileSystem, snapshot_download
15 +from huggingface_hub import HfFileSystem, hf_hub_download, snapshot_download
16 from safetensors.torch import load_file, safe_open, save_file
17 from tqdm.auto import tqdm
18 +from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
19
20 from vllm.config import LoadConfig, ModelConfig
21 from vllm.logger import init_logger
@@ -218,6 +219,67 @@ def download_weights_from_hf(
219 return hf_folder
220
221
222 +def download_safetensors_index_file_from_hf(
223 + model_name_or_path: str,
224 + cache_dir: Optional[str],
225 + revision: Optional[str] = None,
226 +) -> None:
227 + """Download hf safetensors index file from Hugging Face Hub.
228 +
229 + Args:
230 + model_name_or_path (str): The model name or path.
231 + cache_dir (Optional[str]): The cache directory to store the model
232 + weights. If None, will use HF defaults.
233 + revision (Optional[str]): The revision of the model.
234 + """
235 + # Use file lock to prevent multiple processes from
236 + # downloading the same model weights at the same time.
237 + with get_lock(model_name_or_path, cache_dir):
238 + try:
239 + # Download the safetensors index file.
240 + hf_hub_download(
241 + repo_id=model_name_or_path,
242 + filename=SAFE_WEIGHTS_INDEX_NAME,
243 + cache_dir=cache_dir,
244 + revision=revision,
245 + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
246 + )
247 + # If file not found on remote or locally, we should not fail since
248 + # only some models will have SAFE_WEIGHTS_INDEX_NAME.
249 + except huggingface_hub.utils.EntryNotFoundError:
250 + logger.info("No %s found in remote.", SAFE_WEIGHTS_INDEX_NAME)
251 + except huggingface_hub.utils.LocalEntryNotFoundError:
252 + logger.info("No %s found in local cache.", SAFE_WEIGHTS_INDEX_NAME)
253 +
254 +
255 +# For models like Mistral-7B-v0.3, there are both sharded
256 +# safetensors files and a consolidated safetensors file.
257 +# Passing both of these to the weight loader functionality breaks.
258 +# So, we use the SAFE_WEIGHTS_INDEX_NAME to
259 +# look up which safetensors files should be used.
260 +def filter_duplicate_safetensors_files(hf_weights_files: List[str],
261 + hf_folder: str) -> List[str]:
262 + # model.safetensors.index.json is a mapping from keys in the
263 + # torch state_dict to safetensors file holding that weight.
264 + index_file_name = os.path.join(hf_folder, SAFE_WEIGHTS_INDEX_NAME)
265 + if not os.path.isfile(index_file_name):
266 + return hf_weights_files
267 +
268 + # Iterate through the weight_map (weight_name: safetensors files)
269 + # to identify weights that we should use.
270 + with open(index_file_name) as index_file:
271 + weight_map = json.load(index_file)["weight_map"]
272 + weight_files_in_index = set()
273 + for weight_name in weight_map:
274 + weight_files_in_index.add(
275 + os.path.join(hf_folder, weight_map[weight_name]))
276 + # Filter out any fields that are not found in the index file.
277 + hf_weights_files = [
278 + f for f in hf_weights_files if f in weight_files_in_index
279 + ]
280 + return hf_weights_files
281 +
282 +
283 def filter_files_not_needed_for_inference(
284 hf_weights_files: List[str]) -> List[str]:
285 """