@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7e1081139d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Don't download both safetensor and bin files. (#2480)

Nikola Borisov committed Jan 18, 2024 at 11:05 UTC 7e1081139d65e941815234aa0e0e91e1ea5c0ff1
1 file changed +16 -3
vllm/model_executor/weight_utils.py
+16 -3
@@ -1,12 +1,13 @@
1 """Utilities for downloading and initializing model weights."""
2 import filelock
3 import glob
4 +import fnmatch
5 import json
6 import os
7 from collections import defaultdict
8 from typing import Any, Iterator, List, Optional, Tuple
9
9 -from huggingface_hub import snapshot_download
10 +from huggingface_hub import snapshot_download, HfFileSystem
11 import numpy as np
12 from safetensors.torch import load_file, save_file, safe_open
13 import torch
@@ -149,6 +150,20 @@ def prepare_hf_model_weights(
150 allow_patterns += ["*.pt"]
151
152 if not is_local:
153 + # Before we download we look at that is available:
154 + fs = HfFileSystem()
155 + file_list = fs.ls(model_name_or_path, detail=False, revision=revision)
156 +
157 + # depending on what is available we download different things
158 + for pattern in allow_patterns:
159 + matching = fnmatch.filter(file_list, pattern)
160 + if len(matching) > 0:
161 + allow_patterns = [pattern]
162 + if pattern == "*.safetensors":
163 + use_safetensors = True
164 + break
165 +
166 + logger.info(f"Downloading model weights {allow_patterns}")
167 # Use file lock to prevent multiple processes from
168 # downloading the same model weights at the same time.
169 with get_lock(model_name_or_path, cache_dir):
@@ -163,8 +178,6 @@ def prepare_hf_model_weights(
178 for pattern in allow_patterns:
179 hf_weights_files += glob.glob(os.path.join(hf_folder, pattern))
180 if len(hf_weights_files) > 0:
166 - if pattern == "*.safetensors":
167 - use_safetensors = True
181 break
182 if not use_safetensors:
183 # Exclude files that are not needed for inference.