A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Don't download both safetensor and bin files. (#2480)
Nikola Borisov committed
Jan 18, 2024 at 11:05 UTC
7e1081139d65e941815234aa0e0e91e1ea5c0ff1
1 file changed
+16
-3
vllm/model_executor/weight_utils.py
+16
-3
@@ -1,12 +1,13 @@
1
"""Utilities for downloading and initializing model weights."""
2
import filelock
3
import glob
4
+import fnmatch
5
import json
6
import os
7
from collections import defaultdict
8
from typing import Any, Iterator, List, Optional, Tuple
9
9
-from huggingface_hub import snapshot_download
10
+from huggingface_hub import snapshot_download, HfFileSystem
11
import numpy as np
12
from safetensors.torch import load_file, save_file, safe_open
13
import torch
@@ -149,6 +150,20 @@ def prepare_hf_model_weights(
150
allow_patterns += ["*.pt"]
151
152
if not is_local:
153
+ # Before we download we look at that is available:
154
+ fs = HfFileSystem()
155
+ file_list = fs.ls(model_name_or_path, detail=False, revision=revision)
156
+
157
+ # depending on what is available we download different things
158
+ for pattern in allow_patterns:
159
+ matching = fnmatch.filter(file_list, pattern)
160
+ if len(matching) > 0:
161
+ allow_patterns = [pattern]
162
+ if pattern == "*.safetensors":
163
+ use_safetensors = True
164
+ break
165
+
166
+ logger.info(f"Downloading model weights {allow_patterns}")
167
# Use file lock to prevent multiple processes from
168
# downloading the same model weights at the same time.
169
with get_lock(model_name_or_path, cache_dir):
@@ -163,8 +178,6 @@ def prepare_hf_model_weights(
178
for pattern in allow_patterns:
179
hf_weights_files += glob.glob(os.path.join(hf_folder, pattern))
180
if len(hf_weights_files) > 0:
166
- if pattern == "*.safetensors":
167
- use_safetensors = True
181
break
182
if not use_safetensors:
183
# Exclude files that are not needed for inference.