A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[BugFix] Don't scan entire cache dir when loading model (#13302)
Nick Hill committed
Feb 14, 2025 at 21:33 UTC
907632567709760e059a252f95f240f1d0c40585
1 file changed
+8
-10
vllm/model_executor/model_loader/weight_utils.py
+8
-10
@@ -15,8 +15,7 @@ import gguf
15
import huggingface_hub.constants
16
import numpy as np
17
import torch
18
-from huggingface_hub import (HfFileSystem, hf_hub_download, scan_cache_dir,
19
- snapshot_download)
18
+from huggingface_hub import HfFileSystem, hf_hub_download, snapshot_download
19
from safetensors.torch import load_file, safe_open, save_file
20
from tqdm.auto import tqdm
21
@@ -239,7 +238,8 @@ def download_weights_from_hf(
238
Returns:
239
str: The path to the downloaded model weights.
240
"""
242
- if not huggingface_hub.constants.HF_HUB_OFFLINE:
241
+ local_only = huggingface_hub.constants.HF_HUB_OFFLINE
242
+ if not local_only:
243
# Before we download we look at that is available:
244
fs = HfFileSystem()
245
file_list = fs.ls(model_name_or_path, detail=False, revision=revision)
@@ -255,7 +255,6 @@ def download_weights_from_hf(
255
# Use file lock to prevent multiple processes from
256
# downloading the same model weights at the same time.
257
with get_lock(model_name_or_path, cache_dir):
258
- start_size = scan_cache_dir().size_on_disk
258
start_time = time.perf_counter()
259
hf_folder = snapshot_download(
260
model_name_or_path,
@@ -264,13 +263,12 @@ def download_weights_from_hf(
263
cache_dir=cache_dir,
264
tqdm_class=DisabledTqdm,
265
revision=revision,
267
- local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
266
+ local_files_only=local_only,
267
)
269
- end_time = time.perf_counter()
270
- end_size = scan_cache_dir().size_on_disk
271
- if end_size != start_size:
272
- logger.info("Time took to download weights for %s: %.6f seconds",
273
- model_name_or_path, end_time - start_time)
268
+ time_taken = time.perf_counter() - start_time
269
+ if time_taken > 0.5:
270
+ logger.info("Time spent downloading weights for %s: %.6f seconds",
271
+ model_name_or_path, time_taken)
272
return hf_folder
273
274