A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Misc] Log time consumption on weight downloading (#12926)
Jun Duan committed
Feb 8, 2025 at 04:16 UTC
011e612d92c25cb1a3cbfa1536cb8edd871d7715
1 file changed
+10
-1
vllm/model_executor/model_loader/weight_utils.py
+10
-1
@@ -6,6 +6,7 @@ import hashlib
6
import json
7
import os
8
import tempfile
9
+import time
10
from collections import defaultdict
11
from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union
12
@@ -14,7 +15,8 @@ import gguf
15
import huggingface_hub.constants
16
import numpy as np
17
import torch
17
-from huggingface_hub import HfFileSystem, hf_hub_download, snapshot_download
18
+from huggingface_hub import (HfFileSystem, hf_hub_download, scan_cache_dir,
19
+ snapshot_download)
20
from safetensors.torch import load_file, safe_open, save_file
21
from tqdm.auto import tqdm
22
@@ -253,6 +255,8 @@ def download_weights_from_hf(
255
# Use file lock to prevent multiple processes from
256
# downloading the same model weights at the same time.
257
with get_lock(model_name_or_path, cache_dir):
258
+ start_size = scan_cache_dir().size_on_disk
259
+ start_time = time.perf_counter()
260
hf_folder = snapshot_download(
261
model_name_or_path,
262
allow_patterns=allow_patterns,
@@ -262,6 +266,11 @@ def download_weights_from_hf(
266
revision=revision,
267
local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
268
)
269
+ end_time = time.perf_counter()
270
+ end_size = scan_cache_dir().size_on_disk
271
+ if end_size != start_size:
272
+ logger.info("Time took to download weights for %s: %.6f seconds",
273
+ model_name_or_path, end_time - start_time)
274
return hf_folder
275
276