@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 011e612d92
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Misc] Log time consumption on weight downloading (#12926)

Jun Duan committed Feb 8, 2025 at 04:16 UTC 011e612d92c25cb1a3cbfa1536cb8edd871d7715
1 file changed +10 -1
vllm/model_executor/model_loader/weight_utils.py
+10 -1
@@ -6,6 +6,7 @@ import hashlib
6 import json
7 import os
8 import tempfile
9 +import time
10 from collections import defaultdict
11 from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union
12
@@ -14,7 +15,8 @@ import gguf
15 import huggingface_hub.constants
16 import numpy as np
17 import torch
17 -from huggingface_hub import HfFileSystem, hf_hub_download, snapshot_download
18 +from huggingface_hub import (HfFileSystem, hf_hub_download, scan_cache_dir,
19 + snapshot_download)
20 from safetensors.torch import load_file, safe_open, save_file
21 from tqdm.auto import tqdm
22
@@ -253,6 +255,8 @@ def download_weights_from_hf(
255 # Use file lock to prevent multiple processes from
256 # downloading the same model weights at the same time.
257 with get_lock(model_name_or_path, cache_dir):
258 + start_size = scan_cache_dir().size_on_disk
259 + start_time = time.perf_counter()
260 hf_folder = snapshot_download(
261 model_name_or_path,
262 allow_patterns=allow_patterns,
@@ -262,6 +266,11 @@ def download_weights_from_hf(
266 revision=revision,
267 local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
268 )
269 + end_time = time.perf_counter()
270 + end_size = scan_cache_dir().size_on_disk
271 + if end_size != start_size:
272 + logger.info("Time took to download weights for %s: %.6f seconds",
273 + model_name_or_path, end_time - start_time)
274 return hf_folder
275
276