@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 537ee25f43
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core] Enable hf_transfer by default if available (#3817)

Michael Feil committed Apr 3, 2024 at 21:02 UTC 537ee25f43afbb51be6984a9ce781f28fafaf373
2 files changed +42
tests/model_executor/weight_utils.py new
+26
@@ -0,0 +1,26 @@
1 +import os
2 +
3 +import huggingface_hub.constants
4 +import pytest
5 +
6 +from vllm.model_executor.weight_utils import enable_hf_transfer
7 +
8 +
9 +def test_hf_transfer_auto_activation():
10 + if "HF_HUB_ENABLE_HF_TRANSFER" in os.environ:
11 + # in case it is already set, we can't test the auto activation
12 + pytest.skip(
13 + "HF_HUB_ENABLE_HF_TRANSFER is set, can't test auto activation")
14 + enable_hf_transfer()
15 + try:
16 + # enable hf hub transfer if available
17 + import hf_transfer # type: ignore # noqa
18 + HF_TRANFER_ACTIVE = True
19 + except ImportError:
20 + HF_TRANFER_ACTIVE = False
21 + assert (huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER ==
22 + HF_TRANFER_ACTIVE)
23 +
24 +
25 +if __name__ == "__main__":
26 + test_hf_transfer_auto_activation()
vllm/model_executor/weight_utils.py
+16
@@ -8,6 +8,7 @@ from collections import defaultdict
8 from typing import Any, Iterable, Iterator, List, Optional, Tuple
9
10 import filelock
11 +import huggingface_hub.constants
12 import numpy as np
13 import torch
14 from huggingface_hub import HfFileSystem, snapshot_download
@@ -30,6 +31,21 @@ temp_dir = os.environ.get('TMPDIR') or os.environ.get(
31 'TEMP') or os.environ.get('TMP') or "/tmp/"
32
33
34 +def enable_hf_transfer():
35 + """automatically activates hf_transfer
36 + """
37 + if "HF_HUB_ENABLE_HF_TRANSFER" not in os.environ:
38 + try:
39 + # enable hf hub transfer if available
40 + import hf_transfer # type: ignore # noqa
41 + huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER = True
42 + except ImportError:
43 + pass
44 +
45 +
46 +enable_hf_transfer()
47 +
48 +
49 class Disabledtqdm(tqdm):
50
51 def __init__(self, *args, **kwargs):