A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core] Enable hf_transfer by default if available (#3817)
Michael Feil committed
Apr 3, 2024 at 21:02 UTC
537ee25f43afbb51be6984a9ce781f28fafaf373
2 files changed
+42
tests/model_executor/weight_utils.py
new
+26
@@ -0,0 +1,26 @@
1
+import os
2
+
3
+import huggingface_hub.constants
4
+import pytest
5
+
6
+from vllm.model_executor.weight_utils import enable_hf_transfer
7
+
8
+
9
+def test_hf_transfer_auto_activation():
10
+ if "HF_HUB_ENABLE_HF_TRANSFER" in os.environ:
11
+ # in case it is already set, we can't test the auto activation
12
+ pytest.skip(
13
+ "HF_HUB_ENABLE_HF_TRANSFER is set, can't test auto activation")
14
+ enable_hf_transfer()
15
+ try:
16
+ # enable hf hub transfer if available
17
+ import hf_transfer # type: ignore # noqa
18
+ HF_TRANFER_ACTIVE = True
19
+ except ImportError:
20
+ HF_TRANFER_ACTIVE = False
21
+ assert (huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER ==
22
+ HF_TRANFER_ACTIVE)
23
+
24
+
25
+if __name__ == "__main__":
26
+ test_hf_transfer_auto_activation()
vllm/model_executor/weight_utils.py
+16
@@ -8,6 +8,7 @@ from collections import defaultdict
8
from typing import Any, Iterable, Iterator, List, Optional, Tuple
9
10
import filelock
11
+import huggingface_hub.constants
12
import numpy as np
13
import torch
14
from huggingface_hub import HfFileSystem, snapshot_download
@@ -30,6 +31,21 @@ temp_dir = os.environ.get('TMPDIR') or os.environ.get(
31
'TEMP') or os.environ.get('TMP') or "/tmp/"
32
33
34
+def enable_hf_transfer():
35
+ """automatically activates hf_transfer
36
+ """
37
+ if "HF_HUB_ENABLE_HF_TRANSFER" not in os.environ:
38
+ try:
39
+ # enable hf hub transfer if available
40
+ import hf_transfer # type: ignore # noqa
41
+ huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER = True
42
+ except ImportError:
43
+ pass
44
+
45
+
46
+enable_hf_transfer()
47
+
48
+
49
class Disabledtqdm(tqdm):
50
51
def __init__(self, *args, **kwargs):