A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core] Support offline use of local cache for models (#4374)
Signed-off-by: Prashant Gupta <prashantgupta@us.ibm.com> Co-authored-by: Travis Johnson <tjohnson31415@gmail.com>
Prashant Gupta committed
Apr 27, 2024 at 09:59 UTC
d6e520e1700f78de2d5efdb8607a76cbab61182e
4 files changed
+69
-27
tests/model_executor/weight_utils.py
+29
-1
@@ -1,9 +1,12 @@
1
import os
2
+import tempfile
3
4
import huggingface_hub.constants
5
import pytest
6
+from huggingface_hub.utils import LocalEntryNotFoundError
7
6
-from vllm.model_executor.model_loader.weight_utils import enable_hf_transfer
8
+from vllm.model_executor.model_loader.weight_utils import (
9
+ download_weights_from_hf, enable_hf_transfer)
10
11
12
def test_hf_transfer_auto_activation():
@@ -22,5 +25,30 @@ def test_hf_transfer_auto_activation():
25
HF_TRANFER_ACTIVE)
26
27
28
+def test_download_weights_from_hf():
29
+ with tempfile.TemporaryDirectory() as tmpdir:
30
+ # assert LocalEntryNotFoundError error is thrown
31
+ # if offline is set and model is not cached
32
+ huggingface_hub.constants.HF_HUB_OFFLINE = True
33
+ with pytest.raises(LocalEntryNotFoundError):
34
+ download_weights_from_hf("facebook/opt-125m",
35
+ allow_patterns=["*.safetensors", "*.bin"],
36
+ cache_dir=tmpdir)
37
+
38
+ # download the model
39
+ huggingface_hub.constants.HF_HUB_OFFLINE = False
40
+ download_weights_from_hf("facebook/opt-125m",
41
+ allow_patterns=["*.safetensors", "*.bin"],
42
+ cache_dir=tmpdir)
43
+
44
+ # now it should work offline
45
+ huggingface_hub.constants.HF_HUB_OFFLINE = True
46
+ assert download_weights_from_hf(
47
+ "facebook/opt-125m",
48
+ allow_patterns=["*.safetensors", "*.bin"],
49
+ cache_dir=tmpdir) is not None
50
+
51
+
52
if __name__ == "__main__":
53
test_hf_transfer_auto_activation()
54
+ test_download_weights_from_hf()
vllm/model_executor/model_loader/loader.py
+4
-1
@@ -5,6 +5,7 @@ import os
5
from abc import ABC, abstractmethod
6
from typing import Any, Dict, Generator, List, Optional, Tuple, Type
7
8
+import huggingface_hub
9
import torch
10
from torch import nn
11
@@ -131,7 +132,9 @@ class DefaultModelLoader(BaseModelLoader):
132
model_path = snapshot_download(
133
model_id=model,
134
cache_dir=self.load_config.download_dir,
134
- revision=revision)
135
+ local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
136
+ revision=revision,
137
+ )
138
else:
139
model_path = model
140
return model_path
vllm/model_executor/model_loader/weight_utils.py
+34
-25
@@ -127,11 +127,14 @@ def get_quant_config(model_config: ModelConfig,
127
if not is_local:
128
# Download the config files.
129
with get_lock(model_name_or_path, load_config.download_dir):
130
- hf_folder = snapshot_download(model_name_or_path,
131
- revision=model_config.revision,
132
- allow_patterns="*.json",
133
- cache_dir=load_config.download_dir,
134
- tqdm_class=DisabledTqdm)
130
+ hf_folder = snapshot_download(
131
+ model_name_or_path,
132
+ revision=model_config.revision,
133
+ allow_patterns="*.json",
134
+ cache_dir=load_config.download_dir,
135
+ local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
136
+ tqdm_class=DisabledTqdm,
137
+ )
138
else:
139
hf_folder = model_name_or_path
140
@@ -161,12 +164,14 @@ def get_quant_config(model_config: ModelConfig,
164
return quant_cls.from_config(config)
165
166
164
-def download_weights_from_hf(model_name_or_path: str,
165
- cache_dir: Optional[str],
166
- allow_patterns: List[str],
167
- revision: Optional[str] = None) -> str:
167
+def download_weights_from_hf(
168
+ model_name_or_path: str,
169
+ cache_dir: Optional[str],
170
+ allow_patterns: List[str],
171
+ revision: Optional[str] = None,
172
+) -> str:
173
"""Download model weights from Hugging Face Hub.
169
-
174
+
175
Args:
176
model_name_or_path (str): The model name or path.
177
cache_dir (Optional[str]): The cache directory to store the model
@@ -179,26 +184,30 @@ def download_weights_from_hf(model_name_or_path: str,
184
Returns:
185
str: The path to the downloaded model weights.
186
"""
182
- # Before we download we look at that is available:
183
- fs = HfFileSystem()
184
- file_list = fs.ls(model_name_or_path, detail=False, revision=revision)
185
-
186
- # depending on what is available we download different things
187
- for pattern in allow_patterns:
188
- matching = fnmatch.filter(file_list, pattern)
189
- if len(matching) > 0:
190
- allow_patterns = [pattern]
191
- break
187
+ if not huggingface_hub.constants.HF_HUB_OFFLINE:
188
+ # Before we download we look at that is available:
189
+ fs = HfFileSystem()
190
+ file_list = fs.ls(model_name_or_path, detail=False, revision=revision)
191
+
192
+ # depending on what is available we download different things
193
+ for pattern in allow_patterns:
194
+ matching = fnmatch.filter(file_list, pattern)
195
+ if len(matching) > 0:
196
+ allow_patterns = [pattern]
197
+ break
198
199
logger.info("Using model weights format %s", allow_patterns)
200
# Use file lock to prevent multiple processes from
201
# downloading the same model weights at the same time.
202
with get_lock(model_name_or_path, cache_dir):
197
- hf_folder = snapshot_download(model_name_or_path,
198
- allow_patterns=allow_patterns,
199
- cache_dir=cache_dir,
200
- tqdm_class=DisabledTqdm,
201
- revision=revision)
203
+ hf_folder = snapshot_download(
204
+ model_name_or_path,
205
+ allow_patterns=allow_patterns,
206
+ cache_dir=cache_dir,
207
+ tqdm_class=DisabledTqdm,
208
+ revision=revision,
209
+ local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
210
+ )
211
return hf_folder
212
213
vllm/transformers_utils/tokenizer.py
+2
@@ -1,6 +1,7 @@
1
import os
2
from typing import Optional, Union
3
4
+import huggingface_hub
5
from transformers import (AutoTokenizer, PreTrainedTokenizer,
6
PreTrainedTokenizerFast)
7
@@ -76,6 +77,7 @@ def get_tokenizer(
77
model_id=tokenizer_name,
78
cache_dir=download_dir,
79
revision=revision,
80
+ local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
81
# Ignore weights - we only need the tokenizer.
82
ignore_file_pattern=["*.pt", "*.safetensors", "*.bin"])
83
tokenizer_name = tokenizer_path