@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d6e520e170
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core] Support offline use of local cache for models (#4374)

Signed-off-by: Prashant Gupta <prashantgupta@us.ibm.com> Co-authored-by: Travis Johnson <tjohnson31415@gmail.com>

Prashant Gupta committed Apr 27, 2024 at 09:59 UTC d6e520e1700f78de2d5efdb8607a76cbab61182e
4 files changed +69 -27
tests/model_executor/weight_utils.py
+29 -1
@@ -1,9 +1,12 @@
1 import os
2 +import tempfile
3
4 import huggingface_hub.constants
5 import pytest
6 +from huggingface_hub.utils import LocalEntryNotFoundError
7
6 -from vllm.model_executor.model_loader.weight_utils import enable_hf_transfer
8 +from vllm.model_executor.model_loader.weight_utils import (
9 + download_weights_from_hf, enable_hf_transfer)
10
11
12 def test_hf_transfer_auto_activation():
@@ -22,5 +25,30 @@ def test_hf_transfer_auto_activation():
25 HF_TRANFER_ACTIVE)
26
27
28 +def test_download_weights_from_hf():
29 + with tempfile.TemporaryDirectory() as tmpdir:
30 + # assert LocalEntryNotFoundError error is thrown
31 + # if offline is set and model is not cached
32 + huggingface_hub.constants.HF_HUB_OFFLINE = True
33 + with pytest.raises(LocalEntryNotFoundError):
34 + download_weights_from_hf("facebook/opt-125m",
35 + allow_patterns=["*.safetensors", "*.bin"],
36 + cache_dir=tmpdir)
37 +
38 + # download the model
39 + huggingface_hub.constants.HF_HUB_OFFLINE = False
40 + download_weights_from_hf("facebook/opt-125m",
41 + allow_patterns=["*.safetensors", "*.bin"],
42 + cache_dir=tmpdir)
43 +
44 + # now it should work offline
45 + huggingface_hub.constants.HF_HUB_OFFLINE = True
46 + assert download_weights_from_hf(
47 + "facebook/opt-125m",
48 + allow_patterns=["*.safetensors", "*.bin"],
49 + cache_dir=tmpdir) is not None
50 +
51 +
52 if __name__ == "__main__":
53 test_hf_transfer_auto_activation()
54 + test_download_weights_from_hf()
vllm/model_executor/model_loader/loader.py
+4 -1
@@ -5,6 +5,7 @@ import os
5 from abc import ABC, abstractmethod
6 from typing import Any, Dict, Generator, List, Optional, Tuple, Type
7
8 +import huggingface_hub
9 import torch
10 from torch import nn
11
@@ -131,7 +132,9 @@ class DefaultModelLoader(BaseModelLoader):
132 model_path = snapshot_download(
133 model_id=model,
134 cache_dir=self.load_config.download_dir,
134 - revision=revision)
135 + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
136 + revision=revision,
137 + )
138 else:
139 model_path = model
140 return model_path
vllm/model_executor/model_loader/weight_utils.py
+34 -25
@@ -127,11 +127,14 @@ def get_quant_config(model_config: ModelConfig,
127 if not is_local:
128 # Download the config files.
129 with get_lock(model_name_or_path, load_config.download_dir):
130 - hf_folder = snapshot_download(model_name_or_path,
131 - revision=model_config.revision,
132 - allow_patterns="*.json",
133 - cache_dir=load_config.download_dir,
134 - tqdm_class=DisabledTqdm)
130 + hf_folder = snapshot_download(
131 + model_name_or_path,
132 + revision=model_config.revision,
133 + allow_patterns="*.json",
134 + cache_dir=load_config.download_dir,
135 + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
136 + tqdm_class=DisabledTqdm,
137 + )
138 else:
139 hf_folder = model_name_or_path
140
@@ -161,12 +164,14 @@ def get_quant_config(model_config: ModelConfig,
164 return quant_cls.from_config(config)
165
166
164 -def download_weights_from_hf(model_name_or_path: str,
165 - cache_dir: Optional[str],
166 - allow_patterns: List[str],
167 - revision: Optional[str] = None) -> str:
167 +def download_weights_from_hf(
168 + model_name_or_path: str,
169 + cache_dir: Optional[str],
170 + allow_patterns: List[str],
171 + revision: Optional[str] = None,
172 +) -> str:
173 """Download model weights from Hugging Face Hub.
169 -
174 +
175 Args:
176 model_name_or_path (str): The model name or path.
177 cache_dir (Optional[str]): The cache directory to store the model
@@ -179,26 +184,30 @@ def download_weights_from_hf(model_name_or_path: str,
184 Returns:
185 str: The path to the downloaded model weights.
186 """
182 - # Before we download we look at that is available:
183 - fs = HfFileSystem()
184 - file_list = fs.ls(model_name_or_path, detail=False, revision=revision)
185 -
186 - # depending on what is available we download different things
187 - for pattern in allow_patterns:
188 - matching = fnmatch.filter(file_list, pattern)
189 - if len(matching) > 0:
190 - allow_patterns = [pattern]
191 - break
187 + if not huggingface_hub.constants.HF_HUB_OFFLINE:
188 + # Before we download we look at that is available:
189 + fs = HfFileSystem()
190 + file_list = fs.ls(model_name_or_path, detail=False, revision=revision)
191 +
192 + # depending on what is available we download different things
193 + for pattern in allow_patterns:
194 + matching = fnmatch.filter(file_list, pattern)
195 + if len(matching) > 0:
196 + allow_patterns = [pattern]
197 + break
198
199 logger.info("Using model weights format %s", allow_patterns)
200 # Use file lock to prevent multiple processes from
201 # downloading the same model weights at the same time.
202 with get_lock(model_name_or_path, cache_dir):
197 - hf_folder = snapshot_download(model_name_or_path,
198 - allow_patterns=allow_patterns,
199 - cache_dir=cache_dir,
200 - tqdm_class=DisabledTqdm,
201 - revision=revision)
203 + hf_folder = snapshot_download(
204 + model_name_or_path,
205 + allow_patterns=allow_patterns,
206 + cache_dir=cache_dir,
207 + tqdm_class=DisabledTqdm,
208 + revision=revision,
209 + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
210 + )
211 return hf_folder
212
213
vllm/transformers_utils/tokenizer.py
+2
@@ -1,6 +1,7 @@
1 import os
2 from typing import Optional, Union
3
4 +import huggingface_hub
5 from transformers import (AutoTokenizer, PreTrainedTokenizer,
6 PreTrainedTokenizerFast)
7
@@ -76,6 +77,7 @@ def get_tokenizer(
77 model_id=tokenizer_name,
78 cache_dir=download_dir,
79 revision=revision,
80 + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
81 # Ignore weights - we only need the tokenizer.
82 ignore_file_pattern=["*.pt", "*.safetensors", "*.bin"])
83 tokenizer_name = tokenizer_path