A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
Prevent unecessary requests to huggingface hub (#12837)
Maximilien de Bayser committed
Feb 7, 2025 at 02:37 UTC
6e1fc61f0fb90c37f0d4a1a8f76235a6e4e1103c
2 files changed
+96
-40
tests/entrypoints/offline_mode/test_offline_mode.py
+21
@@ -4,6 +4,7 @@ import importlib
4
import sys
5
6
import pytest
7
+import urllib3
8
9
from vllm import LLM
10
from vllm.distributed import cleanup_dist_env_and_memory
@@ -28,6 +29,15 @@ MODEL_CONFIGS = [
29
"tensor_parallel_size": 1,
30
"tokenizer_mode": "mistral",
31
},
32
+ {
33
+ "model": "sentence-transformers/all-MiniLM-L12-v2",
34
+ "enforce_eager": True,
35
+ "gpu_memory_utilization": 0.20,
36
+ "max_model_len": 64,
37
+ "max_num_batched_tokens": 64,
38
+ "max_num_seqs": 64,
39
+ "tensor_parallel_size": 1,
40
+ },
41
]
42
43
@@ -47,6 +57,16 @@ def test_offline_mode(monkeypatch):
57
# Set HF to offline mode and ensure we can still construct an LLM
58
try:
59
monkeypatch.setenv("HF_HUB_OFFLINE", "1")
60
+ monkeypatch.setenv("VLLM_NO_USAGE_STATS", "1")
61
+
62
+ def disable_connect(*args, **kwargs):
63
+ raise RuntimeError("No http calls allowed")
64
+
65
+ monkeypatch.setattr(urllib3.connection.HTTPConnection, "connect",
66
+ disable_connect)
67
+ monkeypatch.setattr(urllib3.connection.HTTPSConnection, "connect",
68
+ disable_connect)
69
+
70
# Need to re-import huggingface_hub and friends to setup offline mode
71
_re_import_modules()
72
# Cached model files should be used in offline mode
@@ -56,6 +76,7 @@ def test_offline_mode(monkeypatch):
76
# Reset the environment after the test
77
# NB: Assuming tests are run in online mode
78
monkeypatch.delenv("HF_HUB_OFFLINE")
79
+ monkeypatch.delenv("VLLM_NO_USAGE_STATS")
80
_re_import_modules()
81
pass
82
vllm/transformers_utils/config.py
+75
-40
@@ -10,7 +10,7 @@ import huggingface_hub
10
from huggingface_hub import (file_exists, hf_hub_download, list_repo_files,
11
try_to_load_from_cache)
12
from huggingface_hub.utils import (EntryNotFoundError, HfHubHTTPError,
13
- LocalEntryNotFoundError,
13
+ HFValidationError, LocalEntryNotFoundError,
14
RepositoryNotFoundError,
15
RevisionNotFoundError)
16
from torch import nn
@@ -265,49 +265,66 @@ def get_config(
265
return config
266
267
268
+def try_get_local_file(model: Union[str, Path],
269
+ file_name: str,
270
+ revision: Optional[str] = 'main') -> Optional[Path]:
271
+ file_path = Path(model) / file_name
272
+ if file_path.is_file():
273
+ return file_path
274
+ else:
275
+ try:
276
+ cached_filepath = try_to_load_from_cache(repo_id=model,
277
+ filename=file_name,
278
+ revision=revision)
279
+ if isinstance(cached_filepath, str):
280
+ return Path(cached_filepath)
281
+ except HFValidationError:
282
+ ...
283
+ return None
284
+
285
+
286
def get_hf_file_to_dict(file_name: str,
287
model: Union[str, Path],
288
revision: Optional[str] = 'main'):
289
"""
272
- Downloads a file from the Hugging Face Hub and returns
290
+ Downloads a file from the Hugging Face Hub and returns
291
its contents as a dictionary.
292
293
Parameters:
294
- file_name (str): The name of the file to download.
295
- model (str): The name of the model on the Hugging Face Hub.
278
- - revision (str): The specific version of the model.
296
+ - revision (str): The specific version of the model.
297
298
Returns:
281
- - config_dict (dict): A dictionary containing
299
+ - config_dict (dict): A dictionary containing
300
the contents of the downloaded file.
301
"""
284
- file_path = Path(model) / file_name
302
286
- if file_or_path_exists(model=model,
287
- config_name=file_name,
288
- revision=revision):
303
+ file_path = try_get_local_file(model=model,
304
+ file_name=file_name,
305
+ revision=revision)
306
290
- if not file_path.is_file():
291
- try:
292
- hf_hub_file = hf_hub_download(model,
293
- file_name,
294
- revision=revision)
295
- except (RepositoryNotFoundError, RevisionNotFoundError,
296
- EntryNotFoundError, LocalEntryNotFoundError) as e:
297
- logger.debug("File or repository not found in hf_hub_download",
298
- e)
299
- return None
300
- except HfHubHTTPError as e:
301
- logger.warning(
302
- "Cannot connect to Hugging Face Hub. Skipping file "
303
- "download for '%s':",
304
- file_name,
305
- exc_info=e)
306
- return None
307
- file_path = Path(hf_hub_file)
307
+ if file_path is None and file_or_path_exists(
308
+ model=model, config_name=file_name, revision=revision):
309
+ try:
310
+ hf_hub_file = hf_hub_download(model, file_name, revision=revision)
311
+ except (RepositoryNotFoundError, RevisionNotFoundError,
312
+ EntryNotFoundError, LocalEntryNotFoundError) as e:
313
+ logger.debug("File or repository not found in hf_hub_download", e)
314
+ return None
315
+ except HfHubHTTPError as e:
316
+ logger.warning(
317
+ "Cannot connect to Hugging Face Hub. Skipping file "
318
+ "download for '%s':",
319
+ file_name,
320
+ exc_info=e)
321
+ return None
322
+ file_path = Path(hf_hub_file)
323
324
+ if file_path is not None and file_path.is_file():
325
with open(file_path) as file:
326
return json.load(file)
327
+
328
return None
329
330
@@ -328,7 +345,12 @@ def get_pooling_config(model: str, revision: Optional[str] = 'main'):
345
"""
346
347
modules_file_name = "modules.json"
331
- modules_dict = get_hf_file_to_dict(modules_file_name, model, revision)
348
+
349
+ modules_dict = None
350
+ if file_or_path_exists(model=model,
351
+ config_name=modules_file_name,
352
+ revision=revision):
353
+ modules_dict = get_hf_file_to_dict(modules_file_name, model, revision)
354
355
if modules_dict is None:
356
return None
@@ -382,17 +404,17 @@ def get_sentence_transformer_tokenizer_config(model: str,
404
revision: Optional[str] = 'main'
405
):
406
"""
385
- Returns the tokenization configuration dictionary for a
407
+ Returns the tokenization configuration dictionary for a
408
given Sentence Transformer BERT model.
409
410
Parameters:
389
- - model (str): The name of the Sentence Transformer
411
+ - model (str): The name of the Sentence Transformer
412
BERT model.
413
- revision (str, optional): The revision of the m
414
odel to use. Defaults to 'main'.
415
416
Returns:
395
- - dict: A dictionary containing the configuration parameters
417
+ - dict: A dictionary containing the configuration parameters
418
for the Sentence Transformer BERT model.
419
"""
420
sentence_transformer_config_files = [
@@ -404,20 +426,33 @@ def get_sentence_transformer_tokenizer_config(model: str,
426
"sentence_xlm-roberta_config.json",
427
"sentence_xlnet_config.json",
428
]
407
- try:
408
- # If model is on HuggingfaceHub, get the repo files
409
- repo_files = list_repo_files(model, revision=revision, token=HF_TOKEN)
410
- except Exception as e:
411
- logger.debug("Error getting repo files", e)
412
- repo_files = []
413
-
429
encoder_dict = None
415
- for config_name in sentence_transformer_config_files:
416
- if config_name in repo_files or Path(model).exists():
417
- encoder_dict = get_hf_file_to_dict(config_name, model, revision)
430
+
431
+ for config_file in sentence_transformer_config_files:
432
+ if try_get_local_file(model=model,
433
+ file_name=config_file,
434
+ revision=revision) is not None:
435
+ encoder_dict = get_hf_file_to_dict(config_file, model, revision)
436
if encoder_dict:
437
break
438
439
+ if not encoder_dict:
440
+ try:
441
+ # If model is on HuggingfaceHub, get the repo files
442
+ repo_files = list_repo_files(model,
443
+ revision=revision,
444
+ token=HF_TOKEN)
445
+ except Exception as e:
446
+ logger.debug("Error getting repo files", e)
447
+ repo_files = []
448
+
449
+ for config_name in sentence_transformer_config_files:
450
+ if config_name in repo_files:
451
+ encoder_dict = get_hf_file_to_dict(config_name, model,
452
+ revision)
453
+ if encoder_dict:
454
+ break
455
+
456
if not encoder_dict:
457
return None
458