@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6e1fc61f0f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

Prevent unecessary requests to huggingface hub (#12837)

Maximilien de Bayser committed Feb 7, 2025 at 02:37 UTC 6e1fc61f0fb90c37f0d4a1a8f76235a6e4e1103c
2 files changed +96 -40
tests/entrypoints/offline_mode/test_offline_mode.py
+21
@@ -4,6 +4,7 @@ import importlib
4 import sys
5
6 import pytest
7 +import urllib3
8
9 from vllm import LLM
10 from vllm.distributed import cleanup_dist_env_and_memory
@@ -28,6 +29,15 @@ MODEL_CONFIGS = [
29 "tensor_parallel_size": 1,
30 "tokenizer_mode": "mistral",
31 },
32 + {
33 + "model": "sentence-transformers/all-MiniLM-L12-v2",
34 + "enforce_eager": True,
35 + "gpu_memory_utilization": 0.20,
36 + "max_model_len": 64,
37 + "max_num_batched_tokens": 64,
38 + "max_num_seqs": 64,
39 + "tensor_parallel_size": 1,
40 + },
41 ]
42
43
@@ -47,6 +57,16 @@ def test_offline_mode(monkeypatch):
57 # Set HF to offline mode and ensure we can still construct an LLM
58 try:
59 monkeypatch.setenv("HF_HUB_OFFLINE", "1")
60 + monkeypatch.setenv("VLLM_NO_USAGE_STATS", "1")
61 +
62 + def disable_connect(*args, **kwargs):
63 + raise RuntimeError("No http calls allowed")
64 +
65 + monkeypatch.setattr(urllib3.connection.HTTPConnection, "connect",
66 + disable_connect)
67 + monkeypatch.setattr(urllib3.connection.HTTPSConnection, "connect",
68 + disable_connect)
69 +
70 # Need to re-import huggingface_hub and friends to setup offline mode
71 _re_import_modules()
72 # Cached model files should be used in offline mode
@@ -56,6 +76,7 @@ def test_offline_mode(monkeypatch):
76 # Reset the environment after the test
77 # NB: Assuming tests are run in online mode
78 monkeypatch.delenv("HF_HUB_OFFLINE")
79 + monkeypatch.delenv("VLLM_NO_USAGE_STATS")
80 _re_import_modules()
81 pass
82
vllm/transformers_utils/config.py
+75 -40
@@ -10,7 +10,7 @@ import huggingface_hub
10 from huggingface_hub import (file_exists, hf_hub_download, list_repo_files,
11 try_to_load_from_cache)
12 from huggingface_hub.utils import (EntryNotFoundError, HfHubHTTPError,
13 - LocalEntryNotFoundError,
13 + HFValidationError, LocalEntryNotFoundError,
14 RepositoryNotFoundError,
15 RevisionNotFoundError)
16 from torch import nn
@@ -265,49 +265,66 @@ def get_config(
265 return config
266
267
268 +def try_get_local_file(model: Union[str, Path],
269 + file_name: str,
270 + revision: Optional[str] = 'main') -> Optional[Path]:
271 + file_path = Path(model) / file_name
272 + if file_path.is_file():
273 + return file_path
274 + else:
275 + try:
276 + cached_filepath = try_to_load_from_cache(repo_id=model,
277 + filename=file_name,
278 + revision=revision)
279 + if isinstance(cached_filepath, str):
280 + return Path(cached_filepath)
281 + except HFValidationError:
282 + ...
283 + return None
284 +
285 +
286 def get_hf_file_to_dict(file_name: str,
287 model: Union[str, Path],
288 revision: Optional[str] = 'main'):
289 """
272 - Downloads a file from the Hugging Face Hub and returns
290 + Downloads a file from the Hugging Face Hub and returns
291 its contents as a dictionary.
292
293 Parameters:
294 - file_name (str): The name of the file to download.
295 - model (str): The name of the model on the Hugging Face Hub.
278 - - revision (str): The specific version of the model.
296 + - revision (str): The specific version of the model.
297
298 Returns:
281 - - config_dict (dict): A dictionary containing
299 + - config_dict (dict): A dictionary containing
300 the contents of the downloaded file.
301 """
284 - file_path = Path(model) / file_name
302
286 - if file_or_path_exists(model=model,
287 - config_name=file_name,
288 - revision=revision):
303 + file_path = try_get_local_file(model=model,
304 + file_name=file_name,
305 + revision=revision)
306
290 - if not file_path.is_file():
291 - try:
292 - hf_hub_file = hf_hub_download(model,
293 - file_name,
294 - revision=revision)
295 - except (RepositoryNotFoundError, RevisionNotFoundError,
296 - EntryNotFoundError, LocalEntryNotFoundError) as e:
297 - logger.debug("File or repository not found in hf_hub_download",
298 - e)
299 - return None
300 - except HfHubHTTPError as e:
301 - logger.warning(
302 - "Cannot connect to Hugging Face Hub. Skipping file "
303 - "download for '%s':",
304 - file_name,
305 - exc_info=e)
306 - return None
307 - file_path = Path(hf_hub_file)
307 + if file_path is None and file_or_path_exists(
308 + model=model, config_name=file_name, revision=revision):
309 + try:
310 + hf_hub_file = hf_hub_download(model, file_name, revision=revision)
311 + except (RepositoryNotFoundError, RevisionNotFoundError,
312 + EntryNotFoundError, LocalEntryNotFoundError) as e:
313 + logger.debug("File or repository not found in hf_hub_download", e)
314 + return None
315 + except HfHubHTTPError as e:
316 + logger.warning(
317 + "Cannot connect to Hugging Face Hub. Skipping file "
318 + "download for '%s':",
319 + file_name,
320 + exc_info=e)
321 + return None
322 + file_path = Path(hf_hub_file)
323
324 + if file_path is not None and file_path.is_file():
325 with open(file_path) as file:
326 return json.load(file)
327 +
328 return None
329
330
@@ -328,7 +345,12 @@ def get_pooling_config(model: str, revision: Optional[str] = 'main'):
345 """
346
347 modules_file_name = "modules.json"
331 - modules_dict = get_hf_file_to_dict(modules_file_name, model, revision)
348 +
349 + modules_dict = None
350 + if file_or_path_exists(model=model,
351 + config_name=modules_file_name,
352 + revision=revision):
353 + modules_dict = get_hf_file_to_dict(modules_file_name, model, revision)
354
355 if modules_dict is None:
356 return None
@@ -382,17 +404,17 @@ def get_sentence_transformer_tokenizer_config(model: str,
404 revision: Optional[str] = 'main'
405 ):
406 """
385 - Returns the tokenization configuration dictionary for a
407 + Returns the tokenization configuration dictionary for a
408 given Sentence Transformer BERT model.
409
410 Parameters:
389 - - model (str): The name of the Sentence Transformer
411 + - model (str): The name of the Sentence Transformer
412 BERT model.
413 - revision (str, optional): The revision of the m
414 odel to use. Defaults to 'main'.
415
416 Returns:
395 - - dict: A dictionary containing the configuration parameters
417 + - dict: A dictionary containing the configuration parameters
418 for the Sentence Transformer BERT model.
419 """
420 sentence_transformer_config_files = [
@@ -404,20 +426,33 @@ def get_sentence_transformer_tokenizer_config(model: str,
426 "sentence_xlm-roberta_config.json",
427 "sentence_xlnet_config.json",
428 ]
407 - try:
408 - # If model is on HuggingfaceHub, get the repo files
409 - repo_files = list_repo_files(model, revision=revision, token=HF_TOKEN)
410 - except Exception as e:
411 - logger.debug("Error getting repo files", e)
412 - repo_files = []
413 -
429 encoder_dict = None
415 - for config_name in sentence_transformer_config_files:
416 - if config_name in repo_files or Path(model).exists():
417 - encoder_dict = get_hf_file_to_dict(config_name, model, revision)
430 +
431 + for config_file in sentence_transformer_config_files:
432 + if try_get_local_file(model=model,
433 + file_name=config_file,
434 + revision=revision) is not None:
435 + encoder_dict = get_hf_file_to_dict(config_file, model, revision)
436 if encoder_dict:
437 break
438
439 + if not encoder_dict:
440 + try:
441 + # If model is on HuggingfaceHub, get the repo files
442 + repo_files = list_repo_files(model,
443 + revision=revision,
444 + token=HF_TOKEN)
445 + except Exception as e:
446 + logger.debug("Error getting repo files", e)
447 + repo_files = []
448 +
449 + for config_name in sentence_transformer_config_files:
450 + if config_name in repo_files:
451 + encoder_dict = get_hf_file_to_dict(config_name, model,
452 + revision)
453 + if encoder_dict:
454 + break
455 +
456 if not encoder_dict:
457 return None
458