@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f2e263b801
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Bugfix] Offline mode fix (#8376)

Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>

Joe Runde committed Sep 12, 2024 at 12:11 UTC f2e263b801743596f5dda0680e0bcb0fc3c05e26
4 files changed +106 -2
.buildkite/test-pipeline.yaml
+1
@@ -91,6 +91,7 @@ steps:
91 - pytest -v -s entrypoints/llm/test_lazy_outlines.py # it needs a clean process
92 - pytest -v -s entrypoints/openai
93 - pytest -v -s entrypoints/test_chat_utils.py
94 + - pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests
95
96
97 - label: Distributed Tests (4 GPUs) # 10min
tests/entrypoints/offline_mode/__init__.py
tests/entrypoints/offline_mode/test_offline_mode.py new
+77
@@ -0,0 +1,77 @@
1 +"""Tests for HF_HUB_OFFLINE mode"""
2 +import importlib
3 +import sys
4 +import weakref
5 +
6 +import pytest
7 +
8 +from vllm import LLM
9 +
10 +from ...conftest import cleanup
11 +
12 +MODEL_NAME = "facebook/opt-125m"
13 +
14 +
15 +@pytest.fixture(scope="module")
16 +def llm():
17 + # pytest caches the fixture so we use weakref.proxy to
18 + # enable garbage collection
19 + llm = LLM(model=MODEL_NAME,
20 + max_num_batched_tokens=4096,
21 + tensor_parallel_size=1,
22 + gpu_memory_utilization=0.10,
23 + enforce_eager=True)
24 +
25 + with llm.deprecate_legacy_api():
26 + yield weakref.proxy(llm)
27 +
28 + del llm
29 +
30 + cleanup()
31 +
32 +
33 +@pytest.mark.skip_global_cleanup
34 +def test_offline_mode(llm: LLM, monkeypatch):
35 + # we use the llm fixture to ensure the model files are in-cache
36 + del llm
37 +
38 + # Set HF to offline mode and ensure we can still construct an LLM
39 + try:
40 + monkeypatch.setenv("HF_HUB_OFFLINE", "1")
41 + # Need to re-import huggingface_hub and friends to setup offline mode
42 + _re_import_modules()
43 + # Cached model files should be used in offline mode
44 + LLM(model=MODEL_NAME,
45 + max_num_batched_tokens=4096,
46 + tensor_parallel_size=1,
47 + gpu_memory_utilization=0.10,
48 + enforce_eager=True)
49 + finally:
50 + # Reset the environment after the test
51 + # NB: Assuming tests are run in online mode
52 + monkeypatch.delenv("HF_HUB_OFFLINE")
53 + _re_import_modules()
54 + pass
55 +
56 +
57 +def _re_import_modules():
58 + hf_hub_module_names = [
59 + k for k in sys.modules if k.startswith("huggingface_hub")
60 + ]
61 + transformers_module_names = [
62 + k for k in sys.modules if k.startswith("transformers")
63 + and not k.startswith("transformers_modules")
64 + ]
65 +
66 + reload_exception = None
67 + for module_name in hf_hub_module_names + transformers_module_names:
68 + try:
69 + importlib.reload(sys.modules[module_name])
70 + except Exception as e:
71 + reload_exception = e
72 + # Try to continue clean up so that other tests are less likely to
73 + # be affected
74 +
75 + # Error this test if reloading a module failed
76 + if reload_exception is not None:
77 + raise reload_exception
vllm/transformers_utils/config.py
+28 -2
@@ -4,7 +4,9 @@ import json
4 from pathlib import Path
5 from typing import Any, Dict, Optional, Type, Union
6
7 -from huggingface_hub import file_exists, hf_hub_download
7 +import huggingface_hub
8 +from huggingface_hub import (file_exists, hf_hub_download,
9 + try_to_load_from_cache)
10 from transformers import GenerationConfig, PretrainedConfig
11 from transformers.models.auto.image_processing_auto import (
12 get_image_processor_config)
@@ -70,7 +72,22 @@ def file_or_path_exists(model: Union[str, Path], config_name, revision,
72 if Path(model).exists():
73 return (Path(model) / config_name).is_file()
74
73 - return file_exists(model, config_name, revision=revision, token=token)
75 + # Offline mode support: Check if config file is cached already
76 + cached_filepath = try_to_load_from_cache(repo_id=model,
77 + filename=config_name,
78 + revision=revision)
79 + if isinstance(cached_filepath, str):
80 + # The config file exists in cache- we can continue trying to load
81 + return True
82 +
83 + # NB: file_exists will only check for the existence of the config file on
84 + # hf_hub. This will fail in offline mode.
85 + try:
86 + return file_exists(model, config_name, revision=revision, token=token)
87 + except huggingface_hub.errors.OfflineModeIsEnabled:
88 + # Don't raise in offline mode, all we know is that we don't have this
89 + # file cached.
90 + return False
91
92
93 def get_config(
@@ -102,6 +119,15 @@ def get_config(
119 token=kwargs.get("token")):
120 config_format = ConfigFormat.MISTRAL
121 else:
122 + # If we're in offline mode and found no valid config format, then
123 + # raise an offline mode error to indicate to the user that they
124 + # don't have files cached and may need to go online.
125 + # This is conveniently triggered by calling file_exists().
126 + file_exists(model,
127 + HF_CONFIG_NAME,
128 + revision=revision,
129 + token=kwargs.get("token"))
130 +
131 raise ValueError(f"No supported config format found in {model}")
132
133 if config_format == ConfigFormat.HF: