A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix] Fix offline mode when using `mistral_common` (#9457)
sasha0552 committed
Oct 19, 2024 at 01:12 UTC
337ed76671812c4599560f73b8fa511927814e37
2 files changed
+62
-28
tests/entrypoints/offline_mode/test_offline_mode.py
+31
-25
@@ -1,50 +1,56 @@
1
"""Tests for HF_HUB_OFFLINE mode"""
2
import importlib
3
import sys
4
-import weakref
4
5
import pytest
6
7
from vllm import LLM
8
from vllm.distributed import cleanup_dist_env_and_memory
9
11
-MODEL_NAME = "facebook/opt-125m"
10
+MODEL_CONFIGS = [
11
+ {
12
+ "model": "facebook/opt-125m",
13
+ "enforce_eager": True,
14
+ "gpu_memory_utilization": 0.20,
15
+ "max_model_len": 64,
16
+ "max_num_batched_tokens": 64,
17
+ "max_num_seqs": 64,
18
+ "tensor_parallel_size": 1,
19
+ },
20
+ {
21
+ "model": "mistralai/Mistral-7B-Instruct-v0.1",
22
+ "enforce_eager": True,
23
+ "gpu_memory_utilization": 0.95,
24
+ "max_model_len": 64,
25
+ "max_num_batched_tokens": 64,
26
+ "max_num_seqs": 64,
27
+ "tensor_parallel_size": 1,
28
+ "tokenizer_mode": "mistral",
29
+ },
30
+]
31
32
33
@pytest.fixture(scope="module")
15
-def llm():
16
- # pytest caches the fixture so we use weakref.proxy to
17
- # enable garbage collection
18
- llm = LLM(model=MODEL_NAME,
19
- max_num_batched_tokens=4096,
20
- tensor_parallel_size=1,
21
- gpu_memory_utilization=0.10,
22
- enforce_eager=True)
34
+def cache_models():
35
+ # Cache model files first
36
+ for model_config in MODEL_CONFIGS:
37
+ LLM(**model_config)
38
+ cleanup_dist_env_and_memory()
39
24
- with llm.deprecate_legacy_api():
25
- yield weakref.proxy(llm)
26
-
27
- del llm
28
-
29
- cleanup_dist_env_and_memory()
40
+ yield
41
42
43
@pytest.mark.skip_global_cleanup
33
-def test_offline_mode(llm: LLM, monkeypatch):
34
- # we use the llm fixture to ensure the model files are in-cache
35
- del llm
36
-
44
+@pytest.mark.usefixtures("cache_models")
45
+def test_offline_mode(monkeypatch):
46
# Set HF to offline mode and ensure we can still construct an LLM
47
try:
48
monkeypatch.setenv("HF_HUB_OFFLINE", "1")
49
# Need to re-import huggingface_hub and friends to setup offline mode
50
_re_import_modules()
51
# Cached model files should be used in offline mode
43
- LLM(model=MODEL_NAME,
44
- max_num_batched_tokens=4096,
45
- tensor_parallel_size=1,
46
- gpu_memory_utilization=0.20,
47
- enforce_eager=True)
52
+ for model_config in MODEL_CONFIGS:
53
+ LLM(**model_config)
54
finally:
55
# Reset the environment after the test
56
# NB: Assuming tests are run in online mode
vllm/transformers_utils/tokenizers/mistral.py
+31
-3
@@ -4,6 +4,7 @@ from dataclasses import dataclass
4
from pathlib import Path
5
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast
6
7
+import huggingface_hub
8
from huggingface_hub import HfApi, hf_hub_download
9
from mistral_common.protocol.instruct.request import ChatCompletionRequest
10
# yapf: disable
@@ -24,6 +25,26 @@ class Encoding:
25
input_ids: List[int]
26
27
28
+def list_local_repo_files(repo_id: str, revision: Optional[str]) -> List[str]:
29
+ repo_cache = os.path.join(
30
+ huggingface_hub.constants.HF_HUB_CACHE,
31
+ huggingface_hub.constants.REPO_ID_SEPARATOR.join(
32
+ ["models", *repo_id.split("/")]))
33
+
34
+ if revision is None:
35
+ revision_file = os.path.join(repo_cache, "refs", "main")
36
+ if os.path.isfile(revision_file):
37
+ with open(revision_file) as file:
38
+ revision = file.read()
39
+
40
+ if revision:
41
+ revision_dir = os.path.join(repo_cache, "snapshots", revision)
42
+ if os.path.isdir(revision_dir):
43
+ return os.listdir(revision_dir)
44
+
45
+ return []
46
+
47
+
48
def find_tokenizer_file(files: List[str]):
49
file_pattern = re.compile(r"^tokenizer\.model\.v.*$|^tekken\.json$")
50
@@ -90,9 +111,16 @@ class MistralTokenizer:
111
@staticmethod
112
def _download_mistral_tokenizer_from_hf(tokenizer_name: str,
113
revision: Optional[str]) -> str:
93
- api = HfApi()
94
- repo_info = api.model_info(tokenizer_name)
95
- files = [s.rfilename for s in repo_info.siblings]
114
+ try:
115
+ hf_api = HfApi()
116
+ files = hf_api.list_repo_files(repo_id=tokenizer_name,
117
+ revision=revision)
118
+ except ConnectionError as exc:
119
+ files = list_local_repo_files(repo_id=tokenizer_name,
120
+ revision=revision)
121
+
122
+ if len(files) == 0:
123
+ raise exc
124
125
filename = find_tokenizer_file(files)
126