@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 337ed76671
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix] Fix offline mode when using `mistral_common` (#9457)

sasha0552 committed Oct 19, 2024 at 01:12 UTC 337ed76671812c4599560f73b8fa511927814e37
2 files changed +62 -28
tests/entrypoints/offline_mode/test_offline_mode.py
+31 -25
@@ -1,50 +1,56 @@
1 """Tests for HF_HUB_OFFLINE mode"""
2 import importlib
3 import sys
4 -import weakref
4
5 import pytest
6
7 from vllm import LLM
8 from vllm.distributed import cleanup_dist_env_and_memory
9
11 -MODEL_NAME = "facebook/opt-125m"
10 +MODEL_CONFIGS = [
11 + {
12 + "model": "facebook/opt-125m",
13 + "enforce_eager": True,
14 + "gpu_memory_utilization": 0.20,
15 + "max_model_len": 64,
16 + "max_num_batched_tokens": 64,
17 + "max_num_seqs": 64,
18 + "tensor_parallel_size": 1,
19 + },
20 + {
21 + "model": "mistralai/Mistral-7B-Instruct-v0.1",
22 + "enforce_eager": True,
23 + "gpu_memory_utilization": 0.95,
24 + "max_model_len": 64,
25 + "max_num_batched_tokens": 64,
26 + "max_num_seqs": 64,
27 + "tensor_parallel_size": 1,
28 + "tokenizer_mode": "mistral",
29 + },
30 +]
31
32
33 @pytest.fixture(scope="module")
15 -def llm():
16 - # pytest caches the fixture so we use weakref.proxy to
17 - # enable garbage collection
18 - llm = LLM(model=MODEL_NAME,
19 - max_num_batched_tokens=4096,
20 - tensor_parallel_size=1,
21 - gpu_memory_utilization=0.10,
22 - enforce_eager=True)
34 +def cache_models():
35 + # Cache model files first
36 + for model_config in MODEL_CONFIGS:
37 + LLM(**model_config)
38 + cleanup_dist_env_and_memory()
39
24 - with llm.deprecate_legacy_api():
25 - yield weakref.proxy(llm)
26 -
27 - del llm
28 -
29 - cleanup_dist_env_and_memory()
40 + yield
41
42
43 @pytest.mark.skip_global_cleanup
33 -def test_offline_mode(llm: LLM, monkeypatch):
34 - # we use the llm fixture to ensure the model files are in-cache
35 - del llm
36 -
44 +@pytest.mark.usefixtures("cache_models")
45 +def test_offline_mode(monkeypatch):
46 # Set HF to offline mode and ensure we can still construct an LLM
47 try:
48 monkeypatch.setenv("HF_HUB_OFFLINE", "1")
49 # Need to re-import huggingface_hub and friends to setup offline mode
50 _re_import_modules()
51 # Cached model files should be used in offline mode
43 - LLM(model=MODEL_NAME,
44 - max_num_batched_tokens=4096,
45 - tensor_parallel_size=1,
46 - gpu_memory_utilization=0.20,
47 - enforce_eager=True)
52 + for model_config in MODEL_CONFIGS:
53 + LLM(**model_config)
54 finally:
55 # Reset the environment after the test
56 # NB: Assuming tests are run in online mode
vllm/transformers_utils/tokenizers/mistral.py
+31 -3
@@ -4,6 +4,7 @@ from dataclasses import dataclass
4 from pathlib import Path
5 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast
6
7 +import huggingface_hub
8 from huggingface_hub import HfApi, hf_hub_download
9 from mistral_common.protocol.instruct.request import ChatCompletionRequest
10 # yapf: disable
@@ -24,6 +25,26 @@ class Encoding:
25 input_ids: List[int]
26
27
28 +def list_local_repo_files(repo_id: str, revision: Optional[str]) -> List[str]:
29 + repo_cache = os.path.join(
30 + huggingface_hub.constants.HF_HUB_CACHE,
31 + huggingface_hub.constants.REPO_ID_SEPARATOR.join(
32 + ["models", *repo_id.split("/")]))
33 +
34 + if revision is None:
35 + revision_file = os.path.join(repo_cache, "refs", "main")
36 + if os.path.isfile(revision_file):
37 + with open(revision_file) as file:
38 + revision = file.read()
39 +
40 + if revision:
41 + revision_dir = os.path.join(repo_cache, "snapshots", revision)
42 + if os.path.isdir(revision_dir):
43 + return os.listdir(revision_dir)
44 +
45 + return []
46 +
47 +
48 def find_tokenizer_file(files: List[str]):
49 file_pattern = re.compile(r"^tokenizer\.model\.v.*$|^tekken\.json$")
50
@@ -90,9 +111,16 @@ class MistralTokenizer:
111 @staticmethod
112 def _download_mistral_tokenizer_from_hf(tokenizer_name: str,
113 revision: Optional[str]) -> str:
93 - api = HfApi()
94 - repo_info = api.model_info(tokenizer_name)
95 - files = [s.rfilename for s in repo_info.siblings]
114 + try:
115 + hf_api = HfApi()
116 + files = hf_api.list_repo_files(repo_id=tokenizer_name,
117 + revision=revision)
118 + except ConnectionError as exc:
119 + files = list_local_repo_files(repo_id=tokenizer_name,
120 + revision=revision)
121 +
122 + if len(files) == 0:
123 + raise exc
124
125 filename = find_tokenizer_file(files)
126