A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
make sure mistral_common not imported for non-mistral models (#12669)
When people use deepseek models, they find that they need to solve cv2 version conflict, see https://zhuanlan.zhihu.com/p/21064432691 . I added the check, and make all imports of `cv2` lazy. --------- Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Feb 3, 2025 at 13:40 UTC
20579c0fae1757c9da9fc35a69960563186f3036
4 files changed
+40
-20
.buildkite/test-pipeline.yaml
+2
-2
@@ -50,9 +50,9 @@ steps:
50
- tests/multimodal
51
- tests/test_utils
52
- tests/worker
53
- - tests/standalone_tests/lazy_torch_compile.py
53
+ - tests/standalone_tests/lazy_imports.py
54
commands:
55
- - python3 standalone_tests/lazy_torch_compile.py
55
+ - python3 standalone_tests/lazy_imports.py
56
- pytest -v -s mq_llm_engine # MQLLMEngine
57
- pytest -v -s async_engine # AsyncLLMEngine
58
- NUM_SCHEDULER_STEPS=4 pytest -v -s async_engine/test_async_llm_engine.py
tests/standalone_tests/lazy_imports.py
renamed
+14
-5
@@ -8,7 +8,17 @@ from contextlib import nullcontext
8
9
from vllm_test_utils import BlameResult, blame
10
11
-module_name = "torch._inductor.async_compile"
11
+# List of modules that should not be imported too early.
12
+# Lazy import `torch._inductor.async_compile` to avoid creating
13
+# too many processes before we set the number of compiler threads.
14
+# Lazy import `cv2` to avoid bothering users who only use text models.
15
+# `cv2` can easily mess up the environment.
16
+module_names = ["torch._inductor.async_compile", "cv2"]
17
+
18
+
19
+def any_module_imported():
20
+ return any(module_name in sys.modules for module_name in module_names)
21
+
22
23
# In CI, we only check finally if the module is imported.
24
# If it is indeed imported, we can rerun the test with `use_blame=True`,
@@ -16,8 +26,7 @@ module_name = "torch._inductor.async_compile"
26
# and help find the root cause.
27
# We don't run it in CI by default because it is slow.
28
use_blame = False
19
-context = blame(
20
- lambda: module_name in sys.modules) if use_blame else nullcontext()
29
+context = blame(any_module_imported) if use_blame else nullcontext()
30
with context as result:
31
import vllm # noqa
32
@@ -25,6 +34,6 @@ if use_blame:
34
assert isinstance(result, BlameResult)
35
print(f"the first import location is:\n{result.trace_stack}")
36
28
-assert module_name not in sys.modules, (
29
- f"Module {module_name} is imported. To see the first"
37
+assert not any_module_imported(), (
38
+ f"Some the modules in {module_names} are imported. To see the first"
39
f" import location, run the test with `use_blame=True`.")
vllm/multimodal/video.py
+2
-1
@@ -6,7 +6,6 @@ from io import BytesIO
6
from pathlib import Path
7
from typing import TYPE_CHECKING, Any, Dict, Optional
8
9
-import cv2
9
import numpy as np
10
import numpy.typing as npt
11
from PIL import Image
@@ -95,6 +94,8 @@ def resize_video(frames: npt.NDArray, size: tuple[int, int]) -> npt.NDArray:
94
new_height, new_width = size
95
resized_frames = np.empty((num_frames, new_height, new_width, channels),
96
dtype=frames.dtype)
97
+ # lazy import cv2 to avoid bothering users who only use text models
98
+ import cv2
99
for i, frame in enumerate(frames):
100
resized_frame = cv2.resize(frame, (new_width, new_height))
101
resized_frames[i] = resized_frame
vllm/transformers_utils/tokenizers/mistral.py
+22
-12
@@ -8,21 +8,18 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast
8
9
import huggingface_hub
10
from huggingface_hub import HfApi, hf_hub_download
11
-from mistral_common.protocol.instruct.request import ChatCompletionRequest
12
-from mistral_common.tokens.tokenizers.base import SpecialTokens
13
-# yapf: disable
14
-from mistral_common.tokens.tokenizers.mistral import (
15
- MistralTokenizer as PublicMistralTokenizer)
16
-# yapf: enable
17
-from mistral_common.tokens.tokenizers.sentencepiece import (
18
- SentencePieceTokenizer)
19
-from mistral_common.tokens.tokenizers.tekken import (SpecialTokenPolicy,
20
- Tekkenizer)
11
12
from vllm.logger import init_logger
13
from vllm.utils import is_list_of
14
15
if TYPE_CHECKING:
16
+ # make sure `mistral_common` is lazy imported,
17
+ # so that users who only use non-mistral models
18
+ # will not be bothered by the dependency.
19
+ from mistral_common.protocol.instruct.request import ChatCompletionRequest
20
+ from mistral_common.tokens.tokenizers.mistral import (
21
+ MistralTokenizer as PublicMistralTokenizer)
22
+
23
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
24
25
logger = init_logger(__name__)
@@ -33,7 +30,7 @@ class Encoding:
30
input_ids: Union[List[int], List[List[int]]]
31
32
36
-def maybe_serialize_tool_calls(request: ChatCompletionRequest):
33
+def maybe_serialize_tool_calls(request: "ChatCompletionRequest"):
34
# SEE: https://github.com/vllm-project/vllm/pull/9951
35
# Credits go to: @gcalmettes
36
# NOTE: There is currently a bug in pydantic where attributes
@@ -108,12 +105,16 @@ def find_tokenizer_file(files: List[str]):
105
106
class MistralTokenizer:
107
111
- def __init__(self, tokenizer: PublicMistralTokenizer) -> None:
108
+ def __init__(self, tokenizer: "PublicMistralTokenizer") -> None:
109
self.mistral = tokenizer
110
self.instruct = tokenizer.instruct_tokenizer
111
112
tokenizer_ = tokenizer.instruct_tokenizer.tokenizer
113
+ from mistral_common.tokens.tokenizers.tekken import (
114
+ SpecialTokenPolicy, Tekkenizer)
115
self.is_tekken = isinstance(tokenizer_, Tekkenizer)
116
+ from mistral_common.tokens.tokenizers.sentencepiece import (
117
+ SentencePieceTokenizer)
118
self.is_spm = isinstance(tokenizer_, SentencePieceTokenizer)
119
if self.is_tekken:
120
# Make sure special tokens will not raise
@@ -153,6 +154,8 @@ class MistralTokenizer:
154
assert Path(
155
path_or_repo_id).is_file(), f"Invalid path: {path_or_repo_id}"
156
157
+ from mistral_common.tokens.tokenizers.mistral import (
158
+ MistralTokenizer as PublicMistralTokenizer)
159
mistral_tokenizer = PublicMistralTokenizer.from_file(tokenizer_file)
160
return cls(mistral_tokenizer)
161
@@ -181,6 +184,8 @@ class MistralTokenizer:
184
# by the guided structured output backends.
185
@property
186
def all_special_tokens_extended(self) -> List[str]:
187
+ from mistral_common.tokens.tokenizers.base import SpecialTokens
188
+
189
# tekken defines its own extended special tokens list
190
if hasattr(self.tokenizer, "SPECIAL_TOKENS"):
191
special_tokens = self.tokenizer.SPECIAL_TOKENS
@@ -284,6 +289,8 @@ class MistralTokenizer:
289
if last_message["role"] == "assistant":
290
last_message["prefix"] = True
291
292
+ from mistral_common.protocol.instruct.request import (
293
+ ChatCompletionRequest)
294
request = ChatCompletionRequest(messages=messages,
295
tools=tools) # type: ignore[type-var]
296
encoded = self.mistral.encode_chat_completion(request)
@@ -292,6 +299,7 @@ class MistralTokenizer:
299
return encoded.tokens
300
301
def convert_tokens_to_string(self, tokens: List[str]) -> str:
302
+ from mistral_common.tokens.tokenizers.base import SpecialTokens
303
if self.is_tekken:
304
tokens = [
305
t for t in tokens
@@ -363,6 +371,8 @@ class MistralTokenizer:
371
ids: List[int],
372
skip_special_tokens: bool = True,
373
) -> List[str]:
374
+ from mistral_common.tokens.tokenizers.base import SpecialTokens
375
+
376
# TODO(Patrick) - potentially allow special tokens to not be skipped
377
assert (
378
skip_special_tokens