@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 20579c0fae
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

make sure mistral_common not imported for non-mistral models (#12669)

When people use deepseek models, they find that they need to solve cv2 version conflict, see https://zhuanlan.zhihu.com/p/21064432691 . I added the check, and make all imports of `cv2` lazy. --------- Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Feb 3, 2025 at 13:40 UTC 20579c0fae1757c9da9fc35a69960563186f3036
4 files changed +40 -20
.buildkite/test-pipeline.yaml
+2 -2
@@ -50,9 +50,9 @@ steps:
50 - tests/multimodal
51 - tests/test_utils
52 - tests/worker
53 - - tests/standalone_tests/lazy_torch_compile.py
53 + - tests/standalone_tests/lazy_imports.py
54 commands:
55 - - python3 standalone_tests/lazy_torch_compile.py
55 + - python3 standalone_tests/lazy_imports.py
56 - pytest -v -s mq_llm_engine # MQLLMEngine
57 - pytest -v -s async_engine # AsyncLLMEngine
58 - NUM_SCHEDULER_STEPS=4 pytest -v -s async_engine/test_async_llm_engine.py
tests/standalone_tests/lazy_imports.py renamed
+14 -5
@@ -8,7 +8,17 @@ from contextlib import nullcontext
8
9 from vllm_test_utils import BlameResult, blame
10
11 -module_name = "torch._inductor.async_compile"
11 +# List of modules that should not be imported too early.
12 +# Lazy import `torch._inductor.async_compile` to avoid creating
13 +# too many processes before we set the number of compiler threads.
14 +# Lazy import `cv2` to avoid bothering users who only use text models.
15 +# `cv2` can easily mess up the environment.
16 +module_names = ["torch._inductor.async_compile", "cv2"]
17 +
18 +
19 +def any_module_imported():
20 + return any(module_name in sys.modules for module_name in module_names)
21 +
22
23 # In CI, we only check finally if the module is imported.
24 # If it is indeed imported, we can rerun the test with `use_blame=True`,
@@ -16,8 +26,7 @@ module_name = "torch._inductor.async_compile"
26 # and help find the root cause.
27 # We don't run it in CI by default because it is slow.
28 use_blame = False
19 -context = blame(
20 - lambda: module_name in sys.modules) if use_blame else nullcontext()
29 +context = blame(any_module_imported) if use_blame else nullcontext()
30 with context as result:
31 import vllm # noqa
32
@@ -25,6 +34,6 @@ if use_blame:
34 assert isinstance(result, BlameResult)
35 print(f"the first import location is:\n{result.trace_stack}")
36
28 -assert module_name not in sys.modules, (
29 - f"Module {module_name} is imported. To see the first"
37 +assert not any_module_imported(), (
38 + f"Some the modules in {module_names} are imported. To see the first"
39 f" import location, run the test with `use_blame=True`.")
vllm/multimodal/video.py
+2 -1
@@ -6,7 +6,6 @@ from io import BytesIO
6 from pathlib import Path
7 from typing import TYPE_CHECKING, Any, Dict, Optional
8
9 -import cv2
9 import numpy as np
10 import numpy.typing as npt
11 from PIL import Image
@@ -95,6 +94,8 @@ def resize_video(frames: npt.NDArray, size: tuple[int, int]) -> npt.NDArray:
94 new_height, new_width = size
95 resized_frames = np.empty((num_frames, new_height, new_width, channels),
96 dtype=frames.dtype)
97 + # lazy import cv2 to avoid bothering users who only use text models
98 + import cv2
99 for i, frame in enumerate(frames):
100 resized_frame = cv2.resize(frame, (new_width, new_height))
101 resized_frames[i] = resized_frame
vllm/transformers_utils/tokenizers/mistral.py
+22 -12
@@ -8,21 +8,18 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast
8
9 import huggingface_hub
10 from huggingface_hub import HfApi, hf_hub_download
11 -from mistral_common.protocol.instruct.request import ChatCompletionRequest
12 -from mistral_common.tokens.tokenizers.base import SpecialTokens
13 -# yapf: disable
14 -from mistral_common.tokens.tokenizers.mistral import (
15 - MistralTokenizer as PublicMistralTokenizer)
16 -# yapf: enable
17 -from mistral_common.tokens.tokenizers.sentencepiece import (
18 - SentencePieceTokenizer)
19 -from mistral_common.tokens.tokenizers.tekken import (SpecialTokenPolicy,
20 - Tekkenizer)
11
12 from vllm.logger import init_logger
13 from vllm.utils import is_list_of
14
15 if TYPE_CHECKING:
16 + # make sure `mistral_common` is lazy imported,
17 + # so that users who only use non-mistral models
18 + # will not be bothered by the dependency.
19 + from mistral_common.protocol.instruct.request import ChatCompletionRequest
20 + from mistral_common.tokens.tokenizers.mistral import (
21 + MistralTokenizer as PublicMistralTokenizer)
22 +
23 from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
24
25 logger = init_logger(__name__)
@@ -33,7 +30,7 @@ class Encoding:
30 input_ids: Union[List[int], List[List[int]]]
31
32
36 -def maybe_serialize_tool_calls(request: ChatCompletionRequest):
33 +def maybe_serialize_tool_calls(request: "ChatCompletionRequest"):
34 # SEE: https://github.com/vllm-project/vllm/pull/9951
35 # Credits go to: @gcalmettes
36 # NOTE: There is currently a bug in pydantic where attributes
@@ -108,12 +105,16 @@ def find_tokenizer_file(files: List[str]):
105
106 class MistralTokenizer:
107
111 - def __init__(self, tokenizer: PublicMistralTokenizer) -> None:
108 + def __init__(self, tokenizer: "PublicMistralTokenizer") -> None:
109 self.mistral = tokenizer
110 self.instruct = tokenizer.instruct_tokenizer
111
112 tokenizer_ = tokenizer.instruct_tokenizer.tokenizer
113 + from mistral_common.tokens.tokenizers.tekken import (
114 + SpecialTokenPolicy, Tekkenizer)
115 self.is_tekken = isinstance(tokenizer_, Tekkenizer)
116 + from mistral_common.tokens.tokenizers.sentencepiece import (
117 + SentencePieceTokenizer)
118 self.is_spm = isinstance(tokenizer_, SentencePieceTokenizer)
119 if self.is_tekken:
120 # Make sure special tokens will not raise
@@ -153,6 +154,8 @@ class MistralTokenizer:
154 assert Path(
155 path_or_repo_id).is_file(), f"Invalid path: {path_or_repo_id}"
156
157 + from mistral_common.tokens.tokenizers.mistral import (
158 + MistralTokenizer as PublicMistralTokenizer)
159 mistral_tokenizer = PublicMistralTokenizer.from_file(tokenizer_file)
160 return cls(mistral_tokenizer)
161
@@ -181,6 +184,8 @@ class MistralTokenizer:
184 # by the guided structured output backends.
185 @property
186 def all_special_tokens_extended(self) -> List[str]:
187 + from mistral_common.tokens.tokenizers.base import SpecialTokens
188 +
189 # tekken defines its own extended special tokens list
190 if hasattr(self.tokenizer, "SPECIAL_TOKENS"):
191 special_tokens = self.tokenizer.SPECIAL_TOKENS
@@ -284,6 +289,8 @@ class MistralTokenizer:
289 if last_message["role"] == "assistant":
290 last_message["prefix"] = True
291
292 + from mistral_common.protocol.instruct.request import (
293 + ChatCompletionRequest)
294 request = ChatCompletionRequest(messages=messages,
295 tools=tools) # type: ignore[type-var]
296 encoded = self.mistral.encode_chat_completion(request)
@@ -292,6 +299,7 @@ class MistralTokenizer:
299 return encoded.tokens
300
301 def convert_tokens_to_string(self, tokens: List[str]) -> str:
302 + from mistral_common.tokens.tokenizers.base import SpecialTokens
303 if self.is_tekken:
304 tokens = [
305 t for t in tokens
@@ -363,6 +371,8 @@ class MistralTokenizer:
371 ids: List[int],
372 skip_special_tokens: bool = True,
373 ) -> List[str]:
374 + from mistral_common.tokens.tokenizers.base import SpecialTokens
375 +
376 # TODO(Patrick) - potentially allow special tokens to not be skipped
377 assert (
378 skip_special_tokens