@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 54e2f83d0a

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 7, 22:45 UTC

[Feature] Lazy import for the "mistral" tokenizer module. (#34651)

Signed-off-by: Neil Schemenauer <nas@arctrix.com>

Neil Schemenauer committed Feb 23, 2026 at 00:43 UTC 54e2f83d0a82462e0128e5d852e3d46fbb566a7f

14 files changed +68 -48

tests/models/multimodal/processing/test_common.py

+2 -2

@@ -23,7 +23,7 @@ from vllm.multimodal.cache import MultiModalProcessorOnlyCache

23 from vllm.multimodal.inputs import MultiModalInputs, batched_tensors_equal

24 from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext

25 from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config

26 -from vllm.tokenizers.mistral import MistralTokenizer

26 +from vllm.utils.mistral import is_mistral_tokenizer

28 from ....multimodal.utils import random_audio, random_image, random_video

29 from ...registry import (

@@ -183,7 +183,7 @@ def get_text_token_prompts(

183

184 text_prompt: str | None

185 token_prompt: list[int]

186 - if isinstance(tokenizer, MistralTokenizer):

186 + if is_mistral_tokenizer(tokenizer):

187 # ChatCompletionRequest only supports ImageChunk natively;

188 # for other modalities (e.g. audio), fall back to the model's

189 # own dummy inputs builder which knows the right placeholders.

tests/reasoning/utils.py

+3 -3

@@ -4,7 +4,7 @@

4 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest

5 from vllm.entrypoints.openai.engine.protocol import DeltaMessage

6 from vllm.reasoning import ReasoningParser

7 -from vllm.tokenizers.mistral import MistralTokenizer

7 +from vllm.utils.mistral import is_mistral_tokenizer

10 class StreamingReasoningReconstructor:

@@ -59,7 +59,7 @@ def run_reasoning_extraction_mistral(

59 request: ChatCompletionRequest | None = None,

60 streaming: bool = False,

61 ) -> tuple[str | None, str | None]:

62 - assert isinstance(reasoning_parser.model_tokenizer, MistralTokenizer), type(

62 + assert is_mistral_tokenizer(reasoning_parser.model_tokenizer), type(

63 reasoning_parser.model_tokenizer

64 )

65 if streaming:

@@ -130,7 +130,7 @@ def run_reasoning_extraction_streaming_mistral(

130 model_deltas: list[int],

131 request: ChatCompletionRequest | None = None,

132 ) -> StreamingReasoningReconstructor:

133 - assert isinstance(reasoning_parser.model_tokenizer, MistralTokenizer), type(

133 + assert is_mistral_tokenizer(reasoning_parser.model_tokenizer), type(

134 reasoning_parser.model_tokenizer

135 )

136 request = request or ChatCompletionRequest(messages=[], model="test-model")

vllm/entrypoints/llm.py

+3 -3

@@ -83,9 +83,9 @@ from vllm.renderers.inputs.preprocess import (

83 from vllm.sampling_params import BeamSearchParams, RequestOutputKind, SamplingParams

84 from vllm.tasks import PoolingTask

85 from vllm.tokenizers import TokenizerLike

86 -from vllm.tokenizers.mistral import MistralTokenizer

86 from vllm.usage.usage_lib import UsageContext

87 from vllm.utils.counter import Counter

88 +from vllm.utils.mistral import is_mistral_tokenizer

89 from vllm.utils.tqdm_utils import maybe_tqdm

90 from vllm.v1.engine.llm_engine import LLMEngine

91 from vllm.v1.sample.logits_processor import LogitsProcessor

@@ -891,7 +891,7 @@ class LLM:

891 add_generation_prompt=add_generation_prompt,

892 continue_final_message=continue_final_message,

893 tools=tools,

894 - tokenize=isinstance(renderer.tokenizer, MistralTokenizer),

894 + tokenize=is_mistral_tokenizer(renderer.tokenizer),

895 ),

896 ),

897 )

@@ -1458,7 +1458,7 @@ class LLM:

1458 model_config = self.model_config

1459 tokenizer = self.get_tokenizer()

1460

1461 - if isinstance(tokenizer, MistralTokenizer):

1461 + if is_mistral_tokenizer(tokenizer):

1462 raise ValueError("Score API is not supported for Mistral tokenizer")

1463

1464 if len(data_1) == 1:

vllm/entrypoints/openai/chat_completion/serving.py

+10 -16

@@ -75,16 +75,12 @@ from vllm.parser import ParserManager

75 from vllm.reasoning import ReasoningParser

76 from vllm.sampling_params import BeamSearchParams, SamplingParams

77 from vllm.tokenizers import TokenizerLike

78 -from vllm.tokenizers.mistral import (

79 - MistralTokenizer,

80 - maybe_serialize_tool_calls,

81 - truncate_tool_call_ids,

82 - validate_request_params,

83 -)

78 from vllm.tool_parsers import ToolParser

79 from vllm.tool_parsers.mistral_tool_parser import MistralToolCall

80 from vllm.tool_parsers.utils import partial_json_loads

81 from vllm.utils.collection_utils import as_list

82 +from vllm.utils.mistral import is_mistral_tokenizer

83 +from vllm.utils.mistral import mt as _mt

85 logger = init_logger(__name__)

@@ -244,18 +240,18 @@ class OpenAIServingChat(OpenAIServing):

240

241 tool_parser = self.tool_parser

242

247 - if isinstance(tokenizer, MistralTokenizer):

243 + if is_mistral_tokenizer(tokenizer):

244 # because of issues with pydantic we need to potentially

245 # re-serialize the tool_calls field of the request

246 # for more info: see comment in `maybe_serialize_tool_calls`

251 - maybe_serialize_tool_calls(request) # type: ignore[arg-type]

252 - truncate_tool_call_ids(request) # type: ignore[arg-type]

253 - validate_request_params(request)

247 + _mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]

248 + _mt.truncate_tool_call_ids(request) # type: ignore[arg-type]

249 + _mt.validate_request_params(request)

250

251 # Check if tool parsing is unavailable (common condition)

252 tool_parsing_unavailable = (

253 tool_parser is None

258 - and not isinstance(tokenizer, MistralTokenizer)

254 + and not is_mistral_tokenizer(tokenizer)

255 and not self.use_harmony

256 )

257

@@ -639,8 +635,6 @@ class OpenAIServingChat(OpenAIServing):

635 request_metadata: RequestResponseMetadata,

636 reasoning_parser: ReasoningParser | None = None,

637 ) -> AsyncGenerator[str, None]:

642 - from vllm.tokenizers.mistral import MistralTokenizer

643 -

638 created_time = int(time.time())

639 chunk_object_type: Final = "chat.completion.chunk"

640 first_iteration = True

@@ -955,7 +949,7 @@ class OpenAIServingChat(OpenAIServing):

949 )

950 else:

951 # Generate ID based on tokenizer type

958 - if isinstance(tokenizer, MistralTokenizer):

952 + if is_mistral_tokenizer(tokenizer):

953 tool_call_id = MistralToolCall.generate_random_id()

954 else:

955 tool_call_id = make_tool_call_id(

@@ -1516,7 +1510,7 @@ class OpenAIServingChat(OpenAIServing):

1510 tool_parser_cls=self.tool_parser,

1511 )

1512 tool_call_class = (

1519 - MistralToolCall if isinstance(tokenizer, MistralTokenizer) else ToolCall

1513 + MistralToolCall if is_mistral_tokenizer(tokenizer) else ToolCall

1514 )

1515 if self.use_harmony:

1516 # Harmony models already have parsed content and tool_calls

@@ -1951,7 +1945,7 @@ class OpenAIServingChat(OpenAIServing):

1945 # because of issues with pydantic we need to potentially

1946 # re-serialize the tool_calls field of the request

1947 # for more info: see comment in `maybe_serialize_tool_calls`

1954 - maybe_serialize_tool_calls(request) # type: ignore[arg-type]

1948 + _mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]

1949

1950 # Add system message.

1951 # NOTE: In Chat Completion API, browsing is enabled by default

vllm/entrypoints/openai/engine/serving.py

+2 -3

@@ -128,6 +128,7 @@ from vllm.utils.async_utils import (

128 collect_from_async_generator,

129 merge_async_iterators,

130 )

131 +from vllm.utils.mistral import is_mistral_tokenizer

132

133

134 class GenerationError(Exception):

@@ -976,15 +977,13 @@ class OpenAIServing:

977 tool_dicts: list[dict[str, Any]] | None = None,

978 tool_parser: Callable[[TokenizerLike], ToolParser] | None = None,

979 ) -> tuple[list[ConversationMessage], list[ProcessorInputs]]:

979 - from vllm.tokenizers.mistral import MistralTokenizer

980 -

980 renderer = self.renderer

981

982 default_template_kwargs = merge_kwargs(

983 default_template_kwargs,

984 dict(

985 tools=tool_dicts,

987 - tokenize=isinstance(renderer.tokenizer, MistralTokenizer),

986 + tokenize=is_mistral_tokenizer(renderer.tokenizer),

987 ),

988 )

989

vllm/entrypoints/pooling/score/serving.py

+2 -2

@@ -41,8 +41,8 @@ from vllm.logger import init_logger

41 from vllm.lora.request import LoRARequest

42 from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput

43 from vllm.tokenizers import TokenizerLike

44 -from vllm.tokenizers.mistral import MistralTokenizer

44 from vllm.utils.async_utils import make_async, merge_async_iterators

45 +from vllm.utils.mistral import is_mistral_tokenizer

47 logger = init_logger(__name__)

@@ -348,7 +348,7 @@ class ServingScores(OpenAIServing):

348 trace_headers: Mapping[str, str] | None = None,

349 ) -> list[PoolingRequestOutput] | ErrorResponse:

350 tokenizer = self.renderer.get_tokenizer()

351 - if isinstance(tokenizer, MistralTokenizer):

351 + if is_mistral_tokenizer(tokenizer):

352 raise ValueError("MistralTokenizer not supported for cross-encoding")

353

354 model_config = self.model_config

vllm/multimodal/processing/context.py

+2 -3

@@ -26,6 +26,7 @@ from vllm.tokenizers import TokenizerLike

26 from vllm.transformers_utils.processor import cached_processor_from_config

27 from vllm.utils.func_utils import get_allowed_kwarg_only_overrides

28 from vllm.utils.jsontree import JSONTree, json_map_leaves

29 +from vllm.utils.mistral import is_mistral_tokenizer

31 if TYPE_CHECKING:

32 from transformers.configuration_utils import PretrainedConfig

@@ -260,10 +261,8 @@ class InputProcessingContext:

261

262 typ = ProcessorMixin

263

263 - from vllm.tokenizers.mistral import MistralTokenizer

264 -

264 tokenizer = self.tokenizer

266 - if isinstance(tokenizer, MistralTokenizer):

265 + if is_mistral_tokenizer(tokenizer):

266 tokenizer = tokenizer.transformers_tokenizer

267

268 merged_kwargs = self.get_merged_mm_kwargs(kwargs)

vllm/sampling_params.py

+4 -4

@@ -16,6 +16,7 @@ from vllm.config import ModelConfig, SpeculativeConfig, StructuredOutputsConfig

16 from vllm.exceptions import VLLMValidationError

17 from vllm.logger import init_logger

18 from vllm.tokenizers import TokenizerLike

19 +from vllm.utils.mistral import is_mistral_tokenizer

20 from vllm.v1.serial_utils import PydanticMsgspecMixin

22 logger = init_logger(__name__)

@@ -731,7 +732,6 @@ class SamplingParams(

732 ):

733 raise ValueError("structured_outputs.grammar cannot be an empty string")

734

734 - from vllm.tokenizers.mistral import MistralTokenizer

735 from vllm.v1.structured_output.backend_guidance import (

736 has_guidance_unsupported_json_features,

737 validate_guidance_grammar,

@@ -752,7 +752,7 @@ class SamplingParams(

752 # allows <|special_token|> and similar, see

753 # https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens

754 # Without tokenizer these are disallowed in grammars.

755 - if isinstance(tokenizer, MistralTokenizer):

755 + if is_mistral_tokenizer(tokenizer):

756 raise ValueError(

757 "Mistral tokenizer is not supported for the 'guidance' "

758 "structured output backend. Please use ['xgrammar', 'outlines'] "

@@ -764,7 +764,7 @@ class SamplingParams(

764 validate_structured_output_request_outlines(self)

765 elif backend == "lm-format-enforcer":

766 # lm format enforcer backend

767 - if isinstance(tokenizer, MistralTokenizer):

767 + if is_mistral_tokenizer(tokenizer):

768 raise ValueError(

769 "Mistral tokenizer is not supported for the 'lm-format-enforcer' "

770 "structured output backend. Please use ['xgrammar', 'outlines'] "

@@ -796,7 +796,7 @@ class SamplingParams(

796 schema = so_params.json

797 skip_guidance = has_guidance_unsupported_json_features(schema)

798

799 - if isinstance(tokenizer, MistralTokenizer) or skip_guidance:

799 + if is_mistral_tokenizer(tokenizer) or skip_guidance:

800 # Fall back to outlines if the tokenizer is Mistral

801 # or if schema contains features unsupported by guidance

802 validate_structured_output_request_outlines(self)

vllm/tokenizers/mistral.py

@@ -210,6 +210,8 @@ def _tekken_token_to_id(tokenizer: "Tekkenizer", t: str | bytes) -> int:

210

211

212 class MistralTokenizer(TokenizerLike):

213 + IS_MISTRAL_TOKENIZER = True # used by vllm.utils.mistral

214 +

215 @classmethod

216 def from_pretrained(

217 cls,

vllm/tool_parsers/hermes_tool_parser.py

+2 -2

@@ -22,10 +22,10 @@ from vllm.entrypoints.openai.engine.protocol import (

22 )

23 from vllm.logger import init_logger

24 from vllm.tokenizers import TokenizerLike

25 -from vllm.tokenizers.mistral import MistralTokenizer

25 from vllm.tool_parsers.abstract_tool_parser import (

26 ToolParser,

27 )

28 +from vllm.utils.mistral import is_mistral_tokenizer

30 logger = init_logger(__name__)

@@ -34,7 +34,7 @@ class Hermes2ProToolParser(ToolParser):

34 def __init__(self, tokenizer: TokenizerLike):

35 super().__init__(tokenizer)

37 - if isinstance(tokenizer, MistralTokenizer):

37 + if is_mistral_tokenizer(tokenizer):

38 logger.error("Detected Mistral tokenizer when using a Hermes model")

39 self.model_tokenizer = tokenizer.tokenizer

vllm/tool_parsers/jamba_tool_parser.py

+2 -2

@@ -22,9 +22,9 @@ from vllm.entrypoints.openai.engine.protocol import (

22 )

23 from vllm.logger import init_logger

24 from vllm.tokenizers import TokenizerLike

25 -from vllm.tokenizers.mistral import MistralTokenizer

25 from vllm.tool_parsers import ToolParser

26 from vllm.tool_parsers.utils import extract_intermediate_diff

27 +from vllm.utils.mistral import is_mistral_tokenizer

29 logger = init_logger(__name__)

@@ -33,7 +33,7 @@ class JambaToolParser(ToolParser):

33 def __init__(self, tokenizer: TokenizerLike):

34 super().__init__(tokenizer)

36 - if isinstance(self.model_tokenizer, MistralTokenizer):

36 + if is_mistral_tokenizer(self.model_tokenizer):

37 raise ValueError(

38 "Detected a MistralTokenizer tokenizer when using a Jamba model"

39 )

vllm/tool_parsers/mistral_tool_parser.py

+4 -6

@@ -25,10 +25,10 @@ from vllm.entrypoints.openai.engine.protocol import (

25 )

26 from vllm.logger import init_logger

27 from vllm.tokenizers import TokenizerLike

28 -from vllm.tokenizers.mistral import MistralTokenizer

28 from vllm.tool_parsers.abstract_tool_parser import (

29 ToolParser,

30 )

31 +from vllm.utils.mistral import is_mistral_tokenizer

33 logger = init_logger(__name__)

@@ -66,9 +66,7 @@ class MistralToolCall(ToolCall):

68 def _is_pre_v11_tokeniser(model_tokenizer: TokenizerLike) -> bool:

69 - return not (

70 - isinstance(model_tokenizer, MistralTokenizer) and model_tokenizer.version >= 11

71 - )

69 + return not (is_mistral_tokenizer(model_tokenizer) and model_tokenizer.version >= 11)

72 class MistralToolParser(ToolParser):

@@ -83,7 +81,7 @@ class MistralToolParser(ToolParser):

81 def __init__(self, tokenizer: TokenizerLike):

82 super().__init__(tokenizer)

86 - if not isinstance(self.model_tokenizer, MistralTokenizer):

84 + if not is_mistral_tokenizer(self.model_tokenizer):

85 logger.info("Non-Mistral tokenizer detected when using a Mistral model...")

87 # initialize properties used for state when parsing tool calls in

@@ -115,7 +113,7 @@ class MistralToolParser(ToolParser):

113 def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:

114 request = super().adjust_request(request)

115 if (

118 - not isinstance(self.model_tokenizer, MistralTokenizer)

116 + not is_mistral_tokenizer(self.model_tokenizer)

117 and request.tools

118 and request.tool_choice != "none"

119 ):

vllm/utils/mistral.py new

+28

@@ -0,0 +1,28 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""Provides lazy import of the vllm.tokenizers.mistral module."""

4 +

5 +from __future__ import annotations

6 +

7 +from typing import TYPE_CHECKING, TypeGuard

8 +

9 +from vllm.tokenizers import TokenizerLike

10 +from vllm.utils.import_utils import LazyLoader

11 +

12 +if TYPE_CHECKING:

13 + # if type checking, eagerly import the module

14 + import vllm.tokenizers.mistral as mt

15 +else:

16 + mt = LazyLoader("mt", globals(), "vllm.tokenizers.mistral")

17 +

18 +

19 +def is_mistral_tokenizer(obj: TokenizerLike | None) -> TypeGuard[mt.MistralTokenizer]:

20 + """Return true if the tokenizer is a MistralTokenizer instance."""

21 + cls = type(obj)

22 + # Check for special class attribute, this avoids importing the class to

23 + # do an isinstance() check. If the attribute is True, do an isinstance

24 + # check to be sure we have the correct type.

25 + return bool(

26 + getattr(cls, "IS_MISTRAL_TOKENIZER", False)

27 + and isinstance(obj, mt.MistralTokenizer)

28 + )

vllm/v1/structured_output/backend_xgrammar.py

+2 -2

@@ -10,8 +10,8 @@ import torch

10 import vllm.envs

11 from vllm.logger import init_logger

12 from vllm.sampling_params import SamplingParams

13 -from vllm.tokenizers.mistral import MistralTokenizer

13 from vllm.utils.import_utils import LazyLoader

14 +from vllm.utils.mistral import is_mistral_tokenizer

15 from vllm.v1.structured_output.backend_types import (

16 StructuredOutputBackend,

17 StructuredOutputGrammar,

@@ -38,7 +38,7 @@ class XgrammarBackend(StructuredOutputBackend):

38 self.vllm_config.structured_outputs_config.disable_any_whitespace

39 )

41 - if isinstance(self.tokenizer, MistralTokenizer):

41 + if is_mistral_tokenizer(self.tokenizer):

42 # NOTE: ideally, xgrammar should handle this accordingly.

43 # refer to https://github.com/mlc-ai/xgrammar/blob/d77c0a0173ef14779c918e3be7966ba852f7910f/python/xgrammar/tokenizer_info.py#L98

44 stop_token_ids = [self.tokenizer.eos_token_id]