@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 56bdf85e10

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 4, 17:30 UTC

[Feature] Avoid eager import of the "mistral_common" package. (#40043)

Signed-off-by: Neil Schemenauer <nas@arctrix.com>

Neil Schemenauer committed Apr 23, 2026 at 19:49 UTC 56bdf85e10b807be13225f659f2593051306c77d

5 files changed +47 -23

vllm/entrypoints/openai/chat_completion/serving.py

+26 -17

@@ -73,13 +73,9 @@ from vllm.reasoning import ReasoningParser

73 from vllm.renderers import ChatParams

74 from vllm.sampling_params import BeamSearchParams, SamplingParams

75 from vllm.tokenizers import TokenizerLike

76 -from vllm.tool_parsers.mistral_tool_parser import (

77 - MistralToolCall,

78 - MistralToolParser,

79 -)

76 from vllm.tool_parsers.utils import partial_json_loads

77 from vllm.utils.collection_utils import as_list

82 -from vllm.utils.mistral import is_mistral_tokenizer

78 +from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser

80 if TYPE_CHECKING:

81 from vllm.entrypoints.serve.render.serving import OpenAIServingRender

@@ -143,10 +139,12 @@ class OpenAIServingChat(OpenAIServing):

139 enable_auto_tools=enable_auto_tools,

140 model_name=self.model_config.model,

141 )

146 - _is_mistral_tool_parser = self.tool_parser is not None and issubclass(

147 - self.tool_parser, MistralToolParser

148 - )

149 - if _is_mistral_tool_parser and self.reasoning_parser_cls is not None:

142 + if (

143 + is_mistral_tool_parser(self.tool_parser)

144 + and self.reasoning_parser_cls is not None

145 + ):

146 + from vllm.tool_parsers.mistral_tool_parser import MistralToolParser

147 +

148 MistralToolParser.model_can_reason = True

149

150 self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none

@@ -823,6 +821,10 @@ class OpenAIServingChat(OpenAIServing):

821 harmony_tools_streamed[i] |= tools_streamed_flag

822 # Mistral grammar path: combined reasoning + tool streaming

823 elif is_mistral_grammar_path:

824 + from vllm.tool_parsers.mistral_tool_parser import (

825 + MistralToolParser,

826 + )

827 +

828 assert tool_parser is not None

829 assert isinstance(tool_parser, MistralToolParser)

830 assert reasoning_end_arr is not None

@@ -904,6 +906,10 @@ class OpenAIServingChat(OpenAIServing):

906 else:

907 # Generate ID based on tokenizer type

908 if is_mistral_tokenizer(tokenizer):

909 + from vllm.tool_parsers.mistral_tool_parser import (

910 + MistralToolCall,

911 + )

912 +

913 tool_call_id = MistralToolCall.generate_random_id()

914 else:

915 tool_call_id = make_tool_call_id(

@@ -1275,8 +1281,6 @@ class OpenAIServingChat(OpenAIServing):

1281 request_metadata: RequestResponseMetadata,

1282 reasoning_parser: ReasoningParser | None = None,

1283 ) -> ErrorResponse | ChatCompletionResponse:

1278 - from vllm.tokenizers.mistral import MistralTokenizer

1279 -

1284 created_time = int(time.time())

1285 final_res: RequestOutput | None = None

1286

@@ -1393,12 +1397,17 @@ class OpenAIServingChat(OpenAIServing):

1397 enable_auto_tools=self.enable_auto_tools,

1398 tool_parser_cls=self.tool_parser,

1399 )

1396 - tool_call_class = (

1397 - MistralToolCall if is_mistral_tokenizer(tokenizer) else ToolCall

1398 - )

1400 + if is_mistral_tokenizer(tokenizer):

1401 + from vllm.tool_parsers.mistral_tool_parser import MistralToolCall

1402 +

1403 + tool_call_class: type[ToolCall] = MistralToolCall

1404 + else:

1405 + tool_call_class = ToolCall

1406

1407 use_mistral_tool_parser = request._grammar_from_tool_parser

1408 if use_mistral_tool_parser:

1409 + from vllm.tool_parsers.mistral_tool_parser import MistralToolParser

1410 +

1411 tool_call_items = MistralToolParser.build_non_streaming_tool_calls(

1412 tool_calls

1413 )

@@ -1436,7 +1445,7 @@ class OpenAIServingChat(OpenAIServing):

1445 # Generate ID using the correct format (kimi_k2 or random),

1446 # but leave it to the class if it's Mistral to preserve

1447 # 9-char IDs

1439 - if isinstance(tokenizer, MistralTokenizer):

1448 + if is_mistral_tokenizer(tokenizer):

1449 tool_call_class_items.append(tool_call_class(function=tc))

1450 else:

1451 generated_id = make_tool_call_id(

@@ -1469,7 +1478,7 @@ class OpenAIServingChat(OpenAIServing):

1478 # Generate ID using the correct format (kimi_k2 or random),

1479 # but leave it to the class if it's Mistral to preserve

1480 # 9-char IDs

1472 - if isinstance(tokenizer, MistralTokenizer):

1481 + if is_mistral_tokenizer(tokenizer):

1482 tool_call_class_items.append(

1483 tool_call_class(function=tool_call)

1484 )

@@ -1519,7 +1528,7 @@ class OpenAIServingChat(OpenAIServing):

1528 # Generate ID using the correct format (kimi_k2 or random),

1529 # but leave it to the class if it's Mistral to preserve

1530 # 9-char IDs

1522 - if isinstance(tokenizer, MistralTokenizer):

1531 + if is_mistral_tokenizer(tokenizer):

1532 tool_call_items.append(tool_call_class(function=tc))

1533 else:

1534 generated_id = make_tool_call_id(

vllm/entrypoints/openai/engine/serving.py

+2 -3

@@ -65,7 +65,6 @@ from vllm.renderers.inputs.preprocess import (

65 from vllm.sampling_params import BeamSearchParams, SamplingParams

66 from vllm.tokenizers import TokenizerLike

67 from vllm.tool_parsers import ToolParser

68 -from vllm.tool_parsers.mistral_tool_parser import MistralToolParser

68 from vllm.tracing import (

69 contains_trace_headers,

70 extract_trace_headers,

@@ -73,6 +72,7 @@ from vllm.tracing import (

72 )

73 from vllm.utils import random_uuid

74 from vllm.utils.async_utils import collect_from_async_generator

75 +from vllm.utils.mistral import is_mistral_tool_parser

77 logger = init_logger(__name__)

@@ -615,8 +615,7 @@ class OpenAIServing:

615 # let the parser handle the output.

616 use_mistral_tool_parser = (

617 isinstance(request, ChatCompletionRequest)

618 - and tool_parser_cls is not None

619 - and issubclass(tool_parser_cls, MistralToolParser)

618 + and is_mistral_tool_parser(tool_parser_cls)

619 and request._grammar_from_tool_parser

620 )

621

vllm/entrypoints/serve/render/serving.py

+2 -3

@@ -55,9 +55,8 @@ from vllm.renderers.inputs.preprocess import (

55 prompt_to_seq,

56 )

57 from vllm.tool_parsers import ToolParser

58 -from vllm.tool_parsers.mistral_tool_parser import MistralToolParser

58 from vllm.utils import random_uuid

60 -from vllm.utils.mistral import is_mistral_tokenizer

59 +from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser

60 from vllm.utils.mistral import mt as _mt

62 logger = init_logger(__name__)

@@ -582,7 +581,7 @@ class OpenAIServingRender:

581 tool_choice = getattr(request, "tool_choice", "none")

582 tokenizer = renderer.get_tokenizer()

583 is_mistral_grammar_eligible = (

585 - issubclass(tool_parser, MistralToolParser)

584 + is_mistral_tool_parser(tool_parser)

585 and is_mistral_tokenizer(tokenizer)

586 and tokenizer.supports_grammar

587 )

vllm/tool_parsers/mistral_tool_parser.py

@@ -118,6 +118,8 @@ class MistralToolParser(ToolParser):

118 set.

119 """

120

121 + IS_MISTRAL_TOOL_PARSER = True # used by vllm.utils.mistral

122 +

123 # Used to generate correct grammar in `adjust_request`

124 model_can_reason: bool = False

125

vllm/utils/mistral.py

+15

@@ -12,8 +12,10 @@ from vllm.utils.import_utils import LazyLoader

12 if TYPE_CHECKING:

13 # if type checking, eagerly import the module

14 import vllm.tokenizers.mistral as mt

15 + import vllm.tool_parsers.mistral_tool_parser as mtp

16 else:

17 mt = LazyLoader("mt", globals(), "vllm.tokenizers.mistral")

18 + mtp = LazyLoader("mtp", globals(), "vllm.tool_parsers.mistral_tool_parser")

21 def is_mistral_tokenizer(obj: TokenizerLike | None) -> TypeGuard[mt.MistralTokenizer]:

@@ -26,3 +28,16 @@ def is_mistral_tokenizer(obj: TokenizerLike | None) -> TypeGuard[mt.MistralToken

28 getattr(cls, "IS_MISTRAL_TOKENIZER", False)

29 and isinstance(obj, mt.MistralTokenizer)

30 )

31 +

32 +

33 +def is_mistral_tool_parser(cls: type | None) -> bool:

34 + """Return true if *cls* is (a subclass of) MistralToolParser.

35 +

36 + Uses a class attribute check so that importing

37 + ``vllm.tool_parsers.mistral_tool_parser`` — and transitively

38 + ``mistral_common`` — is not required.

39 + """

40 + return bool(

41 + getattr(cls, "IS_MISTRAL_TOOL_PARSER", False)

42 + and issubclass(cls, mtp.MistralToolParser) # type: ignore[arg-type]

43 + )