@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fef56c1855
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 5, 12:15 UTC

[Mistral Grammar] Support Grammar Factory (#38150)

Signed-off-by: juliendenize <julien.denize@mistral.ai>

Julien Denize committed Apr 6, 2026 at 16:28 UTC fef56c18555e881c671acf654630732b7271c14f
10 files changed +601 -29
requirements/common.txt
+1 -1
@@ -31,7 +31,7 @@ partial-json-parser # used for parsing partial JSON outputs
31 pyzmq >= 25.0.0
32 msgspec
33 gguf >= 0.17.0
34 -mistral_common[image] >= 1.10.0
34 +mistral_common[image] >= 1.11.0
35 opencv-python-headless >= 4.13.0 # required for video IO
36 pyyaml
37 six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
requirements/rocm-test.txt
+1 -1
@@ -604,7 +604,7 @@ mcp==1.27.0
604 # via -r requirements/common.txt
605 mdurl==0.1.2
606 # via markdown-it-py
607 -mistral-common==1.10.0
607 +mistral-common==1.11.0
608 # via
609 # -c requirements/common.txt
610 # -r requirements/common.txt
requirements/test.txt
+1 -1
@@ -508,7 +508,7 @@ mbstrdecoder==1.1.3
508 # typepy
509 mdurl==0.1.2
510 # via markdown-it-py
511 -mistral-common==1.10.0
511 +mistral-common==1.11.0
512 # via
513 # -c requirements/common.txt
514 # -r requirements/test.in
tests/tokenizers_/test_mistral.py
+28
@@ -3,8 +3,10 @@
3
4 from typing import Any
5
6 +import llguidance
7 import pytest
8 from mistral_common.exceptions import InvalidMessageStructureException
9 +from mistral_common.guidance.grammar_factory import GrammarFactory
10 from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy
11
12 from vllm.tokenizers.mistral import (
@@ -2407,3 +2409,29 @@ class TestMistralTokenizer:
2409 assert actual_tokens == expected_tokens
2410
2411 assert mistral_tokenizer.convert_ids_to_tokens([]) == []
2412 +
2413 + def test_grammar_factory(self, mistral_tokenizer: MistralTokenizer) -> None:
2414 + # works in this case cause Mistral 7B is < v11 and SPM
2415 + if not mistral_tokenizer.is_tekken:
2416 + with pytest.raises(AttributeError):
2417 + mistral_tokenizer.grammar_factory # noqa: B018
2418 + return
2419 + factory = mistral_tokenizer.grammar_factory
2420 + assert isinstance(factory, GrammarFactory)
2421 +
2422 + # Test caching
2423 + factory_2 = mistral_tokenizer.grammar_factory
2424 + assert factory is factory_2
2425 +
2426 + def test_llg_tokenizer(self, mistral_tokenizer: MistralTokenizer) -> None:
2427 + if not mistral_tokenizer.is_tekken:
2428 + with pytest.raises(ValueError):
2429 + mistral_tokenizer.llg_tokenizer # noqa: B018
2430 + return
2431 +
2432 + llg_tokenizer = mistral_tokenizer.llg_tokenizer
2433 + assert isinstance(llg_tokenizer, llguidance.LLTokenizer)
2434 +
2435 + # Test caching
2436 + llg_tokenizer_2 = mistral_tokenizer.llg_tokenizer
2437 + assert llg_tokenizer is llg_tokenizer_2
tests/tool_parsers/test_mistral_tool_parser.py
+344 -3
@@ -3,19 +3,43 @@
3
4 import json
5 from collections.abc import Generator
6 +from unittest.mock import MagicMock, patch
7
8 import partial_json_parser
9 import pytest
10 from mistral_common.protocol.instruct.messages import AssistantMessage
11 from mistral_common.protocol.instruct.request import InstructRequest
11 -from mistral_common.protocol.instruct.tool_calls import FunctionCall, ToolCall
12 +from mistral_common.protocol.instruct.tool_calls import (
13 + FunctionCall,
14 + ToolCall,
15 +)
16 +from mistral_common.protocol.instruct.tool_calls import (
17 + NamedToolChoice as MistralNamedToolChoice,
18 +)
19 +from mistral_common.protocol.instruct.tool_calls import (
20 + ToolChoice as MistralToolChoice,
21 +)
22 +from mistral_common.protocol.instruct.tool_calls import (
23 + ToolChoiceEnum as MistralToolChoiceEnum,
24 +)
25 from partial_json_parser.core.options import Allow
26
14 -from vllm.entrypoints.openai.engine.protocol import DeltaMessage, DeltaToolCall
27 +from vllm.entrypoints.openai.chat_completion.protocol import (
28 + ChatCompletionRequest,
29 +)
30 +from vllm.entrypoints.openai.engine.protocol import (
31 + DeltaMessage,
32 + DeltaToolCall,
33 + StructuralTagResponseFormat,
34 +)
35 +from vllm.sampling_params import StructuredOutputsParams
36 from vllm.tokenizers import TokenizerLike, get_tokenizer
37 from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
38 from vllm.tokenizers.mistral import MistralTokenizer
18 -from vllm.tool_parsers.mistral_tool_parser import MistralToolParser
39 +from vllm.tool_parsers.mistral_tool_parser import (
40 + _DEFAULT_JSON_SCHEMA,
41 + MistralToolParser,
42 +)
43
44
45 @pytest.fixture(scope="module")
@@ -40,6 +64,13 @@ def mistral_tool_parser(mistral_tokenizer):
64 return MistralToolParser(mistral_tokenizer)
65
66
67 +@pytest.fixture
68 +def non_mistral_parser() -> MistralToolParser:
69 + mock_tokenizer = MagicMock()
70 + mock_tokenizer.get_vocab.return_value = {"[TOOL_CALLS]": 1}
71 + return MistralToolParser(mock_tokenizer)
72 +
73 +
74 def assert_tool_calls(
75 actual_tool_calls: list[ToolCall] | list[DeltaToolCall],
76 expected_tool_calls: list[ToolCall],
@@ -951,3 +982,313 @@ def test_fast_detokenization_text_detection_pre_v11(
982 assert len(delta_message.tool_calls) > 0
983 assert delta_message.tool_calls[0].function is not None
984 assert delta_message.tool_calls[0].function.name == "add"
985 +
986 +
987 +SAMPLE_TOOLS_DICTS = [
988 + {
989 + "type": "function",
990 + "function": {
991 + "name": "get_weather",
992 + "description": "Get the weather",
993 + "parameters": {
994 + "type": "object",
995 + "properties": {"city": {"type": "string"}},
996 + "required": ["city"],
997 + },
998 + },
999 + },
1000 + {
1001 + "type": "function",
1002 + "function": {
1003 + "name": "add",
1004 + "description": "Add two numbers",
1005 + "parameters": {
1006 + "type": "object",
1007 + "properties": {
1008 + "a": {"type": "number"},
1009 + "b": {"type": "number"},
1010 + },
1011 + "required": ["a", "b"],
1012 + },
1013 + },
1014 + },
1015 +]
1016 +
1017 +
1018 +def _make_request(**kwargs) -> ChatCompletionRequest:
1019 + defaults: dict = {
1020 + "messages": [],
1021 + "model": "mistralai/Mistral-Small-3.2-24B-Instruct-2506",
1022 + "tools": SAMPLE_TOOLS_DICTS,
1023 + "tool_choice": "auto",
1024 + }
1025 + defaults.update(kwargs)
1026 + return ChatCompletionRequest(**defaults)
1027 +
1028 +
1029 +@pytest.mark.parametrize(
1030 + "request_kwargs,expected_mode,expected_parallel",
1031 + [
1032 + ({"tool_choice": "auto"}, MistralToolChoiceEnum.auto, True),
1033 + ({"tool_choice": "none"}, MistralToolChoiceEnum.none, True),
1034 + ({"tool_choice": "required"}, MistralToolChoiceEnum.required, True),
1035 + ({"tool_choice": None, "tools": None}, MistralToolChoiceEnum.auto, True),
1036 + (
1037 + {
1038 + "tool_choice": {
1039 + "type": "function",
1040 + "function": {"name": "get_weather"},
1041 + }
1042 + },
1043 + MistralNamedToolChoice.model_validate(
1044 + {"type": "function", "function": {"name": "get_weather"}}
1045 + ),
1046 + True,
1047 + ),
1048 + (
1049 + {"tool_choice": "auto", "parallel_tool_calls": False},
1050 + MistralToolChoiceEnum.auto,
1051 + False,
1052 + ),
1053 + (
1054 + {"tool_choice": "auto", "response_format": {"type": "text"}},
1055 + MistralToolChoiceEnum.auto,
1056 + True,
1057 + ),
1058 + ],
1059 + ids=[
1060 + "auto",
1061 + "none",
1062 + "required",
1063 + "null_tool_choice",
1064 + "named_tool_choice",
1065 + "parallel_false",
1066 + "response_format_text",
1067 + ],
1068 +)
1069 +def test_adjust_request_grammar_factory(
1070 + mistral_tool_parser: MistralToolParser,
1071 + request_kwargs: dict,
1072 + expected_mode: MistralToolChoice,
1073 + expected_parallel: bool,
1074 +) -> None:
1075 + request = _make_request(**request_kwargs)
1076 + factory = mistral_tool_parser.model_tokenizer.grammar_factory
1077 +
1078 + with patch.object(
1079 + factory,
1080 + "get_lark_from_jinja",
1081 + wraps=factory.get_lark_from_jinja,
1082 + ) as mock_get_lark:
1083 + result = mistral_tool_parser.adjust_request(request)
1084 +
1085 + mock_get_lark.assert_called_once()
1086 + call_kwargs = mock_get_lark.call_args
1087 +
1088 + assert call_kwargs.kwargs["mode"] == expected_mode
1089 + assert call_kwargs.kwargs["json_schema"] is None
1090 + assert call_kwargs.kwargs["parallel_tool_calls"] == expected_parallel
1091 +
1092 + assert result.structured_outputs is not None
1093 + assert isinstance(result.structured_outputs.grammar, str)
1094 + assert len(result.structured_outputs.grammar) > 0
1095 +
1096 +
1097 +def test_adjust_request_unsupported_grammar_for_tokenizer(mistral_tokenizer) -> None:
1098 + with patch.object(
1099 + type(mistral_tokenizer),
1100 + "supports_grammar",
1101 + new_callable=lambda: property(lambda self: False),
1102 + ):
1103 + parser = MistralToolParser(mistral_tokenizer)
1104 + request = _make_request()
1105 + result = parser.adjust_request(request)
1106 +
1107 + assert result.structured_outputs is None
1108 +
1109 +
1110 +@pytest.mark.parametrize(
1111 + "tool_choice,expected_skip",
1112 + [("auto", False), ("none", True)],
1113 + ids=["auto_skip_false", "none_skip_true"],
1114 +)
1115 +def test_adjust_request_non_mistral_tokenizer(
1116 + non_mistral_parser: MistralToolParser,
1117 + tool_choice: str,
1118 + expected_skip: bool,
1119 +) -> None:
1120 + request = _make_request(tool_choice=tool_choice)
1121 + result = non_mistral_parser.adjust_request(request)
1122 +
1123 + assert result.skip_special_tokens is expected_skip
1124 +
1125 +
1126 +@pytest.mark.parametrize(
1127 + "so_kwargs",
1128 + [
1129 + {"regex": r"\d+"},
1130 + {"choice": ["a", "b"]},
1131 + {"structural_tag": '{"key": "value"}'},
1132 + {"grammar": "start: 'hello'"},
1133 + ],
1134 + ids=["regex", "choice", "structural_tag", "grammar"],
1135 +)
1136 +def test_adjust_request_unsupported_structured_outputs(
1137 + mistral_tool_parser: MistralToolParser,
1138 + so_kwargs: dict,
1139 +) -> None:
1140 + request = _make_request(
1141 + structured_outputs=StructuredOutputsParams(**so_kwargs),
1142 + )
1143 + result = mistral_tool_parser.adjust_request(request)
1144 +
1145 + assert result.structured_outputs == request.structured_outputs
1146 +
1147 +
1148 +def test_adjust_request_unsupported_response_format(
1149 + mistral_tool_parser: MistralToolParser,
1150 +) -> None:
1151 + request = _make_request(
1152 + response_format=StructuralTagResponseFormat(
1153 + type="structural_tag", format={"some": "config"}
1154 + ),
1155 + )
1156 + result = mistral_tool_parser.adjust_request(request)
1157 + assert result.structured_outputs is None
1158 + assert result.response_format == request.response_format
1159 +
1160 +
1161 +@pytest.mark.parametrize(
1162 + "so_kwargs,expected_json_schema",
1163 + [
1164 + ({"json_object": True}, _DEFAULT_JSON_SCHEMA),
1165 + ({"json": '{"type": "object"}'}, {"type": "object"}),
1166 + (
1167 + {"json": {"type": "object", "properties": {"x": {"type": "integer"}}}},
1168 + {"type": "object", "properties": {"x": {"type": "integer"}}},
1169 + ),
1170 + ],
1171 + ids=["json_object", "json_str", "json_dict"],
1172 +)
1173 +def test_adjust_request_structured_outputs_generates_grammar(
1174 + mistral_tool_parser: MistralToolParser,
1175 + so_kwargs: dict,
1176 + expected_json_schema: str,
1177 +) -> None:
1178 + request = _make_request(
1179 + structured_outputs=StructuredOutputsParams(**so_kwargs),
1180 + )
1181 + factory = mistral_tool_parser.model_tokenizer.grammar_factory
1182 +
1183 + with patch.object(
1184 + factory,
1185 + "get_lark_from_jinja",
1186 + wraps=factory.get_lark_from_jinja,
1187 + ) as mock_get_lark:
1188 + result = mistral_tool_parser.adjust_request(request)
1189 +
1190 + mock_get_lark.assert_called_once()
1191 + assert mock_get_lark.call_args.kwargs["json_schema"] == expected_json_schema
1192 +
1193 + assert result.structured_outputs is not None
1194 + assert isinstance(result.structured_outputs.grammar, str)
1195 + assert len(result.structured_outputs.grammar) > 0
1196 +
1197 +
1198 +@pytest.mark.parametrize(
1199 + "response_format_kwargs,expected_json_schema",
1200 + [
1201 + ({"type": "json_object"}, _DEFAULT_JSON_SCHEMA),
1202 + (
1203 + {
1204 + "type": "json_schema",
1205 + "json_schema": {
1206 + "name": "my_schema",
1207 + "schema": {
1208 + "type": "object",
1209 + "properties": {"x": {"type": "integer"}},
1210 + },
1211 + },
1212 + },
1213 + {"type": "object", "properties": {"x": {"type": "integer"}}},
1214 + ),
1215 + ],
1216 + ids=["json_object", "json_schema_with_schema"],
1217 +)
1218 +def test_adjust_request_response_format_generates_grammar(
1219 + mistral_tool_parser: MistralToolParser,
1220 + response_format_kwargs: dict,
1221 + expected_json_schema: str,
1222 +) -> None:
1223 + request = _make_request(response_format=response_format_kwargs)
1224 + factory = mistral_tool_parser.model_tokenizer.grammar_factory
1225 +
1226 + with patch.object(
1227 + factory,
1228 + "get_lark_from_jinja",
1229 + wraps=factory.get_lark_from_jinja,
1230 + ) as mock_get_lark:
1231 + result = mistral_tool_parser.adjust_request(request)
1232 +
1233 + mock_get_lark.assert_called_once()
1234 + assert mock_get_lark.call_args.kwargs["json_schema"] == expected_json_schema
1235 +
1236 + assert result.structured_outputs is not None
1237 + assert isinstance(result.structured_outputs.grammar, str)
1238 + assert len(result.structured_outputs.grammar) > 0
1239 +
1240 +
1241 +def test_adjust_request_tool_choice_none_with_json_schema_uses_json_schema_factory(
1242 + mistral_tool_parser: MistralToolParser,
1243 +) -> None:
1244 + request = _make_request(
1245 + tool_choice="none",
1246 + structured_outputs=StructuredOutputsParams(json='{"type": "object"}'),
1247 + )
1248 + factory = mistral_tool_parser.model_tokenizer.grammar_factory
1249 +
1250 + with patch.object(
1251 + factory,
1252 + "get_lark_for_json_schema",
1253 + wraps=factory.get_lark_for_json_schema,
1254 + ) as mock_json_schema:
1255 + result = mistral_tool_parser.adjust_request(request)
1256 +
1257 + mock_json_schema.assert_called_once()
1258 + assert mock_json_schema.call_args.kwargs["json_schema"] == {"type": "object"}
1259 +
1260 + assert result.structured_outputs is not None
1261 + assert isinstance(result.structured_outputs.grammar, str)
1262 + assert len(result.structured_outputs.grammar) > 0
1263 +
1264 +
1265 +def test_adjust_request_tool_choice_auto_with_json_schema_uses_jinja_factory(
1266 + mistral_tool_parser: MistralToolParser,
1267 +) -> None:
1268 + request = _make_request(
1269 + tool_choice="auto",
1270 + structured_outputs=StructuredOutputsParams(json='{"type": "object"}'),
1271 + )
1272 + factory = mistral_tool_parser.model_tokenizer.grammar_factory
1273 +
1274 + with (
1275 + patch.object(
1276 + factory,
1277 + "get_lark_for_json_schema",
1278 + wraps=factory.get_lark_for_json_schema,
1279 + ) as mock_json_schema,
1280 + patch.object(
1281 + factory,
1282 + "get_lark_from_jinja",
1283 + wraps=factory.get_lark_from_jinja,
1284 + ) as mock_jinja,
1285 + ):
1286 + result = mistral_tool_parser.adjust_request(request)
1287 +
1288 + mock_jinja.assert_called_once()
1289 + assert mock_jinja.call_args.kwargs["json_schema"] == {"type": "object"}
1290 + mock_json_schema.assert_not_called()
1291 +
1292 + assert result.structured_outputs is not None
1293 + assert isinstance(result.structured_outputs.grammar, str)
1294 + assert len(result.structured_outputs.grammar) > 0
tests/v1/structured_output/test_backend_guidance.py
+44
@@ -11,6 +11,7 @@ from vllm.config.model import ModelConfig
11 from vllm.config.parallel import ParallelConfig
12 from vllm.config.speculative import SpeculativeConfig
13 from vllm.sampling_params import SamplingParams, StructuredOutputsParams
14 +from vllm.tokenizers import get_tokenizer
15 from vllm.v1.request import Request
16 from vllm.v1.structured_output import StructuredOutputManager
17 from vllm.v1.structured_output.backend_guidance import GuidanceBackend
@@ -19,6 +20,14 @@ from vllm.v1.structured_output.backend_types import StructuredOutputOptions
20 TOKENIZER = "gpt2"
21
22
23 +@pytest.fixture(scope="module")
24 +def mistral_tokenizer():
25 + return get_tokenizer(
26 + tokenizer_name="mistralai/Mistral-Small-3.2-24B-Instruct-2506",
27 + tokenizer_mode="mistral",
28 + )
29 +
30 +
31 def test_backend_guidance_rollback_terminated():
32 # Test that the backend guidance successfully rollbacks from a
33 # terminated state. This can happen with speculative decoding,
@@ -187,3 +196,38 @@ def test_grammar_init_async_and_sync(async_grammar):
196
197 # Verify the grammar can accept valid tokens
198 assert grammar.accept_tokens(request.request_id, prompt)
199 +
200 +
201 +@pytest.mark.parametrize(
202 + "request_type,grammar_spec",
203 + [
204 + pytest.param(
205 + StructuredOutputOptions.JSON,
206 + '{"type": "object"}',
207 + id="json",
208 + ),
209 + pytest.param(
210 + StructuredOutputOptions.GRAMMAR,
211 + 'start: "hello" | "world"',
212 + id="lark",
213 + ),
214 + ],
215 +)
216 +def test_mistral_tokenizer_compile_grammar(
217 + mistral_tokenizer,
218 + request_type: StructuredOutputOptions,
219 + grammar_spec: str,
220 +) -> None:
221 + vllm_config = VllmConfig(
222 + structured_outputs_config=StructuredOutputsConfig(backend="guidance"),
223 + )
224 + backend = GuidanceBackend(
225 + vllm_config,
226 + tokenizer=mistral_tokenizer,
227 + vocab_size=mistral_tokenizer.vocab_size,
228 + )
229 + assert backend.ll_tokenizer is mistral_tokenizer.llg_tokenizer
230 +
231 + grammar = backend.compile_grammar(request_type, grammar_spec)
232 + assert grammar is not None
233 + assert not grammar.is_terminated()
vllm/sampling_params.py
+17 -11
@@ -153,6 +153,10 @@ class RequestOutputKind(Enum):
153 FINAL_ONLY = 2
154
155
156 +def _is_non_tekken_mistral(tokenizer: TokenizerLike) -> bool:
157 + return is_mistral_tokenizer(tokenizer) and not tokenizer.is_tekken
158 +
159 +
160 class SamplingParams(
161 PydanticMsgspecMixin,
162 msgspec.Struct,
@@ -801,16 +805,17 @@ class SamplingParams(
805 # xgrammar with no fallback
806 validate_xgrammar_grammar(self)
807 elif backend.startswith("guidance"):
808 + if _is_non_tekken_mistral(tokenizer=tokenizer):
809 + raise ValueError(
810 + "Non-tekken Mistral tokenizers are not supported for the 'guidance'"
811 + " structured output backend. Please either use a more recent "
812 + "Mistral model, the ['xgrammar', 'outlines'] "
813 + "backends or tokenizer_mode='hf' instead."
814 + )
815 # TODO: ideally we would have the LLTokenizer here as Lark syntax
816 # allows <|special_token|> and similar, see
817 # https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens
818 # Without tokenizer these are disallowed in grammars.
808 - if is_mistral_tokenizer(tokenizer):
809 - raise ValueError(
810 - "Mistral tokenizer is not supported for the 'guidance' "
811 - "structured output backend. Please use ['xgrammar', 'outlines'] "
812 - "backends or tokenizer_mode='hf' instead."
813 - )
819 validate_guidance_grammar(self, tokenizer=None)
820 elif backend == "outlines":
821 # outlines backend
@@ -839,19 +844,20 @@ class SamplingParams(
844 # or includes some jsonschema feature(s) that
845 # are not supported in xgrammar.
846
847 + skip_guidance = _is_non_tekken_mistral(tokenizer)
848 +
849 # Check if schema has features unsupported by guidance
850 so_params = self.structured_outputs
844 - skip_guidance = False
845 - if so_params.json:
851 + if not skip_guidance and so_params.json:
852 if isinstance(so_params.json, str):
853 schema = json_mod.loads(so_params.json)
854 else:
855 schema = so_params.json
856 skip_guidance = has_guidance_unsupported_json_features(schema)
857
852 - if is_mistral_tokenizer(tokenizer) or skip_guidance:
853 - # Fall back to outlines if the tokenizer is Mistral
854 - # or if schema contains features unsupported by guidance
858 + if skip_guidance:
859 + # Fall back to outlines if the tokenizer is non-tekken Mistral or
860 + # the schema contains features unsupported by guidance
861 validate_structured_output_request_outlines(self)
862 self.structured_outputs._backend = "outlines"
863 else:
vllm/tokenizers/mistral.py
+25
@@ -1,9 +1,12 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 from collections.abc import Sequence
4 +from functools import cached_property
5 from pathlib import Path
6 from typing import TYPE_CHECKING, Any, cast, overload
7
8 +from mistral_common.guidance.grammar_factory import GrammarFactory
9 +from mistral_common.guidance.tokenizer import from_mistral_tokenizer
10 from mistral_common.protocol.instruct.request import (
11 ChatCompletionRequest as MistralChatCompletionRequest,
12 )
@@ -45,6 +48,7 @@ except ImportError:
48 )
49
50 if TYPE_CHECKING:
51 + import llguidance
52 from transformers import BatchEncoding
53
54 logger = init_logger(__name__)
@@ -574,3 +578,24 @@ class MistralTokenizer(TokenizerLike):
578 ]
579
580 return tokens
581 +
582 + @property
583 + def supports_grammar(self) -> bool:
584 + return GrammarFactory.is_supported(self.mistral)
585 +
586 + @cached_property
587 + def grammar_factory(self) -> GrammarFactory:
588 + if not self.supports_grammar:
589 + raise AttributeError(
590 + "This tokenizer does not support `grammar_factory`. "
591 + "This is only supported for tekken tokenizers with "
592 + "version >= 11."
593 + )
594 + # Cache grammar factory to avoid creating a llguidance tokenizer at every usage.
595 + return GrammarFactory(self.mistral)
596 +
597 + @cached_property
598 + def llg_tokenizer(self) -> "llguidance.LLTokenizer":
599 + if not self.is_tekken:
600 + raise ValueError("`llg_tokenizer` is only supported for Tekkenizers.")
601 + return from_mistral_tokenizer(self.mistral)
vllm/tool_parsers/mistral_tool_parser.py
+133 -9
@@ -10,6 +10,18 @@ from typing import Any
10
11 import ijson
12 import regex as re
13 +from mistral_common.protocol.instruct.tool_calls import (
14 + NamedToolChoice as MistralNamedToolChoice,
15 +)
16 +from mistral_common.protocol.instruct.tool_calls import (
17 + Tool as MistralTool,
18 +)
19 +from mistral_common.protocol.instruct.tool_calls import (
20 + ToolChoice as MistralToolChoice,
21 +)
22 +from mistral_common.protocol.instruct.tool_calls import (
23 + ToolChoiceEnum as MistralToolChoiceEnum,
24 +)
25 from pydantic import Field
26
27 from vllm.entrypoints.openai.chat_completion.protocol import (
@@ -25,6 +37,7 @@ from vllm.entrypoints.openai.engine.protocol import (
37 )
38 from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
39 from vllm.logger import init_logger
40 +from vllm.sampling_params import StructuredOutputsParams
41 from vllm.tokenizers import TokenizerLike
42 from vllm.tool_parsers.abstract_tool_parser import (
43 Tool,
@@ -36,6 +49,8 @@ logger = init_logger(__name__)
49
50 ALPHANUMERIC = ascii_letters + digits
51
52 +_DEFAULT_JSON_SCHEMA = {"anyOf": [{"type": "object"}, {"type": "array"}]}
53 +
54
55 class StreamingState(Enum):
56 """Enum for tracking the current streaming parsing state."""
@@ -80,6 +95,9 @@ class MistralToolParser(ToolParser):
95 Used when --enable-auto-tool-choice --tool-call-parser mistral are all set
96 """
97
98 + # Used to generate correct grammar in `adjust_request`
99 + model_can_reason: bool = False
100 +
101 def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
102 super().__init__(tokenizer, tools)
103
@@ -115,18 +133,124 @@ class MistralToolParser(ToolParser):
133 def adjust_request(
134 self, request: ChatCompletionRequest | ResponsesRequest
135 ) -> ChatCompletionRequest | ResponsesRequest:
118 - request = super().adjust_request(request)
136 + so_non_supported_attributes = [
137 + "regex",
138 + "choice",
139 + "grammar",
140 + # whitespace_pattern is not a constraint type but an option;
141 + # Mistral grammar factory does not support it.
142 + "whitespace_pattern",
143 + "structural_tag",
144 + ]
145 + any_so_non_supported_active = request.structured_outputs is not None and any(
146 + getattr(request.structured_outputs, attribute) is not None
147 + for attribute in so_non_supported_attributes
148 + )
149 + response_format_non_supported_active = (
150 + isinstance(request, ResponsesRequest)
151 + or request.response_format is not None
152 + and request.response_format.type == "structural_tag"
153 + )
154 +
155 if (
156 not is_mistral_tokenizer(self.model_tokenizer)
121 - and request.tools
122 - and request.tool_choice != "none"
157 + or isinstance(request, ResponsesRequest)
158 + or not self.model_tokenizer.supports_grammar
159 + or any_so_non_supported_active
160 + or response_format_non_supported_active
161 + ):
162 + request = super().adjust_request(request)
163 + if request.tools and request.tool_choice != "none":
164 + # Do not skip special tokens when using chat template
165 + # with Mistral parser as TOOL_CALL token is needed
166 + # for tool detection.
167 + # Note: we don't want skip_special_tokens=False
168 + # with MistralTokenizer as it is incompatible
169 + request.skip_special_tokens = False
170 + return request
171 +
172 + json_schema: dict[str, Any] | None = None
173 + if request.structured_outputs is not None:
174 + if request.structured_outputs.json_object is not None:
175 + json_schema = _DEFAULT_JSON_SCHEMA
176 + elif request.structured_outputs.json is not None:
177 + if isinstance(request.structured_outputs.json, str):
178 + json_schema = json.loads(request.structured_outputs.json)
179 + else:
180 + json_schema = request.structured_outputs.json
181 + else:
182 + raise ValueError(
183 + "Unsupported request.structured_outputs for MistralToolParser. "
184 + "Only `json` and `json_object` are supported."
185 + )
186 + elif (
187 + request.response_format is not None
188 + and request.response_format.type != "text"
189 ):
124 - # Do not skip special tokens when using chat template
125 - # with Mistral parser as TOOL_CALL token is needed
126 - # for tool detection.
127 - # Note: we don't want skip_special_tokens=False
128 - # with MistralTokenizer as it is incompatible
129 - request.skip_special_tokens = False
190 + if request.response_format.type == "json_object":
191 + json_schema = _DEFAULT_JSON_SCHEMA
192 + elif request.response_format.type == "json_schema":
193 + if request.response_format.json_schema is not None:
194 + json_schema = request.response_format.json_schema.json_schema
195 + else:
196 + json_schema = _DEFAULT_JSON_SCHEMA
197 + else:
198 + raise ValueError(
199 + "MistralToolParser only accepts `text`, `json_object` or "
200 + f"`json_schema`, got {request.response_format=}"
201 + )
202 + # Structured Outputs will be defined.
203 + request.response_format = None
204 +
205 + grammar_factory = self.model_tokenizer.grammar_factory
206 +
207 + # TODO: Once unified parser, improve this.
208 + # The issue is figuring out when a model is a reasoning one or not.
209 + template = grammar_factory.select_jinja_template(
210 + reasoning=self.model_can_reason
211 + )
212 +
213 + tools = (
214 + [
215 + MistralTool.from_openai(openai_tool=tool.model_dump())
216 + for tool in request.tools
217 + ]
218 + if request.tools is not None
219 + else None
220 + )
221 +
222 + tool_choice: MistralToolChoice
223 + match request.tool_choice:
224 + case "none" | "auto" | "required":
225 + tool_choice = MistralToolChoiceEnum(request.tool_choice)
226 + case None:
227 + tool_choice = MistralToolChoiceEnum.auto
228 + # _ == Named tool choice
229 + case _:
230 + tool_choice = MistralNamedToolChoice.model_validate(
231 + {
232 + "type": "function",
233 + "function": {"name": request.tool_choice.function.name},
234 + }
235 + )
236 +
237 + # Rendering grammar is cached in mistral-common given tools, template and mode.
238 + match tool_choice, json_schema is not None:
239 + case MistralToolChoiceEnum.none, True:
240 + lark_grammar = grammar_factory.get_lark_for_json_schema(
241 + template=template, json_schema=json_schema
242 + )
243 + case _, _:
244 + lark_grammar = grammar_factory.get_lark_from_jinja(
245 + template=template,
246 + mode=tool_choice,
247 + tools=tools,
248 + json_schema=json_schema,
249 + parallel_tool_calls=request.parallel_tool_calls,
250 + json_only=False,
251 + )
252 +
253 + request.structured_outputs = StructuredOutputsParams(grammar=lark_grammar)
254 return request
255
256 def extract_tool_calls(
vllm/v1/structured_output/backend_guidance.py
+7 -3
@@ -12,6 +12,7 @@ import torch
12 from vllm.logger import init_logger
13 from vllm.sampling_params import SamplingParams
14 from vllm.utils.import_utils import LazyLoader
15 +from vllm.utils.mistral import is_mistral_tokenizer
16 from vllm.v1.structured_output.backend_types import (
17 StructuredOutputBackend,
18 StructuredOutputGrammar,
@@ -92,9 +93,12 @@ class GuidanceBackend(StructuredOutputBackend):
93 self.vllm_config.structured_outputs_config.disable_additional_properties
94 )
95
95 - self.ll_tokenizer = llguidance_hf.from_tokenizer(
96 - self.tokenizer, max(self.vocab_size, len(self.tokenizer))
97 - )
96 + if is_mistral_tokenizer(self.tokenizer):
97 + self.ll_tokenizer = self.tokenizer.llg_tokenizer
98 + else:
99 + self.ll_tokenizer = llguidance_hf.from_tokenizer(
100 + self.tokenizer, max(self.vocab_size, len(self.tokenizer))
101 + )
102
103 def compile_grammar(
104 self, request_type: StructuredOutputOptions, grammar_spec: str