@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 1a20d23dab

[PARSER][Mistral] unified engine-based parser for reasoning and tool calls (#48947)

Julien Denize committed Jul 30, 2026 at 15:38 UTC 1a20d23dab6eef0ce6ab97b7e03c944683cdf90b
30 files changed +2931 -1492
requirements/common.txt
+1 -1
@@ -31,7 +31,7 @@ partial-json-parser # used for parsing partial JSON outputs
31 jsonschema >= 4.23.0 # required for MiniMax M3 tool schema validation
32 pyzmq >= 25.0.0
33 msgspec
34 -mistral_common[image] >= 1.11.5
34 +mistral_common[image] >= 1.11.6
35 opencv-python-headless >= 4.13.0 # required for video IO
36 pyyaml
37 six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
requirements/test/cpu.txt
+1 -1
@@ -476,7 +476,7 @@ mcp==1.28.1
476 # via -r requirements/test/../common.txt
477 mdurl==0.1.2
478 # via markdown-it-py
479 -mistral-common==1.11.5
479 +mistral-common==1.11.6
480 # via
481 # -r requirements/test/../common.txt
482 # -r requirements/test/cuda.in
requirements/test/cuda.in
+1 -1
@@ -33,7 +33,7 @@ torchaudio==2.11.0
33 torchvision==0.28.0
34 transformers_stream_generator # required for qwen-vl test
35 matplotlib # required for qwen-vl test
36 -mistral_common[image,audio] >= 1.11.5 # required for voxtral test
36 +mistral_common[image,audio] >= 1.11.6 # required for voxtral test
37 num2words # required for smolvlm test
38 open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py
39 datamodel_code_generator # required for minicpm3 test
requirements/test/cuda.txt
+1 -1
@@ -502,7 +502,7 @@ mcp==1.28.1
502 # via -r requirements/test/../common.txt
503 mdurl==0.1.2
504 # via markdown-it-py
505 -mistral-common==1.11.5
505 +mistral-common==1.11.6
506 # via
507 # -c requirements/common.txt
508 # -r requirements/test/../common.txt
requirements/test/nightly-torch.txt
+1 -1
@@ -23,7 +23,7 @@ jiwer # required for audio tests
23 timm # required for internvl test
24 transformers_stream_generator # required for qwen-vl test
25 matplotlib # required for qwen-vl test
26 -mistral_common[image,audio] >= 1.11.5 # required for voxtral test
26 +mistral_common[image,audio] >= 1.11.6 # required for voxtral test
27 num2words # required for smolvlm test
28 opencv-python-headless >= 4.13.0 # required for video test
29 datamodel_code_generator # required for minicpm3 test
requirements/test/rocm.in
+1 -1
@@ -29,7 +29,7 @@ tblib # for pickling test exceptions
29 timm>=1.0.17 # required for internvl and gemma3n-mm test
30 transformers_stream_generator # required for qwen-vl test
31 matplotlib # required for qwen-vl test
32 -mistral_common[image,audio]>=1.11.5 # required for voxtral test
32 +mistral_common[image,audio]>=1.11.6 # required for voxtral test
33 num2words # required for smolvlm test
34 open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py
35 datamodel_code_generator # required for minicpm3 test
requirements/test/rocm.txt
+1 -1
@@ -497,7 +497,7 @@ mcp==1.27.0
497 # via -r requirements/test/../common.txt
498 mdurl==0.1.2
499 # via markdown-it-py
500 -mistral-common==1.11.5
500 +mistral-common==1.11.6
501 # via
502 # -c requirements/common.txt
503 # -r requirements/test/../common.txt
requirements/test/xpu.txt
+1 -1
@@ -373,7 +373,7 @@ mcp==1.28.1
373 # via -r requirements/test/../common.txt
374 mdurl==0.1.2
375 # via markdown-it-py
376 -mistral-common==1.11.5
376 +mistral-common==1.11.6
377 # via
378 # -c requirements/common.txt
379 # -r requirements/test/../common.txt
tests/models/language/generation/test_mistral.py
+2 -4
@@ -5,12 +5,10 @@ import json
5
6 import pytest
7
8 +from vllm.parser.mistral import MistralToolCall
9 from vllm.sampling_params import SamplingParams
10 from vllm.tokenizers.mistral import MistralTokenizer
10 -from vllm.tool_parsers.mistral_tool_parser import (
11 - MistralToolCall,
12 - MistralToolParser,
13 -)
11 +from vllm.tool_parsers.mistral_tool_parser import MistralToolParser
12
13 from ...utils import check_logprobs_close
14
tests/parser/engine/test_delegating_replay.py
+6
@@ -40,6 +40,7 @@ from vllm.parser.engine.adapters import (
40 ParserEngineReasoningAdapter,
41 ParserEngineToolAdapter,
42 )
43 +from vllm.parser.mistral import MistralParser
44
45 _TOOLS_VALIDATOR = TypeAdapter(list[ChatCompletionToolsParam])
46
@@ -84,6 +85,11 @@ def _discover_pairings() -> list[_PairingInfo]:
85 for engine_cls, adapters in engines.items():
86 if "tool" not in adapters or "reasoning" not in adapters:
87 continue
88 + if engine_cls is MistralParser:
89 + # Mistral uses brace-balanced JSON tool args with no TOOL_END
90 + # token, so it does not fit this TOOL_END-based replay harness.
91 + # It is covered by tests/parser/mistral/ instead.
92 + continue
93 cfg = engine_cls(bare_tok, None).parser_engine_config
94 if cfg.name not in _BUILDERS:
95 missing_builders.append(f"{engine_cls.__name__} (config.name={cfg.name!r})")
tests/parser/engine/test_replay.py
+6
@@ -34,6 +34,7 @@ from tests.parser.engine.trace_builder import _BUILDERS, build_samples
34 from vllm.parser.engine import registered_adapters as _adapters_mod
35 from vllm.parser.engine.parser_engine import ParserEngine
36 from vllm.parser.engine.parser_engine_config import ParserState
37 +from vllm.parser.mistral import MistralParser
38
39 # ── Parser discovery ─────────────────────────────────────────────────
40
@@ -64,6 +65,11 @@ def _discover_parsers() -> list[_ParserInfo]:
65 and obj is not ParserEngine
66 ):
67 continue
68 + if obj is MistralParser:
69 + # Mistral uses brace-balanced JSON tool args with no TOOL_END
70 + # token, so it does not fit this TOOL_END-based replay harness.
71 + # It is covered by tests/parser/mistral/ instead.
72 + continue
73 cfg = obj(bare_tok, None).parser_engine_config
74 if cfg.name not in _BUILDERS:
75 missing_builders.append(f"{obj.__name__} (config.name={cfg.name!r})")
tests/parser/mistral/__init__.py new
+2
@@ -0,0 +1,2 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
tests/parser/mistral/test_reasoning.py new
+599
@@ -0,0 +1,599 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import json
5 +
6 +import pytest
7 +from mistral_common.protocol.instruct.messages import (
8 + AssistantMessage,
9 + ThinkChunk,
10 + ToolMessage,
11 + UserMessage,
12 +)
13 +from mistral_common.protocol.instruct.request import InstructRequest
14 +from mistral_common.protocol.instruct.tool_calls import FunctionCall, ToolCall
15 +
16 +from tests.reasoning.utils import run_reasoning_extraction_mistral
17 +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
18 +from vllm.parser.parser_manager import ParserManager
19 +from vllm.reasoning import ReasoningParser, ReasoningParserManager
20 +from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
21 +from vllm.tokenizers.mistral import MistralTokenizer
22 +
23 +_PARSER_NAME = "mistral"
24 +_MODEL_V13 = "mistralai/Magistral-Small-2509"
25 +_MODEL_V11 = "mistralai/Magistral-Small-2506"
26 +
27 +_SAMPLE_TOOLS = [
28 + {
29 + "type": "function",
30 + "function": {
31 + "name": "get_weather",
32 + "description": "Get the current weather in a city.",
33 + "parameters": {
34 + "type": "object",
35 + "properties": {"city": {"type": "string"}},
36 + "required": ["city"],
37 + },
38 + },
39 + }
40 +]
41 +
42 +
43 +@pytest.fixture(scope="module")
44 +def mistral_tokenizer() -> MistralTokenizer:
45 + """v13 Magistral tokenizer with `[THINK]`/`[/THINK]` special tokens."""
46 + return MistralTokenizer.from_pretrained(_MODEL_V13)
47 +
48 +
49 +@pytest.fixture(scope="module")
50 +def mistral_v11_tokenizer() -> MistralTokenizer:
51 + """v11 Magistral tokenizer using plain-text `<think>`/`</think>` tags."""
52 + return MistralTokenizer.from_pretrained(_MODEL_V11)
53 +
54 +
55 +def _encode_v13(tokenizer: MistralTokenizer, output: str) -> list[int]:
56 + """Encode output string with `[THINK]`/`[/THINK]` markers into token IDs.
57 +
58 + `[THINK]` and `[/THINK]` placeholders in `output` are replaced by their
59 + special-token IDs; all surrounding text is encoded with the base tokenizer.
60 + Mirrors the encoding helper from the deleted
61 + `tests/reasoning/test_mistral_reasoning_parser.py`.
62 + """
63 + think_start = "[THINK]"
64 + think_end = "[/THINK]"
65 + len_start = len(think_start)
66 + len_end = len(think_end)
67 +
68 + index_start = output.find(think_start)
69 + index_end = output.find(think_end)
70 +
71 + out: list[int] = []
72 + if index_start != -1:
73 + out += tokenizer.tokenizer.encode(output[:index_start], False, False)
74 + out += [tokenizer.instruct.BEGIN_THINK]
75 +
76 + if index_end != -1:
77 + middle = output[index_start + len_start : index_end]
78 + suffix = output[index_end + len_end :]
79 + out += tokenizer.tokenizer.encode(middle, False, False)
80 + out += [tokenizer.instruct.END_THINK]
81 + out += tokenizer.tokenizer.encode(suffix, False, False)
82 + else:
83 + out += tokenizer.tokenizer.encode(
84 + output[index_start + len_start :], False, False
85 + )
86 + elif index_end != -1:
87 + out += tokenizer.tokenizer.encode(output[:index_end], False, False)
88 + out += [tokenizer.instruct.END_THINK]
89 + out += tokenizer.tokenizer.encode(output[index_end + len_end :], False, False)
90 + else:
91 + out += tokenizer.tokenizer.encode(output, False, False)
92 + return out
93 +
94 +
95 +_TEST_CASES = [
96 + # v13 special-token encoding: [THINK]…[/THINK] with trailing content.
97 + pytest.param(
98 + False,
99 + "[THINK]r[/THINK]c",
100 + "r",
101 + "c",
102 + id="v13_basic",
103 + ),
104 + pytest.param(
105 + True,
106 + "[THINK]r[/THINK]c",
107 + "r",
108 + "c",
109 + id="v13_basic_streaming",
110 + ),
111 + # No trailing content after reasoning ends.
112 + pytest.param(
113 + False,
114 + "[THINK]r[/THINK]",
115 + "r",
116 + None,
117 + id="v13_no_trailing_content",
118 + ),
119 + pytest.param(
120 + True,
121 + "[THINK]r[/THINK]",
122 + "r",
123 + None,
124 + id="v13_no_trailing_content_streaming",
125 + ),
126 + # Multi-line reasoning and multi-line content.
127 + pytest.param(
128 + False,
129 + "[THINK]a\nb[/THINK]c\nd",
130 + "a\nb",
131 + "c\nd",
132 + id="v13_multiline",
133 + ),
134 + pytest.param(
135 + True,
136 + "[THINK]a\nb[/THINK]c\nd",
137 + "a\nb",
138 + "c\nd",
139 + id="v13_multiline_streaming",
140 + ),
141 + # No reasoning at all: output is pure content.
142 + pytest.param(
143 + False,
144 + "hello world",
145 + None,
146 + "hello world",
147 + id="v13_no_reasoning",
148 + ),
149 + pytest.param(
150 + True,
151 + "hello world",
152 + None,
153 + "hello world",
154 + id="v13_no_reasoning_streaming",
155 + ),
156 + # [THINK] opened but never closed; reasoning accumulates, content is None.
157 + pytest.param(
158 + False,
159 + "[THINK]r",
160 + "r",
161 + None,
162 + id="v13_think_no_end",
163 + ),
164 + pytest.param(
165 + True,
166 + "[THINK]r",
167 + "r",
168 + None,
169 + id="v13_think_no_end_streaming",
170 + ),
171 + # Stray [/THINK] with no matching [THINK]: no reasoning, content follows.
172 + pytest.param(
173 + False,
174 + "[/THINK]This is the rest",
175 + None,
176 + "This is the rest",
177 + id="v13_stray_think_end",
178 + ),
179 + pytest.param(
180 + True,
181 + "[/THINK]This is the rest",
182 + None,
183 + "This is the rest",
184 + id="v13_stray_think_end_streaming",
185 + ),
186 + # Content before [THINK]: leading segment joins with trailing segment.
187 + pytest.param(
188 + False,
189 + "Before\n[THINK]r[/THINK]\nAfter",
190 + "r",
191 + "Before\n\nAfter",
192 + id="v13_leading_content",
193 + ),
194 + pytest.param(
195 + True,
196 + "Before\n[THINK]r[/THINK]\nAfter",
197 + "r",
198 + "Before\n\nAfter",
199 + id="v13_leading_content_streaming",
200 + ),
201 + # Empty output: no reasoning, no content.
202 + pytest.param(
203 + False,
204 + "",
205 + None,
206 + None,
207 + id="v13_empty",
208 + ),
209 + pytest.param(
210 + True,
211 + "",
212 + None,
213 + None,
214 + id="v13_empty_streaming",
215 + ),
216 +]
217 +
218 +
219 +@pytest.mark.parametrize(
220 + "streaming, output, expected_reasoning, expected_content",
221 + _TEST_CASES,
222 +)
223 +def test_mistral_reasoning_v13(
224 + streaming: bool,
225 + output: str,
226 + expected_reasoning: str | None,
227 + expected_content: str | None,
228 + mistral_tokenizer: MistralTokenizer,
229 +) -> None:
230 + output_tokens = _encode_v13(mistral_tokenizer, output)
231 + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(_PARSER_NAME)(
232 + mistral_tokenizer
233 + )
234 + reasoning, content = run_reasoning_extraction_mistral(
235 + parser, output_tokens, streaming=streaming
236 + )
237 + assert reasoning == expected_reasoning
238 + assert content == expected_content
239 +
240 +
241 +def test_system_prompt_think_ignored(
242 + mistral_tokenizer: MistralTokenizer,
243 +) -> None:
244 + """Streaming content extraction must not
245 + include `[THINK]`/`[/THINK]` examples from the system prompt.
246 + """
247 + output = "[THINK]real[/THINK]answer"
248 + output_tokens = _encode_v13(mistral_tokenizer, output)
249 + for streaming in (False, True):
250 + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(
251 + _PARSER_NAME
252 + )(mistral_tokenizer)
253 + reasoning, content = run_reasoning_extraction_mistral(
254 + parser, output_tokens, streaming=streaming
255 + )
256 + assert reasoning == "real"
257 + assert content == "answer"
258 +
259 +
260 +def test_reasoning_with_tool_calls(
261 + mistral_tokenizer: MistralTokenizer,
262 +) -> None:
263 + """Reasoning extraction with a tool call in the generated output."""
264 + tool_calls_token_id = mistral_tokenizer.get_vocab().get("[TOOL_CALLS]")
265 + if tool_calls_token_id is None:
266 + pytest.skip("Tokenizer does not have [TOOL_CALLS] in vocab")
267 +
268 + # Token sequence representing: [THINK]r[/THINK][TOOL_CALLS]f{"a":1}
269 + output_tokens = (
270 + [mistral_tokenizer.instruct.BEGIN_THINK]
271 + + mistral_tokenizer.tokenizer.encode("r", False, False)
272 + + [mistral_tokenizer.instruct.END_THINK]
273 + + [tool_calls_token_id]
274 + + mistral_tokenizer.tokenizer.encode('f{"a":1}', False, False)
275 + )
276 + for streaming in (False, True):
277 + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(
278 + _PARSER_NAME
279 + )(mistral_tokenizer)
280 + reasoning, _content = run_reasoning_extraction_mistral(
281 + parser, output_tokens, streaming=streaming
282 + )
283 + assert reasoning == "r"
284 + assert "[TOOL_CALLS]" not in (reasoning or "")
285 +
286 +
287 +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstream", "stream"])
288 +def test_reasoning_v11_plain_text_think(
289 + mistral_v11_tokenizer: MistralTokenizer, streaming: bool
290 +) -> None:
291 + """v11 tokenizers use plain-text `<think>`/`</think>` (no special tokens)."""
292 + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(_PARSER_NAME)(
293 + mistral_v11_tokenizer
294 + )
295 +
296 + tokens = mistral_v11_tokenizer.tokenizer.encode("<think>r</think>c", False, False)
297 + reasoning, content = run_reasoning_extraction_mistral(
298 + parser, tokens, streaming=streaming
299 + )
300 + assert reasoning == "r"
301 + assert content == "c"
302 +
303 +
304 +def test_is_reasoning_end(
305 + mistral_tokenizer: MistralTokenizer,
306 +) -> None:
307 + """is_reasoning_end returns True iff reasoning has finished."""
308 + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(_PARSER_NAME)(
309 + mistral_tokenizer
310 + )
311 +
312 + # Complete reasoning: END_THINK present → True.
313 + complete_ids = _encode_v13(mistral_tokenizer, "[THINK]r[/THINK]c")
314 + assert parser.is_reasoning_end(complete_ids)
315 +
316 + # Open-only: no END_THINK, BEGIN_THINK present → False.
317 + open_ids = _encode_v13(mistral_tokenizer, "[THINK]r")
318 + assert not parser.is_reasoning_end(open_ids)
319 +
320 + # No reasoning markers at all → False.
321 + no_reasoning_ids = _encode_v13(mistral_tokenizer, "hello")
322 + assert not parser.is_reasoning_end(no_reasoning_ids)
323 +
324 + # Explicit END_THINK closes reasoning → True.
325 + explicit_end_ids = (
326 + [mistral_tokenizer.instruct.BEGIN_THINK]
327 + + mistral_tokenizer.tokenizer.encode("r", False, False)
328 + + [mistral_tokenizer.instruct.END_THINK]
329 + )
330 + assert parser.is_reasoning_end(explicit_end_ids)
331 +
332 + # [TOOL_CALLS] acts as implicit reasoning-end marker.
333 + tool_calls_token_id = mistral_tokenizer.get_vocab().get("[TOOL_CALLS]")
334 + if tool_calls_token_id is None:
335 + pytest.skip("Tokenizer does not have [TOOL_CALLS] in vocab")
336 +
337 + implicit_end_ids = (
338 + [mistral_tokenizer.instruct.BEGIN_THINK]
339 + + mistral_tokenizer.tokenizer.encode("r", False, False)
340 + + [tool_calls_token_id]
341 + )
342 + assert parser.is_reasoning_end(implicit_end_ids)
343 +
344 +
345 +def _stream_parse_delta(parser, tokenizer, gen_ids, request, prompt_ids):
346 + """Stream `gen_ids` through `parse_delta` and reconstruct reasoning/content."""
347 + reasoning = ""
348 + content = ""
349 + previous_tokens: list[str] | None = None
350 + prefix_offset = 0
351 + read_offset = 0
352 + for i, tok in enumerate(gen_ids):
353 + new_tokens, delta_text, prefix_offset, read_offset = detokenize_incrementally(
354 + tokenizer=tokenizer,
355 + all_input_ids=gen_ids[: i + 1],
356 + prev_tokens=previous_tokens,
357 + prefix_offset=prefix_offset,
358 + read_offset=read_offset,
359 + skip_special_tokens=True,
360 + spaces_between_special_tokens=True,
361 + )
362 + previous_tokens = (
363 + previous_tokens + new_tokens if previous_tokens else new_tokens
364 + )
365 + delta_message = parser.parse_delta(
366 + delta_text=delta_text,
367 + delta_token_ids=[tok],
368 + request=request,
369 + prompt_token_ids=prompt_ids,
370 + finished=(i == len(gen_ids) - 1),
371 + )
372 + if delta_message is not None:
373 + reasoning += delta_message.reasoning or ""
374 + content += delta_message.content or ""
375 + return reasoning, content
376 +
377 +
378 +def test_parse_delta_keeps_reasoning_open_after_closed_think_prompt(
379 + mistral_tokenizer: MistralTokenizer,
380 +) -> None:
381 + """A closed `[THINK]` block in the prompt must not finalize reasoning.
382 +
383 + Regression: reasoning system prompts end the prompt with a closed
384 + `[THINK]...[/THINK]` block. With a parser-injected grammar the generation's
385 + reasoning must still be captured as ``reasoning_content`` rather than
386 + leaking into ``content`` (the prompt-based reasoning-end heuristic must be
387 + skipped). This exercises ``DelegatingParser.parse_delta``'s prompt check,
388 + which the isolated reasoning-parser tests do not cover.
389 + """
390 + parser_cls = ParserManager.get_parser(
391 + tool_parser_name="mistral",
392 + reasoning_parser_name="mistral",
393 + enable_auto_tools=True,
394 + )
395 + parser = parser_cls(mistral_tokenizer, None)
396 + request = parser.adjust_request(
397 + ChatCompletionRequest(
398 + messages=[],
399 + model="test",
400 + tools=_SAMPLE_TOOLS,
401 + tool_choice="auto",
402 + )
403 + )
404 + assert getattr(request, "_grammar_from_parser", False)
405 +
406 + prompt_ids = _encode_v13(mistral_tokenizer, "[THINK]system reasoning[/THINK]")
407 + gen_ids = _encode_v13(
408 + mistral_tokenizer,
409 + "[THINK]because two plus two is four[/THINK]The answer is 4.",
410 + )
411 +
412 + reasoning, content = _stream_parse_delta(
413 + parser, mistral_tokenizer, gen_ids, request, prompt_ids
414 + )
415 +
416 + assert "because two plus two is four" in reasoning
417 + assert "because two plus two is four" not in content
418 + assert "The answer is 4." in content
419 +
420 +
421 +def _encode_gen(tokenizer: MistralTokenizer, text: str) -> list[int]:
422 + """Encode `text` mapping `[THINK]`, `[/THINK]`, `[TOOL_CALLS]`, `[ARGS]`
423 + to their special-token IDs; all other text uses the base tokenizer.
424 + """
425 + vocab = tokenizer.get_vocab()
426 + markers: dict[str, int] = {
427 + "[THINK]": tokenizer.instruct.BEGIN_THINK,
428 + "[/THINK]": tokenizer.instruct.END_THINK,
429 + "[TOOL_CALLS]": vocab["[TOOL_CALLS]"],
430 + "[ARGS]": vocab["[ARGS]"],
431 + }
432 + out: list[int] = []
433 + remaining = text
434 + while remaining:
435 + earliest: int | None = None
436 + earliest_marker: str | None = None
437 + for marker in markers:
438 + idx = remaining.find(marker)
439 + if idx != -1 and (earliest is None or idx < earliest):
440 + earliest = idx
441 + earliest_marker = marker
442 + if earliest_marker is None:
443 + out += tokenizer.tokenizer.encode(remaining, False, False)
444 + break
445 + assert earliest is not None # set together with earliest_marker
446 + if earliest > 0:
447 + out += tokenizer.tokenizer.encode(remaining[:earliest], False, False)
448 + out.append(markers[earliest_marker])
449 + remaining = remaining[earliest + len(earliest_marker) :]
450 + return out
451 +
452 +
453 +def _stream_turn(
454 + parser,
455 + tokenizer: MistralTokenizer,
456 + gen_ids: list[int],
457 + request,
458 + prompt_ids: list[int],
459 +) -> tuple[str, str, list[tuple[str, str]]]:
460 + """Stream `gen_ids` through `parse_delta`, returning `(reasoning, content,
461 + tool_calls)`.
462 +
463 + `tool_calls` is a list of `(name, arguments)` tuples ordered by
464 + `DeltaToolCall.index`. Arguments are concatenated across deltas.
465 + """
466 + reasoning = ""
467 + content = ""
468 + tool_call_acc: dict[int, dict[str, str]] = {}
469 + previous_tokens: list[str] | None = None
470 + prefix_offset = 0
471 + read_offset = 0
472 + for i, tok in enumerate(gen_ids):
473 + new_tokens, delta_text, prefix_offset, read_offset = detokenize_incrementally(
474 + tokenizer=tokenizer,
475 + all_input_ids=gen_ids[: i + 1],
476 + prev_tokens=previous_tokens,
477 + prefix_offset=prefix_offset,
478 + read_offset=read_offset,
479 + skip_special_tokens=True,
480 + spaces_between_special_tokens=True,
481 + )
482 + previous_tokens = (
483 + previous_tokens + new_tokens if previous_tokens else new_tokens
484 + )
485 + delta_message = parser.parse_delta(
486 + delta_text=delta_text,
487 + delta_token_ids=[tok],
488 + request=request,
489 + prompt_token_ids=prompt_ids,
490 + finished=(i == len(gen_ids) - 1),
491 + )
492 + if delta_message is not None:
493 + reasoning += delta_message.reasoning or ""
494 + content += delta_message.content or ""
495 + for tc in delta_message.tool_calls:
496 + if tc.index not in tool_call_acc:
497 + tool_call_acc[tc.index] = {"name": "", "args": ""}
498 + if tc.function:
499 + if tc.function.name:
500 + tool_call_acc[tc.index]["name"] = tc.function.name
501 + if tc.function.arguments:
502 + tool_call_acc[tc.index]["args"] += tc.function.arguments
503 + tool_calls = [
504 + (tool_call_acc[k]["name"], tool_call_acc[k]["args"])
505 + for k in sorted(tool_call_acc)
506 + ]
507 + return reasoning, content, tool_calls
508 +
509 +
510 +def test_parse_delta_multi_turn_reason_and_tool(
511 + mistral_tokenizer: MistralTokenizer,
512 +) -> None:
513 + """Two-turn loop: each assistant turn carries reasoning + a tool call.
514 +
515 + For each turn, the mistral-common conversation is encoded into `prompt_ids`
516 + before streaming the next generation. Turn 2's prompt therefore contains
517 + a `[THINK]`/`[/THINK]` block and `[TOOL_CALLS]` from turn 1 — the
518 + accumulated tool-call/result history that single-turn tests never exercise.
519 +
520 + Asserts that across both turns the parser correctly classifies think tokens
521 + as `reasoning_content` (never leaking into `content`), and that the tool
522 + call name/arguments are reconstructed correctly from the streaming deltas.
523 + """
524 + parser_cls = ParserManager.get_parser(
525 + tool_parser_name="mistral",
526 + reasoning_parser_name="mistral",
527 + enable_auto_tools=True,
528 + )
529 + request = parser_cls(mistral_tokenizer, None).adjust_request(
530 + ChatCompletionRequest(
531 + messages=[],
532 + model="test",
533 + tools=_SAMPLE_TOOLS,
534 + tool_choice="auto",
535 + )
536 + )
537 + assert getattr(request, "_grammar_from_parser", False)
538 +
539 + cities = ["Dallas", "Paris"]
540 + conversation: list = [UserMessage(content="What is the weather in Dallas?")]
541 +
542 + for i, city in enumerate(cities):
543 + # Fresh parser per turn: streaming state must not bleed across turns.
544 + parser = parser_cls(mistral_tokenizer, None)
545 +
546 + prompt_ids = mistral_tokenizer.instruct.encode_instruct(
547 + InstructRequest(messages=conversation)
548 + ).tokens
549 +
550 + gen_str = (
551 + f"[THINK]I should check {city} weather.[/THINK]"
552 + f'[TOOL_CALLS]get_weather[ARGS]{{"city": "{city}"}}'
553 + )
554 + gen_ids = _encode_gen(mistral_tokenizer, gen_str)
555 +
556 + reasoning, content, tool_calls = _stream_turn(
557 + parser, mistral_tokenizer, gen_ids, request, prompt_ids
558 + )
559 +
560 + assert f"check {city} weather" in reasoning, f"turn {i}: reasoning not captured"
561 + assert f"check {city} weather" not in content, (
562 + f"turn {i}: reasoning leaked into content"
563 + )
564 + assert "[THINK]" not in content, f"turn {i}: [THINK] leaked"
565 + assert "[TOOL_CALLS]" not in content, f"turn {i}: [TOOL_CALLS] leaked"
566 + assert len(tool_calls) == 1, f"turn {i}: expected 1 tool call"
567 + name, args = tool_calls[0]
568 + assert name == "get_weather", f"turn {i}: wrong tool name {name!r}"
569 + assert city in args, f"turn {i}: city not in args {args!r}"
570 +
571 + # Extend conversation with this turn's history for the next turn.
572 + tc_id = f"tc{i:07d}" # 9-char alphanumeric id: "tc0000000", "tc0000001"
573 + conversation.extend(
574 + [
575 + AssistantMessage(
576 + content=[
577 + ThinkChunk(
578 + thinking=f"I should check {city} weather.",
579 + closed=True,
580 + )
581 + ],
582 + tool_calls=[
583 + ToolCall(
584 + id=tc_id,
585 + function=FunctionCall(
586 + name="get_weather",
587 + arguments=json.dumps({"city": city}),
588 + ),
589 + )
590 + ],
591 + ),
592 + ToolMessage(
593 + content=f"{city}: 70F sunny",
594 + tool_call_id=tc_id,
595 + ),
596 + ]
597 + )
598 + if i < len(cities) - 1:
599 + conversation.append(UserMessage(content=f"And in {cities[i + 1]}?"))
tests/parser/mistral/test_tool_calls.py renamed
+1150 -116
@@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch
7
8 import partial_json_parser
9 import pytest
10 +import regex as re
11 from mistral_common.protocol.instruct.messages import AssistantMessage
12 from mistral_common.protocol.instruct.request import InstructRequest
13 from mistral_common.protocol.instruct.tool_calls import (
@@ -28,21 +29,26 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
29 ChatCompletionRequest,
30 )
31 from vllm.entrypoints.openai.engine.protocol import (
32 + DeltaFunctionCall,
33 DeltaMessage,
34 DeltaToolCall,
35 ExtractedToolCallInformation,
36 StructuralTagResponseFormat,
37 )
38 +from vllm.parser.engine.events import EventType
39 +from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
40 +from vllm.parser.mistral import _DEFAULT_JSON_SCHEMA, MistralParser, mistral_config
41 +from vllm.parser.parser_manager import ParserManager
42 from vllm.sampling_params import StructuredOutputsParams
43 from vllm.tokenizers import TokenizerLike, get_tokenizer
44 from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
45 from vllm.tokenizers.mistral import MistralTokenizer
40 -from vllm.tool_parsers.mistral_tool_parser import (
41 - _DEFAULT_JSON_SCHEMA,
42 - MistralToolParser,
43 -)
46 +from vllm.tool_parsers.mistral_tool_parser import MistralToolParser
47
48 _DUMMY_REQUEST = ChatCompletionRequest(messages=[], model="test")
49 +# Minimal request for the engine driver: tool_choice=None falls through to
50 +# auto extraction without triggering "tools must be set" validation.
51 +_ENGINE_REQUEST = ChatCompletionRequest(messages=[], model="test", tool_choice=None)
52
53
54 @pytest.fixture(scope="module")
@@ -71,6 +77,8 @@ def mistral_tool_parser(mistral_tokenizer):
77 def non_mistral_parser() -> MistralToolParser:
78 mock_tokenizer = MagicMock()
79 mock_tokenizer.get_vocab.return_value = {"[TOOL_CALLS]": 1}
80 + # Ensure the legacy (non-grammar) path is taken by MistralParser.
81 + mock_tokenizer.supports_grammar = False
82 return MistralToolParser(mock_tokenizer)
83
84
@@ -96,9 +104,18 @@ def assert_tool_calls(
104 assert actual_tool_call.function.name == expected_tool_call.function.name, (
105 f"got wrong function name:${actual_tool_call.function.name}"
106 )
99 - assert (
100 - actual_tool_call.function.arguments == expected_tool_call.function.arguments
101 - ), f"got wrong function argument:${actual_tool_call.function.arguments}"
107 + actual_args = actual_tool_call.function.arguments
108 + # Streaming deltas (DeltaToolCall) may have partial JSON because the
109 + # engine holds back the final closing brace until finish() is called.
110 + # Apply partial_json_parser so the comparison works for both the
111 + # legacy (complete JSON) and engine (potentially partial) paths.
112 + if isinstance(actual_tool_call, DeltaToolCall) and actual_args is not None:
113 + actual_args = partial_json_parser.ensure_json(
114 + actual_args, Allow.OBJ | Allow.STR
115 + )
116 + assert actual_args == expected_tool_call.function.arguments, (
117 + f"got wrong function argument:${actual_tool_call.function.arguments}"
118 + )
119
120
121 def fix_tool_call_tokenization(
@@ -106,45 +123,82 @@ def fix_tool_call_tokenization(
123 mistral_tool_parser: MistralToolParser,
124 mistral_tokenizer: TokenizerLike,
125 ):
109 - """
110 - Replaces the textual token sequence for [TOOL_CALLS]
111 - with its single special token ID.
112 - """
113 - textual_tool_call_token_ids = mistral_tokenizer.encode(
114 - text=mistral_tool_parser.bot_token,
115 - add_special_tokens=False,
116 - )
117 - # textual_tool_call_token_ids must not contain special tokens like bos, eos etc
118 - special_tool_call_token_ids = [mistral_tool_parser.bot_token_id]
126 + """Replace textual token sequences for the control markers ([TOOL_CALLS]
127 + and [ARGS]) with their single special token IDs.
128
120 - # If the input is too short to contain the sequence, no replacement is possible
121 - if not tokens or len(tokens) < len(textual_tool_call_token_ids):
122 - return tokens
129 + Real generation emits these as control tokens; encoding the marker *string*
130 + can instead yield literal text tokens. The engine detects markers by id, so
131 + the test stream must carry the special ids.
132 + """
133 + vocab = mistral_tokenizer.get_vocab()
134 + markers: list[tuple[str, int]] = [
135 + (
136 + mistral_tool_parser._parser_engine.bot_token,
137 + mistral_tool_parser._parser_engine.bot_token_id,
138 + )
139 + ]
140 + args_id = vocab.get("[ARGS]")
141 + if args_id is not None:
142 + markers.append(("[ARGS]", args_id))
143 +
144 + # (textual token sequence, special id) pairs to replace.
145 + replacements: list[tuple[list[int], int]] = []
146 + for text, special_id in markers:
147 + if special_id is None:
148 + continue
149 + textual = mistral_tokenizer.encode(text=text, add_special_tokens=False)
150 + if textual:
151 + replacements.append((textual, special_id))
152
124 - result_tokens = []
153 + result_tokens: list[int] = []
154 i = 0
126 - target_len = len(textual_tool_call_token_ids)
127 -
155 while i < len(tokens):
129 - # Check if the slice from the current position matches the target sequence
130 - if tokens[i : i + target_len] == textual_tool_call_token_ids:
131 - # If it matches, add the replacement and jump the index forward
132 - result_tokens.extend(special_tool_call_token_ids)
133 - i += target_len
156 + for textual, special_id in replacements:
157 + if tokens[i : i + len(textual)] == textual:
158 + result_tokens.append(special_id)
159 + i += len(textual)
160 + break
161 else:
135 - # Otherwise, just add the current token and move to the next one
162 result_tokens.append(tokens[i])
163 i += 1
164
165 return result_tokens
166
167
168 +def encode_mistral_output(tokenizer: MistralTokenizer, text: str) -> list[int]:
169 + """Encode generated output with control markers as their special-token ids.
170 +
171 + Real generation emits ``[TOOL_CALLS]``/``[ARGS]`` as single control tokens.
172 + Encoding the marker *strings* can instead glue them to neighbours (e.g.
173 + ``}[``) and yield literal text tokens, which the id-based engine never sees.
174 + Split on the markers and insert their special ids so the token stream
175 + matches real generation.
176 + """
177 + vocab = tokenizer.get_vocab()
178 + marker_ids = {
179 + "[TOOL_CALLS]": vocab.get("[TOOL_CALLS]"),
180 + "[ARGS]": vocab.get("[ARGS]"),
181 + }
182 + ids: list[int] = []
183 + for part in re.split(r"(\[TOOL_CALLS\]|\[ARGS\])", text):
184 + if part in marker_ids:
185 + # Fall back to text encoding here and the id-based engine path
186 + # silently stops being exercised while the tests still pass.
187 + assert marker_ids[part] is not None, f"{part} missing from vocab"
188 + ids.append(marker_ids[part])
189 + elif part:
190 + ids.extend(tokenizer.encode(part, add_special_tokens=False))
191 + return ids
192 +
193 +
194 def stream_delta_message_generator(
195 mistral_tool_parser: MistralToolParser,
196 mistral_tokenizer: TokenizerLike,
197 model_output: str | None,
198 tools: list[tuple[str, str]] | None,
199 chunk_size: int = 1,
200 + driver: str = "legacy",
201 + reasoning_parser: str | None = None,
202 ) -> Generator[DeltaMessage, None, None]:
203 if (
204 isinstance(mistral_tokenizer, MistralTokenizer)
@@ -179,6 +233,33 @@ def stream_delta_message_generator(
233 all_token_ids, mistral_tool_parser, mistral_tokenizer
234 )
235
236 + if isinstance(mistral_tokenizer, MistralTokenizer):
237 + # Real serving streams only generated tokens; encode_instruct adds
238 + # framing (BOS/EOS) the parser never receives. Strip them so the
239 + # id-based engine detection sees a realistic stream.
240 + if all_token_ids and all_token_ids[0] == mistral_tokenizer.bos_token_id:
241 + all_token_ids = all_token_ids[1:]
242 + if all_token_ids and all_token_ids[-1] == mistral_tokenizer.eos_token_id:
243 + all_token_ids = all_token_ids[:-1]
244 +
245 + if driver == "engine":
246 + _parser_cls = ParserManager.get_parser(
247 + tool_parser_name="mistral",
248 + reasoning_parser_name=reasoning_parser,
249 + enable_auto_tools=True,
250 + )
251 + _engine_parser = _parser_cls(mistral_tokenizer, None)
252 + if reasoning_parser is not None:
253 + _req_with_tools = ChatCompletionRequest(
254 + messages=[],
255 + model="test",
256 + tools=SAMPLE_TOOLS_DICTS,
257 + tool_choice="auto",
258 + )
259 + _engine_req = _engine_parser.adjust_request(_req_with_tools)
260 + else:
261 + _engine_req = _ENGINE_REQUEST
262 +
263 previous_text = ""
264 previous_tokens = None
265 prefix_offset = 0
@@ -215,15 +296,24 @@ def stream_delta_message_generator(
296 current_token_ids = all_token_ids[: i + 1]
297 current_text = previous_text + pending_text
298
218 - delta_message = mistral_tool_parser.extract_tool_calls_streaming(
219 - previous_text,
220 - current_text,
221 - pending_text,
222 - previous_token_ids,
223 - current_token_ids,
224 - pending_token_ids,
225 - request=_DUMMY_REQUEST,
226 - )
299 + if driver == "legacy":
300 + delta_message = mistral_tool_parser.extract_tool_calls_streaming(
301 + previous_text,
302 + current_text,
303 + pending_text,
304 + previous_token_ids,
305 + current_token_ids,
306 + pending_token_ids,
307 + request=_DUMMY_REQUEST,
308 + )
309 + else: # engine: per-delta windows, mirroring parse_delta behaviour
310 + delta_message = _engine_parser.parse_delta(
311 + delta_text=pending_text,
312 + delta_token_ids=list(pending_token_ids),
313 + request=_engine_req,
314 + prompt_token_ids=[1],
315 + finished=(i == len(all_token_ids) - 1),
316 + )
317 if delta_message:
318 yield delta_message
319
@@ -231,6 +321,14 @@ def stream_delta_message_generator(
321 pending_text = ""
322 pending_token_ids = []
323
324 + # For the legacy driver the engine holds back the final closing brace
325 + # in _args_buffer until finish_streaming() is called, mirroring how
326 + # the real serving layer flushes at end of stream.
327 + if driver == "legacy":
328 + flush_delta = mistral_tool_parser.finish_streaming()
329 + if flush_delta:
330 + yield flush_delta
331 +
332
333 @pytest.mark.parametrize(
334 "parser_fixture",
@@ -424,8 +522,14 @@ def test_extract_tool_calls_pre_v11_regex_fallback_fails(
522 "single_tool_add",
523 "single_tool_weather",
524 "multiple_tool_calls",
525 + "single_tool_add_args",
526 + "single_tool_weather_args",
527 + "multiple_tool_calls_args",
528 + "content_before_tool_args",
529 "complex",
530 "wrong_json",
531 + "trailing_text_after_json",
532 + "trailing_text_after_args_json",
533 ],
534 argnames=["model_output", "expected_tool_calls", "expected_content"],
535 argvalues=[
@@ -471,6 +575,60 @@ def test_extract_tool_calls_pre_v11_regex_fallback_fails(
575 ],
576 None,
577 ),
578 + (
579 + # v11+ emits an explicit [ARGS] separator between name and args.
580 + """[TOOL_CALLS]add_this_and_that[ARGS]{"a": 3.5, "b": 4}""", # noqa: E501
581 + [
582 + ToolCall(
583 + function=FunctionCall(
584 + name="add_this_and_that",
585 + arguments=json.dumps({"a": 3.5, "b": 4}),
586 + )
587 + )
588 + ],
589 + None,
590 + ),
591 + (
592 + """[TOOL_CALLS]get_current_weather[ARGS]{"city": "San Francisco", "state": "CA", "unit": "celsius"}""", # noqa: E501
593 + [
594 + ToolCall(
595 + function=FunctionCall(
596 + name="get_current_weather",
597 + arguments=json.dumps(
598 + {"city": "San Francisco", "state": "CA", "unit": "celsius"}
599 + ),
600 + )
601 + )
602 + ],
603 + None,
604 + ),
605 + (
606 + """[TOOL_CALLS]add[ARGS]{"a": 3.5, "b": 4}[TOOL_CALLS]multiply[ARGS]{"a": 3, "b": 6}""", # noqa: E501
607 + [
608 + ToolCall(
609 + function=FunctionCall(
610 + name="add", arguments=json.dumps({"a": 3.5, "b": 4})
611 + )
612 + ),
613 + ToolCall(
614 + function=FunctionCall(
615 + name="multiply", arguments=json.dumps({"a": 3, "b": 6})
616 + )
617 + ),
618 + ],
619 + None,
620 + ),
621 + (
622 + """hi[TOOL_CALLS]add[ARGS]{"a": 1, "b": 2}""", # noqa: E501
623 + [
624 + ToolCall(
625 + function=FunctionCall(
626 + name="add", arguments=json.dumps({"a": 1, "b": 2})
627 + )
628 + )
629 + ],
630 + "hi",
631 + ),
632 (
633 # Complex
634 """hi{hi[TOOL_CALLS]bash{"command": "print(\\"hello world!\\")\\nre.compile(r\'{}\')""", # noqa: E501
@@ -501,6 +659,33 @@ def test_extract_tool_calls_pre_v11_regex_fallback_fails(
659 ],
660 "hi{hi",
661 ),
662 + (
663 + # gh#48975: trailing text after the JSON object must not leak into
664 + # arguments (v11+ name{args} format has no terminator).
665 + """[TOOL_CALLS]get_current_weather{"city": "San Francisco"}Estimating the weather...""", # noqa: E501
666 + [
667 + ToolCall(
668 + function=FunctionCall(
669 + name="get_current_weather",
670 + arguments=json.dumps({"city": "San Francisco"}),
671 + )
672 + )
673 + ],
674 + None,
675 + ),
676 + (
677 + # gh#48975 with the explicit [ARGS] separator.
678 + """[TOOL_CALLS]get_current_weather[ARGS]{"city": "San Francisco"}Estimating the weather...""", # noqa: E501
679 + [
680 + ToolCall(
681 + function=FunctionCall(
682 + name="get_current_weather",
683 + arguments=json.dumps({"city": "San Francisco"}),
684 + )
685 + )
686 + ],
687 + None,
688 + ),
689 ],
690 )
691 def test_extract_tool_calls(
@@ -517,17 +702,128 @@ def test_extract_tool_calls(
702
703
704 def test_extract_tool_calls_v11_without_args_skipped(mistral_tool_parser):
705 + # Before Stage 2: the legacy path skipped tool calls with no arg separator,
706 + # returning tools_called=True with an empty list. After Stage 2 the engine
707 + # path is used for v11; it recognises the name and emits a tool call with
708 + # empty arguments ("{}"). The semantic result is equivalent — a valid zero-
709 + # argument call — and is strictly more correct than silently dropping it.
710 model_output = "[TOOL_CALLS]toolname_no_args"
711 result = mistral_tool_parser.extract_tool_calls(
712 model_output, request=_DUMMY_REQUEST
713 )
524 - assert result == ExtractedToolCallInformation(
525 - tools_called=True, tool_calls=[], content=None
714 + assert result.tools_called
715 + assert len(result.tool_calls) == 1
716 + assert result.tool_calls[0].function.name == "toolname_no_args"
717 + assert result.tool_calls[0].function.arguments == "{}"
718 + assert result.content is None
719 +
720 +
721 +def test_extract_tool_calls_malformed_name_before_marker_emits_empty_name(
722 + mistral_tool_parser,
723 +):
724 + model_output = 'bash[TOOL_CALLS]{"command": "cat file.py"}'
725 + result = mistral_tool_parser.extract_tool_calls(
726 + model_output, request=_DUMMY_REQUEST
727 + )
728 + assert result.tools_called
729 + assert len(result.tool_calls) == 1
730 + assert result.tool_calls[0].function.name == ""
731 + assert result.tool_calls[0].function.arguments == json.dumps(
732 + {"command": "cat file.py"}
733 + )
734 + assert result.content == "bash"
735 +
736 +
737 +def test_extract_tool_calls_well_formed_name_unaffected(mistral_tool_parser):
738 + model_output = '[TOOL_CALLS]bash[ARGS]{"command": "ls -la"}'
739 + result = mistral_tool_parser.extract_tool_calls(
740 + model_output, request=_DUMMY_REQUEST
741 + )
742 + assert result.tools_called
743 + assert len(result.tool_calls) == 1
744 + assert result.tool_calls[0].function.name == "bash"
745 + assert result.tool_calls[0].function.arguments == json.dumps({"command": "ls -la"})
746 + assert result.content is None
747 +
748 +
749 +def test_extract_tool_calls_empty_name_unparsable_args_no_crash(
750 + mistral_tool_parser,
751 +):
752 + # Empty name AND args that never form valid JSON: still emitted (raw
753 + # text as arguments) instead of raising.
754 + model_output = "bash[TOOL_CALLS]{not valid json at all"
755 + result = mistral_tool_parser.extract_tool_calls(
756 + model_output, request=_DUMMY_REQUEST
757 + )
758 + assert result.tools_called
759 + assert len(result.tool_calls) == 1
760 + assert result.tool_calls[0].function.name == ""
761 + assert result.tool_calls[0].function.arguments == "{not valid json at all"
762 + assert result.content == "bash"
763 +
764 +
765 +def test_extract_tool_calls_streaming_malformed_name_before_marker(
766 + mistral_tool_parser,
767 +):
768 + # Streaming counterpart: once the args start the slot can never go back to
769 + # naming, so the (empty) name is already final and streams immediately
770 + # alongside the args.
771 + model_output = 'bash[TOOL_CALLS]{"command": "cat file.py"}'
772 + mid_delta = mistral_tool_parser.extract_tool_calls_streaming(
773 + previous_text="",
774 + current_text=model_output,
775 + delta_text=model_output,
776 + previous_token_ids=[],
777 + current_token_ids=[],
778 + delta_token_ids=[],
779 + request=_DUMMY_REQUEST,
780 + )
781 + assert mid_delta is not None
782 + assert mid_delta.content == "bash"
783 + assert len(mid_delta.tool_calls) == 1
784 + assert mid_delta.tool_calls[0].function.name == ""
785 + assert mid_delta.tool_calls[0].function.arguments == '{"command": "cat file.py"'
786 +
787 + final_delta = mistral_tool_parser.finish_streaming()
788 + assert final_delta is not None
789 + assert len(final_delta.tool_calls) == 1
790 + assert final_delta.tool_calls[0].function.name is None
791 + assert final_delta.tool_calls[0].function.arguments == "}"
792 +
793 +
794 +def test_extract_tool_calls_parallel_malformed_then_well_formed(
795 + mistral_tool_parser,
796 +):
797 + # A malformed empty-name call followed by a well-formed one must not
798 + # regress into brace-splitting: nested braces in the second call's
799 + # arguments must stay intact, and both calls must be correctly separated.
800 + model_output = (
801 + 'bash[TOOL_CALLS]{"command": "ls"}'
802 + '[TOOL_CALLS]submit[ARGS]{"answer": {"nested": 1}}'
803 + )
804 + result = mistral_tool_parser.extract_tool_calls(
805 + model_output, request=_DUMMY_REQUEST
806 + )
807 + assert result.tools_called
808 + assert len(result.tool_calls) == 2
809 + assert result.tool_calls[0].function.name == ""
810 + assert result.tool_calls[0].function.arguments == json.dumps({"command": "ls"})
811 + assert result.tool_calls[1].function.name == "submit"
812 + assert result.tool_calls[1].function.arguments == json.dumps(
813 + {"answer": {"nested": 1}}
814 )
815 + assert result.content == "bash"
816
817
818 def _test_extract_tool_calls_streaming(
530 - tool_parser, tokenizer, model_output, tools, expected_tool_calls, expected_content
819 + tool_parser,
820 + tokenizer,
821 + model_output,
822 + tools,
823 + expected_tool_calls,
824 + expected_content,
825 + driver: str = "legacy",
826 + reasoning_parser: str | None = None,
827 ):
828 other_content: str = ""
829 function_names: list[str] = []
@@ -536,7 +832,12 @@ def _test_extract_tool_calls_streaming(
832 tool_call_ids: list[str | None] = []
833
834 for delta_message in stream_delta_message_generator(
539 - tool_parser, tokenizer, model_output, tools
835 + tool_parser,
836 + tokenizer,
837 + model_output,
838 + tools,
839 + driver=driver,
840 + reasoning_parser=reasoning_parser,
841 ):
842 # role should never be streamed from tool parser
843 assert not delta_message.role
@@ -547,12 +848,36 @@ def _test_extract_tool_calls_streaming(
848 streamed_tool_calls = delta_message.tool_calls
849
850 if streamed_tool_calls and len(streamed_tool_calls) > 0:
851 + # On the final finished=True delta the engine's _flush_engine_parsers
852 + # may append the flushed '}' as a separate DeltaToolCall for the
853 + # same index without coalescing. Merge entries sharing an index so
854 + # the "one diff per delta" invariant can be checked after merging.
855 + if len(streamed_tool_calls) > 1:
856 + assert len({tc.index for tc in streamed_tool_calls}) == 1, (
857 + "only within-chunk finish/flush of one tool is allowed; "
858 + "distinct indices in one delta indicate a regression: "
859 + f"{[tc.index for tc in streamed_tool_calls]}"
860 + )
861 + by_index: dict[int, DeltaToolCall] = {}
862 + for tc in streamed_tool_calls:
863 + existing = by_index.get(tc.index)
864 + if existing is None:
865 + by_index[tc.index] = tc
866 + else:
867 + if tc.id and not existing.id:
868 + existing.id = tc.id
869 + if tc.type and not existing.type:
870 + existing.type = tc.type
871 + if tc.function and existing.function and tc.function.arguments:
872 + existing.function.arguments = (
873 + existing.function.arguments or ""
874 + ) + tc.function.arguments
875 + streamed_tool_calls = list(by_index.values())
876 +
877 # make sure only one diff is present - correct even for parallel
878 assert len(streamed_tool_calls) == 1
879 tool_call = streamed_tool_calls[0]
880
554 - assert len(tool_parser.prev_tool_call_arr) > 0
555 -
881 # if a new tool is being called, set up empty arguments
882 if tool_call.index != tool_call_idx:
883 tool_call_idx = tool_call.index
@@ -595,34 +920,8 @@ def _test_extract_tool_calls_streaming(
920 ]
921 assert_tool_calls(actual_tool_calls, expected_tool_calls)
922
598 - if expected_tool_calls:
599 - assert len(tool_parser.streamed_args_for_tool) == len(expected_tool_calls)
600 - assert len(tool_parser.prev_tool_call_arr) == len(expected_tool_calls)
601 - for i in range(len(expected_tool_calls)):
602 - assert (
603 - tool_parser.prev_tool_call_arr[i]["arguments"]
604 - == tool_parser.streamed_args_for_tool[i]
605 - )
606 - assert tool_parser.streamed_args_for_tool[i] == function_args_strs[i]
607 - assert (
608 - tool_parser.prev_tool_call_arr[i]["name"]
609 - == expected_tool_calls[i].function.name
610 - )
611 -
612 - # Simulate the serving layer's unstreamed-args check
613 - index = len(tool_parser.prev_tool_call_arr) - 1
614 - args = tool_parser.prev_tool_call_arr[index].get("arguments", {})
615 - expected_call = (
616 - args if isinstance(args, str) else json.dumps(args, ensure_ascii=False)
617 - )
618 - actual_call = tool_parser.streamed_args_for_tool[index]
619 - remaining_call = expected_call.replace(actual_call, "", 1)
620 - assert remaining_call == ""
621 - else:
622 - assert len(tool_parser.streamed_args_for_tool) == 0
623 - assert len(tool_parser.prev_tool_call_arr) == 0
624 -
923
924 +@pytest.mark.parametrize("driver", ["legacy", "engine"])
925 @pytest.mark.parametrize(
926 ids=[
927 "no_tools",
@@ -748,6 +1047,7 @@ def test_extract_tool_calls_streaming_pre_v11_tokenizer(
1047 model_output,
1048 expected_tool_calls,
1049 expected_content,
1050 + driver,
1051 ):
1052 _test_extract_tool_calls_streaming(
1053 mistral_pre_v11_tool_parser,
@@ -756,9 +1056,15 @@ def test_extract_tool_calls_streaming_pre_v11_tokenizer(
1056 None,
1057 expected_tool_calls,
1058 expected_content,
1059 + driver=driver,
1060 )
1061
1062
1063 +@pytest.mark.parametrize(
1064 + "driver,reasoning_parser",
1065 + [("legacy", None), ("engine", None), ("engine", "mistral")],
1066 + ids=["legacy", "engine_no_reasoning", "engine_with_reasoning"],
1067 +)
1068 @pytest.mark.parametrize(
1069 ids=[
1070 "single_tool_add",
@@ -826,6 +1132,8 @@ def test_extract_tool_calls_streaming(
1132 tools,
1133 expected_tool_calls,
1134 expected_content,
1135 + driver,
1136 + reasoning_parser,
1137 ):
1138 _test_extract_tool_calls_streaming(
1139 mistral_tool_parser,
@@ -834,17 +1142,19 @@ def test_extract_tool_calls_streaming(
1142 tools,
1143 expected_tool_calls,
1144 expected_content,
1145 + driver=driver,
1146 + reasoning_parser=reasoning_parser,
1147 )
1148
1149
1150 +# Drives extract_tool_calls_streaming directly (bespoke detokenization loop),
1151 +# not via stream_delta_message_generator — driver parametrization not needed.
1152 def test_extract_tool_calls_streaming_v11_no_tools(
1153 mistral_tool_parser, mistral_tokenizer
1154 ):
1155 model_output = "This is a test"
844 - if isinstance(mistral_tokenizer, MistralTokenizer):
845 - all_token_ids = mistral_tokenizer.encode(model_output)
846 - else:
847 - all_token_ids = mistral_tokenizer.encode(model_output, add_special_tokens=False)
1156 + # add_special_tokens=False: real serving streams only generated tokens.
1157 + all_token_ids = mistral_tokenizer.encode(model_output, add_special_tokens=False)
1158 skip_special = isinstance(mistral_tokenizer, MistralTokenizer)
1159 collected_content = ""
1160 previous_text = ""
@@ -887,8 +1197,47 @@ def test_extract_tool_calls_streaming_v11_no_tools(
1197 previous_text = current_text
1198
1199 assert collected_content == model_output
890 - assert len(mistral_tool_parser.streamed_args_for_tool) == 0
891 - assert len(mistral_tool_parser.prev_tool_call_arr) == 0
1200 +
1201 +
1202 +def test_mistral_parser_drops_eos_from_output(mistral_tokenizer):
1203 + """EOS token must never surface as content/reasoning; literal EOS string
1204 + must be preserved when the EOS token id is absent.
1205 +
1206 + The engine drops special tokens by id, so:
1207 + Case A: the real EOS token id causes the text to be dropped.
1208 + Case B: the same EOS string with a non-EOS token id is preserved.
1209 + """
1210 + if not isinstance(mistral_tokenizer, MistralTokenizer):
1211 + pytest.skip("Requires MistralTokenizer")
1212 +
1213 + eos_id: int = mistral_tokenizer.eos_token_id
1214 + eos_text: str = mistral_tokenizer.decode([eos_id])
1215 + assert eos_text
1216 +
1217 + # Case A: real EOS token — text must be dropped.
1218 + parser_a = MistralParser(mistral_tokenizer)
1219 + parser_a.initialize_streaming()
1220 + events_a = parser_a._feed(eos_text, [eos_id])
1221 + events_a.extend(parser_a._engine.finish())
1222 + delta_a = parser_a._events_to_delta(events_a, finished=True)
1223 + content_a = (delta_a.content if delta_a else None) or ""
1224 + reasoning_a = (delta_a.reasoning if delta_a else None) or ""
1225 + assert eos_text not in content_a
1226 + assert eos_text not in reasoning_a
1227 +
1228 + # Case B: EOS text with a non-EOS token id — text must be preserved.
1229 + # This proves content equal to the EOS string is not silently dropped
1230 + # when it did not come from the real EOS token.
1231 + non_eos_ids = mistral_tokenizer.encode(text="hello", add_special_tokens=False)
1232 + non_eos_id = next(tid for tid in non_eos_ids if tid != eos_id)
1233 + parser_b = MistralParser(mistral_tokenizer)
1234 + parser_b.initialize_streaming()
1235 + events_b = parser_b._feed(eos_text, [non_eos_id])
1236 + events_b.extend(parser_b._engine.finish())
1237 + delta_b = parser_b._events_to_delta(events_b, finished=True)
1238 + content_b = (delta_b.content if delta_b else None) or ""
1239 + reasoning_b = (delta_b.reasoning if delta_b else None) or ""
1240 + assert eos_text in content_b or eos_text in reasoning_b
1241
1242
1243 @pytest.mark.parametrize(
@@ -961,6 +1310,55 @@ def test_extract_tool_calls_streaming_v11_no_tools(
1310 "bla",
1311 id="v11-content_before_tool",
1312 ),
1313 + pytest.param(
1314 + "mistral_tool_parser",
1315 + "mistral_tokenizer",
1316 + """[TOOL_CALLS]add_this_and_that[ARGS]{"a": 3.5, "b": 4}""", # noqa: E501
1317 + [
1318 + ToolCall(
1319 + function=FunctionCall(
1320 + name="add_this_and_that",
1321 + arguments=json.dumps({"a": 3.5, "b": 4}),
1322 + )
1323 + )
1324 + ],
1325 + "",
1326 + id="v11-single_tool_add_args",
1327 + ),
1328 + pytest.param(
1329 + "mistral_tool_parser",
1330 + "mistral_tokenizer",
1331 + """[TOOL_CALLS]add[ARGS]{"a": 3.5, "b": 4}[TOOL_CALLS]multiply[ARGS]{"a": 3, "b": 6}""", # noqa: E501
1332 + [
1333 + ToolCall(
1334 + function=FunctionCall(
1335 + name="add", arguments=json.dumps({"a": 3.5, "b": 4})
1336 + )
1337 + ),
1338 + ToolCall(
1339 + function=FunctionCall(
1340 + name="multiply", arguments=json.dumps({"a": 3, "b": 6})
1341 + )
1342 + ),
1343 + ],
1344 + "",
1345 + id="v11-multiple_tool_calls_args",
1346 + ),
1347 + pytest.param(
1348 + "mistral_tool_parser",
1349 + "mistral_tokenizer",
1350 + """bla[TOOL_CALLS]add_this_and_that[ARGS]{"a": 3.5, "b": 4}""", # noqa: E501
1351 + [
1352 + ToolCall(
1353 + function=FunctionCall(
1354 + name="add_this_and_that",
1355 + arguments=json.dumps({"a": 3.5, "b": 4}),
1356 + )
1357 + )
1358 + ],
1359 + "bla",
1360 + id="v11-content_before_tool_args",
1361 + ),
1362 pytest.param(
1363 "mistral_tool_parser",
1364 "mistral_tokenizer",
@@ -1116,11 +1514,15 @@ def test_extract_tool_calls_streaming_one_chunk(
1514 tool_parser = request.getfixturevalue(parser_fixture)
1515 tokenizer = request.getfixturevalue(tokenizer_fixture)
1516
1119 - if isinstance(tokenizer, MistralTokenizer):
1120 - all_token_ids = tokenizer.encode(model_output)
1517 + # Real serving streams only generated tokens (no BOS/EOS framing) with the
1518 + # control markers as special ids.
1519 + if isinstance(tokenizer, MistralTokenizer) and tokenizer.version >= 11:
1520 + all_token_ids = encode_mistral_output(tokenizer, model_output)
1521 else:
1522 all_token_ids = tokenizer.encode(model_output, add_special_tokens=False)
1123 - all_token_ids = fix_tool_call_tokenization(all_token_ids, tool_parser, tokenizer)
1523 + all_token_ids = fix_tool_call_tokenization(
1524 + all_token_ids, tool_parser, tokenizer
1525 + )
1526
1527 delta_message = tool_parser.extract_tool_calls_streaming(
1528 previous_text="",
@@ -1131,6 +1533,26 @@ def test_extract_tool_calls_streaming_one_chunk(
1533 delta_token_ids=all_token_ids,
1534 request=_DUMMY_REQUEST,
1535 )
1536 + # The engine buffers the final closing brace ('}') in _args_buffer until
1537 + # finish_streaming() is called, mirroring real serving. Flush it and
1538 + # merge the result so the asserted arguments are complete.
1539 + flush_delta = tool_parser.finish_streaming()
1540 + if flush_delta is not None and flush_delta.tool_calls:
1541 + if delta_message is not None and delta_message.tool_calls:
1542 + for flush_tc in flush_delta.tool_calls:
1543 + for existing_tc in delta_message.tool_calls:
1544 + if existing_tc.index == flush_tc.index and flush_tc.function:
1545 + existing_tc.function = (
1546 + existing_tc.function or DeltaFunctionCall()
1547 + )
1548 + existing_tc.function.arguments = (
1549 + existing_tc.function.arguments or ""
1550 + ) + (flush_tc.function.arguments or "")
1551 + elif delta_message is not None:
1552 + delta_message.tool_calls = flush_delta.tool_calls
1553 + else:
1554 + delta_message = flush_delta
1555 +
1556 assert isinstance(delta_message, DeltaMessage)
1557 assert len(delta_message.tool_calls) == len(expected_tool_calls)
1558
@@ -1145,13 +1567,6 @@ def test_extract_tool_calls_streaming_one_chunk(
1567 @pytest.mark.parametrize(
1568 "parser_fixture, model_output, fake_count, two_phase",
1569 [
1148 - pytest.param(
1149 - "mistral_tool_parser",
1150 - '[TOOL_CALLS]add{"a": 1, "b": 2}',
1151 - 20,
1152 - True,
1153 - id="v11",
1154 - ),
1570 pytest.param(
1571 "mistral_pre_v11_tool_parser",
1572 '[TOOL_CALLS] [{"name": "add", "arguments":{"a": 1, "b": 2}}]',
@@ -1164,7 +1579,11 @@ def test_extract_tool_calls_streaming_one_chunk(
1579 def test_fast_detokenization_text_detection(
1580 parser_fixture, model_output, fake_count, two_phase, request
1581 ):
1167 - """Regression: bot_token in text but not token_ids (PR #37209)."""
1582 + """Regression: bot_token in text but not token_ids (PR #37209).
1583 +
1584 + Only the pre-v11 legacy path detects the marker from text; v11+ routes
1585 + through the engine and detects the marker by token id.
1586 + """
1587 parser = request.getfixturevalue(parser_fixture)
1588 # Token IDs that do NOT contain bot_token_id.
1589 fake_token_ids = list(range(99, 99 + fake_count))
@@ -1210,34 +1629,24 @@ def test_fast_detokenization_text_detection(
1629 assert delta_message.tool_calls[0].function.name == "add"
1630
1631
1213 -@pytest.mark.parametrize(
1214 - "parser_fixture, patched_method, current_text",
1215 - [
1216 - (
1217 - "mistral_tool_parser",
1218 - "_extract_tool_calls_streaming",
1219 - "[TOOL_CALLS]add{}",
1220 - ),
1221 - (
1222 - "mistral_pre_v11_tool_parser",
1223 - "_extract_tool_calls_streaming_pre_v11_tokenizer",
1224 - '[TOOL_CALLS] [{"name":"a","arguments":{}}]',
1225 - ),
1226 - ],
1227 - ids=["v11", "pre_v11"],
1228 -)
1632 def test_extract_tool_calls_streaming_exception_returns_none(
1230 - parser_fixture, patched_method, current_text, request
1633 + mistral_pre_v11_tool_parser,
1634 ):
1232 - parser = request.getfixturevalue(parser_fixture)
1233 - with patch.object(parser, patched_method, side_effect=RuntimeError("boom")):
1635 + # v11 routes through the ParserEngine and does not swallow exceptions;
1636 + # only the pre-v11 legacy state-machine path has explicit exception handling.
1637 + parser = mistral_pre_v11_tool_parser
1638 + patched_method = "_extract_tool_calls_streaming_pre_v11_tokenizer"
1639 + current_text = '[TOOL_CALLS] [{"name":"a","arguments":{}}]'
1640 + with patch.object(
1641 + parser._parser_engine, patched_method, side_effect=RuntimeError("boom")
1642 + ):
1643 result = parser.extract_tool_calls_streaming(
1644 previous_text="",
1645 current_text=current_text,
1646 delta_text=current_text,
1647 previous_token_ids=[],
1239 - current_token_ids=[parser.bot_token_id],
1240 - delta_token_ids=[parser.bot_token_id],
1648 + current_token_ids=[parser._parser_engine.bot_token_id],
1649 + delta_token_ids=[parser._parser_engine.bot_token_id],
1650 request=_DUMMY_REQUEST,
1651 )
1652 assert result is None
@@ -1368,14 +1777,17 @@ def test_adjust_request_unsupported_grammar_for_tokenizer(mistral_tokenizer) ->
1777
1778 @pytest.mark.parametrize(
1779 "tool_choice,expected_skip",
1371 - [("auto", False), ("none", True)],
1372 - ids=["auto_skip_false", "none_skip_true"],
1780 + [("auto", False), ("none", False)],
1781 + ids=["auto_skip_false", "none_skip_false"],
1782 )
1783 def test_adjust_request_non_mistral_tokenizer(
1784 non_mistral_parser: MistralToolParser,
1785 tool_choice: str,
1786 expected_skip: bool,
1787 ) -> None:
1788 + # MistralParser (ParserEngine) always sets skip_special_tokens=False so
1789 + # that special tokens like [TOOL_CALLS] are visible to the parser even
1790 + # when tool_choice="none".
1791 request = _make_request(tool_choice=tool_choice)
1792 result = non_mistral_parser.adjust_request(request)
1793
@@ -1572,22 +1984,23 @@ def test_adjust_request_tool_choice_with_json_schema_factory_routing(
1984 assert len(result.structured_outputs.grammar) > 0
1985
1986
1575 -def test_grammar_from_tool_parser_default_false() -> None:
1987 +def test_grammar_from_parser_default_false() -> None:
1988 request = _make_request()
1577 - assert request._grammar_from_tool_parser is False
1989 + assert request._grammar_from_parser is False
1990
1991
1580 -def test_grammar_from_tool_parser_set_by_adjust_request(
1992 +def test_grammar_from_parser_set_by_adjust_request(
1993 mistral_tool_parser: MistralToolParser,
1994 ) -> None:
1995 request = _make_request()
1996 result = mistral_tool_parser.adjust_request(request)
1585 - assert result._grammar_from_tool_parser is True
1997 + assert result._grammar_from_parser is True
1998
1999
2000 +@pytest.mark.parametrize("driver", ["legacy", "engine"])
2001 @pytest.mark.parametrize("chunk_size", [2, 3, 4, 5])
2002 def test_streaming_pre_v11_parallel_calls_batched_deltas(
1590 - mistral_pre_v11_tool_parser, mistral_pre_v11_tokenizer, chunk_size
2003 + mistral_pre_v11_tool_parser, mistral_pre_v11_tokenizer, chunk_size, driver
2004 ):
2005 """A batched delta spanning the boundary between two parallel calls must
2006 keep them on distinct indices (the bug collapsed both onto index 0)."""
@@ -1605,6 +2018,7 @@ def test_streaming_pre_v11_parallel_calls_batched_deltas(
2018 model_output,
2019 tools=None,
2020 chunk_size=chunk_size,
2021 + driver=driver,
2022 ):
2023 for tool_call in delta_message.tool_calls or []:
2024 if tool_call.index != idx:
@@ -1619,3 +2033,623 @@ def test_streaming_pre_v11_parallel_calls_batched_deltas(
2033 assert len(args) == 2
2034 # trailing args of the final call are flushed by the serving layer
2035 assert json.loads(args[0]) == {"a": 3.5, "b": 4}
2036 +
2037 +
2038 +@pytest.mark.parametrize(
2039 + "reasoning_encoding",
2040 + ["text", "special_token"],
2041 + ids=["text_encoding", "special_token_encoding"],
2042 +)
2043 +def test_content_tool_calls_transition_emits_reasoning_end(
2044 + mistral_tokenizer,
2045 + reasoning_encoding,
2046 +):
2047 + """(CONTENT, TOOL_CALLS) must emit REASONING_END when reasoning is enabled."""
2048 +
2049 + cfg = mistral_config(reasoning_encoding=reasoning_encoding)
2050 + engine = StreamingParserEngine(
2051 + config=cfg, tokenizer=mistral_tokenizer, vocab=mistral_tokenizer.get_vocab()
2052 + )
2053 + engine.skip_tool_parsing = True
2054 +
2055 + bot_token_id = mistral_tokenizer.get_vocab().get("[TOOL_CALLS]")
2056 + assert bot_token_id is not None
2057 +
2058 + events = engine.feed(delta_text="[TOOL_CALLS]", delta_token_ids=[bot_token_id])
2059 + event_types = [e.type for e in events]
2060 +
2061 + assert EventType.REASONING_END in event_types, (
2062 + f"REASONING_END missing for reasoning_encoding={reasoning_encoding!r}; "
2063 + f"got events: {event_types}"
2064 + )
2065 + # Tool call start must NOT appear in skip_tool_parsing mode
2066 + assert EventType.TOOL_CALL_START not in event_types
2067 +
2068 +
2069 +@pytest.mark.parametrize("chunk_size", [1, 3])
2070 +@pytest.mark.parametrize(
2071 + "tools,expected_tool_calls,expected_content",
2072 + [
2073 + (
2074 + [("add", '{"a": 3, "b": 4}')],
2075 + [
2076 + ToolCall(
2077 + function=FunctionCall(
2078 + name="add", arguments=json.dumps({"a": 3, "b": 4})
2079 + )
2080 + )
2081 + ],
2082 + "",
2083 + ),
2084 + (
2085 + [
2086 + ("add", '{"a": 3.5, "b": 4}'),
2087 + (
2088 + "get_current_weather",
2089 + '{"city": "San Francisco", "state": "CA", "unit": "celsius"}',
2090 + ),
2091 + ],
2092 + [
2093 + ToolCall(
2094 + function=FunctionCall(
2095 + name="add", arguments=json.dumps({"a": 3.5, "b": 4})
2096 + )
2097 + ),
2098 + ToolCall(
2099 + function=FunctionCall(
2100 + name="get_current_weather",
2101 + arguments=json.dumps(
2102 + {
2103 + "city": "San Francisco",
2104 + "state": "CA",
2105 + "unit": "celsius",
2106 + }
2107 + ),
2108 + )
2109 + ),
2110 + ],
2111 + "",
2112 + ),
2113 + ],
2114 + ids=["single_tool_add", "parallel_tools"],
2115 +)
2116 +def test_reasoning_active_no_think_block_no_leak(
2117 + mistral_tool_parser,
2118 + mistral_tokenizer,
2119 + tools,
2120 + expected_tool_calls,
2121 + expected_content,
2122 + chunk_size,
2123 +):
2124 + """Tool call without a preceding think block must not leak as content
2125 + when the reasoning parser is active."""
2126 + accumulated_content = ""
2127 + function_names: list[str] = []
2128 + function_args_strs: list[str] = []
2129 + tool_call_idx = -1
2130 + tool_call_ids: list[str | None] = []
2131 +
2132 + for delta_message in stream_delta_message_generator(
2133 + mistral_tool_parser,
2134 + mistral_tokenizer,
2135 + model_output=None,
2136 + tools=tools,
2137 + chunk_size=chunk_size,
2138 + driver="engine",
2139 + reasoning_parser="mistral",
2140 + ):
2141 + if delta_message.content:
2142 + accumulated_content += delta_message.content
2143 +
2144 + for tool_call in delta_message.tool_calls or []:
2145 + if tool_call.index != tool_call_idx:
2146 + tool_call_idx = tool_call.index
2147 + function_args_strs.append("")
2148 + tool_call_ids.append(None)
2149 + if tool_call.id and not tool_call_ids[tool_call.index]:
2150 + tool_call_ids[tool_call.index] = tool_call.id
2151 + if tool_call.function:
2152 + if tool_call.function.name:
2153 + function_names.append(tool_call.function.name)
2154 + if tool_call.function.arguments:
2155 + function_args_strs[tool_call.index] += tool_call.function.arguments
2156 +
2157 + assert "[TOOL_CALLS]" not in accumulated_content
2158 + assert "[ARGS]" not in accumulated_content
2159 + assert accumulated_content == expected_content
2160 +
2161 + actual_tool_calls = [
2162 + ToolCall(
2163 + id=tc_id,
2164 + function=FunctionCall(
2165 + name=name,
2166 + arguments=partial_json_parser.ensure_json(args, Allow.OBJ | Allow.STR),
2167 + ),
2168 + )
2169 + for tc_id, name, args in zip(tool_call_ids, function_names, function_args_strs)
2170 + ]
2171 + assert_tool_calls(actual_tool_calls, expected_tool_calls)
2172 +
2173 +
2174 +# ---------------------------------------------------------------------------
2175 +# Pre-v11 guided schema injection and bare-array extraction tests
2176 +# ---------------------------------------------------------------------------
2177 +
2178 +
2179 +@pytest.mark.parametrize(
2180 + "tool_choice,expected_items_count",
2181 + [
2182 + ("required", 2), # all tools
2183 + (
2184 + {"type": "function", "function": {"name": "get_weather"}},
2185 + 1,
2186 + ), # named: single tool
2187 + ],
2188 + ids=["required", "named"],
2189 +)
2190 +def test_adjust_request_pre_v11_guided_schema_injected(
2191 + mistral_pre_v11_tool_parser: MistralToolParser,
2192 + tool_choice: object,
2193 + expected_items_count: int,
2194 +) -> None:
2195 + request = _make_request(tool_choice=tool_choice)
2196 + result = mistral_pre_v11_tool_parser.adjust_request(request)
2197 +
2198 + assert result.structured_outputs is not None
2199 + schema = result.structured_outputs.json
2200 + if isinstance(schema, str):
2201 + schema = json.loads(schema)
2202 + assert schema["type"] == "array"
2203 + assert schema["minItems"] == 1
2204 + items = schema["items"]
2205 + assert "anyOf" in items
2206 + assert len(items["anyOf"]) == expected_items_count
2207 + for entry in items["anyOf"]:
2208 + props = entry["properties"]
2209 + assert "name" in props
2210 + assert "arguments" in props
2211 +
2212 +
2213 +@pytest.mark.parametrize(
2214 + "response_format",
2215 + [{"type": "json_object"}, {"type": "json_schema", "json_schema": {"name": "s"}}],
2216 + ids=["json_object", "json_schema"],
2217 +)
2218 +def test_adjust_request_pre_v11_required_clears_response_format(
2219 + mistral_pre_v11_tool_parser: MistralToolParser,
2220 + response_format: dict,
2221 +) -> None:
2222 + """required + response_format must inject the tool schema and clear
2223 + response_format so the tool schema is the sole structured-output
2224 + constraint, matching the base ToolParser (otherwise the request either
2225 + hits the "multiple constraints" engine error or, with no constraint,
2226 + rambles to finish_reason='length')."""
2227 + request = _make_request(tool_choice="required", response_format=response_format)
2228 + result = mistral_pre_v11_tool_parser.adjust_request(request)
2229 +
2230 + assert result.response_format is None
2231 + assert result.structured_outputs is not None
2232 + schema = result.structured_outputs.json
2233 + if isinstance(schema, str):
2234 + schema = json.loads(schema)
2235 + assert schema["type"] == "array"
2236 +
2237 +
2238 +def test_adjust_request_non_mistral_tokenizer_required_injects_schema(
2239 + non_mistral_parser: MistralToolParser,
2240 +) -> None:
2241 + """The guided-schema injection must also fire for non-Mistral (e.g. HF-mode)
2242 + tokenizers driving the Mistral tool parser, mirroring the base ToolParser so
2243 + required + response_format does not fall back to an unconstrained ramble."""
2244 + request = _make_request(
2245 + tool_choice="required", response_format={"type": "json_object"}
2246 + )
2247 + result = non_mistral_parser.adjust_request(request)
2248 +
2249 + assert result.response_format is None
2250 + assert result.structured_outputs is not None
2251 + assert result.structured_outputs.json is not None
2252 +
2253 +
2254 +@pytest.mark.parametrize(
2255 + "tool_choice",
2256 + ["auto", "none"],
2257 + ids=["auto", "none"],
2258 +)
2259 +def test_adjust_request_pre_v11_no_injection_for_auto_none(
2260 + mistral_pre_v11_tool_parser: MistralToolParser,
2261 + tool_choice: str,
2262 +) -> None:
2263 + request = _make_request(tool_choice=tool_choice)
2264 + result = mistral_pre_v11_tool_parser.adjust_request(request)
2265 +
2266 + assert result.structured_outputs is None
2267 +
2268 +
2269 +def test_legacy_extract_tool_calls_guided_bare_array_required(
2270 + mistral_pre_v11_tool_parser: MistralToolParser,
2271 +) -> None:
2272 + model_output = '[{"name": "get_current_weather", "arguments": {"city": "Dallas"}}]'
2273 + request = _make_request(tool_choice="required")
2274 +
2275 + result = mistral_pre_v11_tool_parser.extract_tool_calls(
2276 + model_output, request=request
2277 + )
2278 +
2279 + assert result.tools_called
2280 + assert len(result.tool_calls) == 1
2281 + tc = result.tool_calls[0]
2282 + assert tc.function.name == "get_current_weather"
2283 + assert json.loads(tc.function.arguments) == {"city": "Dallas"}
2284 + assert isinstance(tc.id, str)
2285 + assert len(tc.id) == 9
2286 + assert result.content is None
2287 +
2288 +
2289 +def test_legacy_extract_tool_calls_none_with_tools(
2290 + mistral_pre_v11_tool_parser: MistralToolParser,
2291 +) -> None:
2292 + model_output = (
2293 + '[TOOL_CALLS] [{"name": "get_current_weather",'
2294 + ' "arguments": {"city": "Dallas"}}]'
2295 + )
2296 + request = _make_request(tool_choice="none")
2297 +
2298 + result = mistral_pre_v11_tool_parser.extract_tool_calls(
2299 + model_output, request=request
2300 + )
2301 +
2302 + assert not result.tools_called
2303 + assert result.tool_calls == []
2304 + assert result.content == model_output
2305 +
2306 +
2307 +def test_guided_streaming_required_pre_v11(
2308 + mistral_pre_v11_tool_parser: MistralToolParser,
2309 + mistral_pre_v11_tokenizer,
2310 +) -> None:
2311 + model_output = '[{"name": "get_current_weather", "arguments": {"city": "Dallas"}}]'
2312 + request = _make_request(tool_choice="required")
2313 + all_token_ids = mistral_pre_v11_tokenizer.encode(
2314 + model_output, add_special_tokens=False
2315 + )
2316 +
2317 + function_name: str | None = None
2318 + function_args = ""
2319 + tool_call_id: str | None = None
2320 + previous_text = ""
2321 + previous_tokens = None
2322 + prefix_offset = 0
2323 + read_offset = 0
2324 +
2325 + for i, token_id in enumerate(all_token_ids):
2326 + (new_tokens, delta_text, prefix_offset, read_offset) = detokenize_incrementally(
2327 + tokenizer=mistral_pre_v11_tokenizer,
2328 + all_input_ids=all_token_ids[: i + 1],
2329 + prev_tokens=previous_tokens,
2330 + prefix_offset=prefix_offset,
2331 + read_offset=read_offset,
2332 + skip_special_tokens=False,
2333 + spaces_between_special_tokens=True,
2334 + )
2335 + previous_tokens = (
2336 + (previous_tokens + new_tokens) if previous_tokens else new_tokens
2337 + )
2338 + current_text = previous_text + delta_text
2339 +
2340 + delta_message = mistral_pre_v11_tool_parser.extract_tool_calls_streaming(
2341 + previous_text=previous_text,
2342 + current_text=current_text,
2343 + delta_text=delta_text,
2344 + previous_token_ids=all_token_ids[:i],
2345 + current_token_ids=all_token_ids[: i + 1],
2346 + delta_token_ids=[token_id],
2347 + request=request,
2348 + )
2349 + previous_text = current_text
2350 +
2351 + if delta_message and delta_message.tool_calls:
2352 + for tc in delta_message.tool_calls:
2353 + if tc.id and not tool_call_id:
2354 + tool_call_id = tc.id
2355 + if tc.function:
2356 + if tc.function.name:
2357 + function_name = tc.function.name
2358 + if tc.function.arguments:
2359 + function_args += tc.function.arguments
2360 +
2361 + assert function_name == "get_current_weather"
2362 + assert tool_call_id is not None
2363 + assert len(tool_call_id) == 9
2364 + assert json.loads(function_args) == {"city": "Dallas"}
2365 +
2366 +
2367 +# ---------------------------------------------------------------------------
2368 +# Malformed-name-before-marker matrix: {malformed, well-formed, mixed
2369 +# parallel} x {streaming, non-streaming}.
2370 +#
2371 +# These tests drive a real MistralParser built from the module's
2372 +# `mistral_tokenizer` fixture (grammar-capable v11+ tokenizer, so the
2373 +# declarative ParserEngine path is exercised, not the pre-v11 legacy state
2374 +# machine) with actual bash/search_replace/submit tools, mirroring the tools
2375 +# present in the real captured traces this bug was found in.
2376 +# ---------------------------------------------------------------------------
2377 +
2378 +_BASH_SEARCH_SUBMIT_TOOLS_DICTS = [
2379 + {
2380 + "type": "function",
2381 + "function": {
2382 + "name": "bash",
2383 + "description": "Run a bash command",
2384 + "parameters": {
2385 + "type": "object",
2386 + "properties": {"command": {"type": "string"}},
2387 + "required": ["command"],
2388 + },
2389 + },
2390 + },
2391 + {
2392 + "type": "function",
2393 + "function": {
2394 + "name": "search_replace",
2395 + "description": "Search and replace text in a file",
2396 + "parameters": {
2397 + "type": "object",
2398 + "properties": {
2399 + "path": {"type": "string"},
2400 + "search": {"type": "string"},
2401 + "replace": {"type": "string"},
2402 + },
2403 + "required": ["path", "search", "replace"],
2404 + },
2405 + },
2406 + },
2407 + {
2408 + "type": "function",
2409 + "function": {
2410 + "name": "submit",
2411 + "description": "Submit the final answer",
2412 + "parameters": {
2413 + "type": "object",
2414 + "properties": {"answer": {"type": "object"}},
2415 + "required": ["answer"],
2416 + },
2417 + },
2418 + },
2419 +]
2420 +
2421 +
2422 +def _malformed_matrix_request() -> ChatCompletionRequest:
2423 + return ChatCompletionRequest(
2424 + messages=[],
2425 + model="test",
2426 + tools=_BASH_SEARCH_SUBMIT_TOOLS_DICTS,
2427 + tool_choice="auto",
2428 + )
2429 +
2430 +
2431 +def _stream_via_parse_delta(
2432 + parser: MistralParser,
2433 + tokenizer: MistralTokenizer,
2434 + model_output: str,
2435 + request: ChatCompletionRequest,
2436 +) -> tuple[str, list[str | None], list[str]]:
2437 + """Feed `model_output` through `parser.parse_delta` token-by-token.
2438 +
2439 + This is the exact API production serving drives on every generated
2440 + token (see `ParserEngine.parse_delta`), unlike
2441 + `extract_tool_calls_streaming` which some legacy call sites drive with
2442 + pre-computed previous/current text windows.
2443 + """
2444 + parser.initialize_streaming()
2445 + all_token_ids = encode_mistral_output(tokenizer, model_output)
2446 +
2447 + content = ""
2448 + tool_call_idx = -1
2449 + names: list[str | None] = []
2450 + args: list[str] = []
2451 +
2452 + previous_tokens = None
2453 + prefix_offset = 0
2454 + read_offset = 0
2455 + for i, token_id in enumerate(all_token_ids):
2456 + new_tokens, delta_text, prefix_offset, read_offset = detokenize_incrementally(
2457 + tokenizer=tokenizer,
2458 + all_input_ids=all_token_ids[: i + 1],
2459 + prev_tokens=previous_tokens,
2460 + prefix_offset=prefix_offset,
2461 + read_offset=read_offset,
2462 + skip_special_tokens=False,
2463 + spaces_between_special_tokens=True,
2464 + )
2465 + previous_tokens = (
2466 + previous_tokens + new_tokens if previous_tokens else new_tokens
2467 + )
2468 + finished = i == len(all_token_ids) - 1
2469 +
2470 + delta_message = parser.parse_delta(
2471 + delta_text=delta_text,
2472 + delta_token_ids=[token_id],
2473 + request=request,
2474 + prompt_token_ids=[1],
2475 + finished=finished,
2476 + )
2477 + if delta_message is None:
2478 + continue
2479 + if delta_message.content:
2480 + content += delta_message.content
2481 + for tool_call in delta_message.tool_calls or []:
2482 + if tool_call.index != tool_call_idx:
2483 + tool_call_idx = tool_call.index
2484 + names.append(None)
2485 + args.append("")
2486 + if tool_call.function:
2487 + if tool_call.function.name is not None:
2488 + names[tool_call.index] = tool_call.function.name
2489 + if tool_call.function.arguments:
2490 + args[tool_call.index] += tool_call.function.arguments
2491 +
2492 + return content, names, args
2493 +
2494 +
2495 +_MALFORMED_MATRIX_CASES = [
2496 + pytest.param(
2497 + # 2x2 corner: name present, `[ARGS]` token present.
2498 + '[TOOL_CALLS]bash[ARGS]{"command": "ls -la"}',
2499 + ["bash"],
2500 + [json.dumps({"command": "ls -la"})],
2501 + None,
2502 + id="wellformed_single",
2503 + ),
2504 + pytest.param(
2505 + # 2x2 corner: name present, no `[ARGS]` token.
2506 + '[TOOL_CALLS]bash{"command": "ls -la"}',
2507 + ["bash"],
2508 + [json.dumps({"command": "ls -la"})],
2509 + None,
2510 + id="wellformed_single_no_args_token",
2511 + ),
2512 + pytest.param(
2513 + # 2x2 corner: name skipped, no `[ARGS]` token. "bash" lands in
2514 + # content before the marker, then generation jumps straight to
2515 + # args (no NAME slot, no [ARGS]).
2516 + 'bash[TOOL_CALLS]{"command": "cat file.py"}',
2517 + [""],
2518 + [json.dumps({"command": "cat file.py"})],
2519 + "bash",
2520 + id="malformed_single",
2521 + ),
2522 + pytest.param(
2523 + # 2x2 corner: name skipped, `[ARGS]` token present.
2524 + 'bash[TOOL_CALLS][ARGS]{"command": "cat file.py"}',
2525 + [""],
2526 + [json.dumps({"command": "cat file.py"})],
2527 + "bash",
2528 + id="malformed_single_args_token",
2529 + ),
2530 + pytest.param(
2531 + # parallel well-formed x2; the second call's nested braces in its
2532 + # arguments must not confuse call splitting.
2533 + '[TOOL_CALLS]bash[ARGS]{"command": "ls"}'
2534 + '[TOOL_CALLS]submit[ARGS]{"answer": {"nested": 1}}',
2535 + ["bash", "submit"],
2536 + [json.dumps({"command": "ls"}), json.dumps({"answer": {"nested": 1}})],
2537 + None,
2538 + id="parallel_wellformed",
2539 + ),
2540 + pytest.param(
2541 + # parallel malformed x2: both calls skip the NAME slot entirely
2542 + # (marker immediately followed by `{`), each ending with an empty
2543 + # name and correctly separated (nested-brace) arguments.
2544 + 'bash[TOOL_CALLS]{"command": "ls"}[TOOL_CALLS]{"answer": {"nested": 1}}',
2545 + ["", ""],
2546 + [json.dumps({"command": "ls"}), json.dumps({"answer": {"nested": 1}})],
2547 + "bash",
2548 + id="parallel_malformed",
2549 + ),
2550 + pytest.param(
2551 + # parallel malformed x2, `[ARGS]` token variant: both calls skip
2552 + # the NAME slot but enter TOOL_ARGS via `[ARGS]` instead of `{`.
2553 + 'bash[TOOL_CALLS][ARGS]{"command": "ls"}'
2554 + '[TOOL_CALLS][ARGS]{"answer": {"nested": 1}}',
2555 + ["", ""],
2556 + [json.dumps({"command": "ls"}), json.dumps({"answer": {"nested": 1}})],
2557 + "bash",
2558 + id="parallel_malformed_args_token",
2559 + ),
2560 + pytest.param(
2561 + # mixed: malformed first call, well-formed second call. The
2562 + # important case: proves calls are split on the `[TOOL_CALLS]`
2563 + # marker, not on braces — the first call's simple args and the
2564 + # second call's nested-brace args must not bleed into each other.
2565 + 'bash[TOOL_CALLS]{"command": "ls"}'
2566 + '[TOOL_CALLS]submit[ARGS]{"answer": {"nested": 1}}',
2567 + ["", "submit"],
2568 + [json.dumps({"command": "ls"}), json.dumps({"answer": {"nested": 1}})],
2569 + "bash",
2570 + id="mixed_malformed_then_wellformed",
2571 + ),
2572 + pytest.param(
2573 + # mixed, `[ARGS]` token variant: malformed first call enters
2574 + # TOOL_ARGS via `[ARGS]` instead of `{`, second call well-formed.
2575 + 'bash[TOOL_CALLS][ARGS]{"command": "ls"}'
2576 + '[TOOL_CALLS]submit[ARGS]{"answer": {"nested": 1}}',
2577 + ["", "submit"],
2578 + [json.dumps({"command": "ls"}), json.dumps({"answer": {"nested": 1}})],
2579 + "bash",
2580 + id="mixed_malformed_args_token_then_wellformed",
2581 + ),
2582 + pytest.param(
2583 + # mixed: well-formed first call, malformed second call (NAME slot
2584 + # skipped entirely for the second call).
2585 + '[TOOL_CALLS]bash[ARGS]{"command": "ls"}[TOOL_CALLS]{"answer": {"nested": 1}}',
2586 + ["bash", ""],
2587 + [json.dumps({"command": "ls"}), json.dumps({"answer": {"nested": 1}})],
2588 + None,
2589 + id="mixed_wellformed_then_malformed",
2590 + ),
2591 +]
2592 +
2593 +
2594 +@pytest.mark.parametrize(
2595 + "model_output,expected_names,expected_args,expected_content",
2596 + _MALFORMED_MATRIX_CASES,
2597 +)
2598 +def test_malformed_tool_call_matrix_non_streaming(
2599 + mistral_tokenizer,
2600 + model_output,
2601 + expected_names,
2602 + expected_args,
2603 + expected_content,
2604 +):
2605 + parser = MistralParser(mistral_tokenizer)
2606 + request = _malformed_matrix_request()
2607 +
2608 + result = parser.extract_tool_calls_from_content(model_output, request)
2609 +
2610 + assert result.tools_called
2611 + assert [tc.function.name for tc in result.tool_calls] == expected_names
2612 + assert [tc.function.arguments for tc in result.tool_calls] == expected_args
2613 + assert result.content == expected_content
2614 +
2615 +
2616 +@pytest.mark.parametrize(
2617 + "model_output,expected_names,expected_args,expected_content",
2618 + _MALFORMED_MATRIX_CASES,
2619 +)
2620 +def test_malformed_tool_call_matrix_streaming(
2621 + mistral_tokenizer,
2622 + model_output,
2623 + expected_names,
2624 + expected_args,
2625 + expected_content,
2626 +):
2627 + parser = MistralParser(mistral_tokenizer)
2628 + request = _malformed_matrix_request()
2629 +
2630 + content, names, args = _stream_via_parse_delta(
2631 + parser, mistral_tokenizer, model_output, request
2632 + )
2633 +
2634 + assert content == (expected_content or "")
2635 + assert names == expected_names
2636 + assert args == expected_args
2637 +
2638 +
2639 +# Real payload captured from a production SWE-bench-agent.
2640 +_REAL_LEAK_CONTENT = 'bash[TOOL_CALLS]{"command": "cd /testbed && git diff xarray/core/weighted.py"}ometracediff --git a/xarray/core/weighted.py b/xarray/core/weighted.py\\nindex 5f2d138e..9c38b0b7 100644\\n--- a/xarray/core/weighted.py\\n+++ b/xarray/core/weighted.py\\n@@ -1,4 +1,6 @@\\n from typing import TYPE_CHECKING, Hashable, Iterable, Optional, Union, overload\\n+\\n+import numpy as np\\n \\n from .computation import dot\\n from .options import _get_keep_attrs\\n@@ -105,6 +107,10 @@ class Weighted:\\n )\\n \\n self.obj = obj\\n- self.weights = weights\\n+ # Ensure weights are numeric (float64) to avoid boolean arithmetic issues\\n+ # in dot products and weighted operations\\n+ if not np.issubdtype(weights.dtype, np.number):\\n+ weights = weights.astype(np.float64)\\n+ self.weights = weights\\n "}现在 hareview the changes we made:bash[ARGS]{"command": "cat /testbed/xarray/core/weighted.py | head -120 | tail -20"}' # noqa: E501
2641 +
2642 +
2643 +def test_extract_tool_calls_real_captured_malformed_output(mistral_tokenizer):
2644 + parser = MistralParser(mistral_tokenizer)
2645 + request = _malformed_matrix_request()
2646 +
2647 + result = parser.extract_tool_calls_from_content(_REAL_LEAK_CONTENT, request)
2648 +
2649 + assert result.tools_called
2650 + assert len(result.tool_calls) == 1
2651 + assert result.tool_calls[0].function.name == ""
2652 + assert result.tool_calls[0].function.arguments == json.dumps(
2653 + {"command": "cd /testbed && git diff xarray/core/weighted.py"}
2654 + )
2655 + assert result.content == "bash"
tests/reasoning/test_mistral_reasoning_parser.py deleted
-348
@@ -1,348 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -import pytest
5 -
6 -from tests.reasoning.utils import run_reasoning_extraction_mistral
7 -from vllm.reasoning import ReasoningParser, ReasoningParserManager
8 -from vllm.tokenizers.mistral import MistralTokenizer
9 -
10 -parser_name = "mistral"
11 -
12 -
13 -@pytest.fixture(scope="module")
14 -def mistral_tokenizer():
15 - mistral_tokenizer = MistralTokenizer.from_pretrained(
16 - "mistralai/Magistral-Small-2509"
17 - )
18 - return mistral_tokenizer
19 -
20 -
21 -INVALID_SIMPLE_REASONING = {
22 - "output": "This is a reasoning section[/THINK]This is the rest",
23 - "reasoning": None,
24 - "content": "This is a reasoning sectionThis is the rest",
25 - "is_reasoning_end": False,
26 -}
27 -INVALID_COMPLETE_REASONING = {
28 - "output": "This is a reasoning section[/THINK]",
29 - "reasoning": None,
30 - "content": "This is a reasoning section",
31 - "is_reasoning_end": False,
32 -}
33 -NO_CONTENT = {
34 - "output": "[THINK]This is reasoning",
35 - "reasoning": "This is reasoning",
36 - "content": None,
37 - "is_reasoning_end": False,
38 -}
39 -NO_REASONING = {
40 - "output": "This is content",
41 - "reasoning": None,
42 - "content": "This is content",
43 - "is_reasoning_end": False,
44 -}
45 -NO_REASONING_STREAMING = {
46 - "output": "This is a reasoning section",
47 - "reasoning": None,
48 - "content": "This is a reasoning section",
49 - "is_reasoning_end": False,
50 -}
51 -INVALID_MULTIPLE_LINES = {
52 - "output": "This\nThat[/THINK]This is the rest\nThat",
53 - "reasoning": None,
54 - "content": "This\nThatThis is the rest\nThat",
55 - "is_reasoning_end": False,
56 -}
57 -INVALID_SHORTEST_REASONING_NO_STREAMING = {
58 - "output": "[/THINK]This is the rest",
59 - "reasoning": None,
60 - "content": "This is the rest",
61 - "is_reasoning_end": False,
62 -}
63 -INVALID_SHORTEST_REASONING = {
64 - "output": "[/THINK]This is the rest",
65 - "reasoning": None,
66 - "content": "This is the rest",
67 - "is_reasoning_end": False,
68 -}
69 -REASONING_WITH_THINK = {
70 - "output": "[THINK]This is a reasoning section[/THINK]This is the rest",
71 - "reasoning": "This is a reasoning section",
72 - "content": "This is the rest",
73 - "is_reasoning_end": True,
74 -}
75 -COMPLETE_REASONING_WITH_THINK = {
76 - "output": "[THINK]This is a reasoning section[/THINK]",
77 - "reasoning": "This is a reasoning section",
78 - "content": None,
79 - "is_reasoning_end": True,
80 -}
81 -MULTIPLE_LINES_WITH_THINK = {
82 - "output": "[THINK]This\nThat[/THINK]This is the rest\nThat",
83 - "reasoning": "This\nThat",
84 - "content": "This is the rest\nThat",
85 - "is_reasoning_end": True,
86 -}
87 -INVALID_SHORTEST_REASONING_NO_STREAMING_WITH_THINK = {
88 - "output": "[/THINK]This is the rest",
89 - "reasoning": None,
90 - "content": "This is the rest",
91 - "is_reasoning_end": False,
92 -}
93 -INVALID_SHORTEST_REASONING_WITH_THINK = {
94 - "output": "[/THINK]This is the rest",
95 - "reasoning": None,
96 - "content": "This is the rest",
97 - "is_reasoning_end": False,
98 -}
99 -THINK_NO_END = {
100 - "output": "[THINK]This is a reasoning section",
101 - "reasoning": "This is a reasoning section",
102 - "content": None,
103 - "is_reasoning_end": False,
104 -}
105 -EMPTY = {
106 - "output": "",
107 - "reasoning": None,
108 - "content": "",
109 - "is_reasoning_end": False,
110 -}
111 -EMPTY_STREAMING = {
112 - "output": "",
113 - "reasoning": None,
114 - "content": None,
115 - "is_reasoning_end": False,
116 -}
117 -NEW_LINE = {
118 - "output": "Before\n[THINK]This is a reasoning section[/THINK]\nThis is the rest",
119 - "reasoning": "This is a reasoning section",
120 - "content": "Before\n\nThis is the rest",
121 - "is_reasoning_end": True,
122 -}
123 -NEW_LINE_STREAMING = {
124 - "output": "Before\n[THINK]This is a reasoning section[/THINK]\nThis is the rest",
125 - "reasoning": "This is a reasoning section",
126 - "content": "Before\n\nThis is the rest",
127 - "is_reasoning_end": True,
128 -}
129 -
130 -TEST_CASES = [
131 - pytest.param(
132 - False,
133 - INVALID_SIMPLE_REASONING,
134 - id="invalid_simple_reasoning",
135 - ),
136 - pytest.param(
137 - True,
138 - INVALID_SIMPLE_REASONING,
139 - id="invalid_simple_reasoning_streaming",
140 - ),
141 - pytest.param(
142 - False,
143 - INVALID_COMPLETE_REASONING,
144 - id="invalid_complete_reasoning",
145 - ),
146 - pytest.param(
147 - True,
148 - INVALID_COMPLETE_REASONING,
149 - id="invalid_complete_reasoning_streaming",
150 - ),
151 - pytest.param(
152 - False,
153 - NO_CONTENT,
154 - id="no_content",
155 - ),
156 - pytest.param(
157 - False,
158 - NO_REASONING,
159 - id="no_reasoning",
160 - ),
161 - pytest.param(
162 - True,
163 - NO_REASONING_STREAMING,
164 - id="no_reasoning_token_streaming",
165 - ),
166 - pytest.param(
167 - False,
168 - INVALID_MULTIPLE_LINES,
169 - id="invalid_multiple_lines",
170 - ),
171 - pytest.param(
172 - True,
173 - INVALID_MULTIPLE_LINES,
174 - id="invalid_multiple_lines_streaming",
175 - ),
176 - pytest.param(
177 - True,
178 - INVALID_SHORTEST_REASONING,
179 - id="invalid_shortest",
180 - ),
181 - pytest.param(
182 - False,
183 - INVALID_SHORTEST_REASONING_NO_STREAMING,
184 - id="invalid_shortest_streaming",
185 - ),
186 - pytest.param(
187 - False,
188 - REASONING_WITH_THINK,
189 - id="reasoning_with_think",
190 - ),
191 - pytest.param(
192 - True,
193 - REASONING_WITH_THINK,
194 - id="reasoning_with_think_streaming",
195 - ),
196 - pytest.param(
197 - False,
198 - COMPLETE_REASONING_WITH_THINK,
199 - id="complete_reasoning_with_think",
200 - ),
201 - pytest.param(
202 - True,
203 - COMPLETE_REASONING_WITH_THINK,
204 - id="complete_reasoning_with_think_streaming",
205 - ),
206 - pytest.param(
207 - False,
208 - MULTIPLE_LINES_WITH_THINK,
209 - id="multiple_lines_with_think",
210 - ),
211 - pytest.param(
212 - True,
213 - MULTIPLE_LINES_WITH_THINK,
214 - id="multiple_lines_with_think_streaming",
215 - ),
216 - pytest.param(
217 - False,
218 - INVALID_SHORTEST_REASONING_NO_STREAMING_WITH_THINK,
219 - id="invalid_shortest_with_think",
220 - ),
221 - pytest.param(
222 - True,
223 - INVALID_SHORTEST_REASONING_WITH_THINK,
224 - id="invalid_shortest_with_think_streaming",
225 - ),
226 - pytest.param(
227 - False,
228 - THINK_NO_END,
229 - id="think_no_end",
230 - ),
231 - pytest.param(
232 - True,
233 - THINK_NO_END,
234 - id="think_no_end_streaming",
235 - ),
236 - pytest.param(
237 - False,
238 - EMPTY,
239 - id="empty",
240 - ),
241 - pytest.param(
242 - True,
243 - EMPTY_STREAMING,
244 - id="empty_streaming",
245 - ),
246 - pytest.param(
247 - False,
248 - NEW_LINE,
249 - id="new_line",
250 - ),
251 - pytest.param(
252 - True,
253 - NEW_LINE_STREAMING,
254 - id="new_line_streaming",
255 - ),
256 -]
257 -
258 -
259 -@pytest.mark.parametrize("streaming, param_dict", TEST_CASES)
260 -def test_mistral_reasoning(
261 - streaming: bool,
262 - param_dict: dict,
263 - mistral_tokenizer: MistralTokenizer,
264 -):
265 - output = param_dict["output"]
266 -
267 - index_think = output.find("[THINK]")
268 - len_think = len("[THINK]")
269 - index_end_think = output.find("[/THINK]")
270 - len_end_think = len("[/THINK]")
271 -
272 - # encode everything to tokens ids
273 - output_tokens = []
274 - if index_think != -1:
275 - output_before_think = output[:index_think]
276 - output_tokens += mistral_tokenizer.tokenizer.encode(
277 - output_before_think, False, False
278 - )
279 - output_tokens += [mistral_tokenizer.instruct.BEGIN_THINK]
280 -
281 - if index_end_think != -1:
282 - output_middle = output[index_think + len_think : index_end_think]
283 - output_after_think = output[index_end_think + len_end_think :]
284 - output_tokens += mistral_tokenizer.tokenizer.encode(
285 - output_middle, False, False
286 - )
287 - output_tokens += [mistral_tokenizer.instruct.END_THINK]
288 - output_tokens += mistral_tokenizer.tokenizer.encode(
289 - output_after_think, False, False
290 - )
291 - else:
292 - output_middle = output[index_think + len_think :]
293 - output_tokens += mistral_tokenizer.tokenizer.encode(
294 - output_middle, False, False
295 - )
296 - elif index_end_think != -1:
297 - output_before_think = output[:index_end_think]
298 - output_after_think = output[index_end_think + len_end_think :]
299 - output_tokens += mistral_tokenizer.tokenizer.encode(
300 - output_before_think, False, False
301 - )
302 - output_tokens += [mistral_tokenizer.instruct.END_THINK]
303 - output_tokens += mistral_tokenizer.tokenizer.encode(
304 - output_after_think, False, False
305 - )
306 - else:
307 - output_tokens += mistral_tokenizer.tokenizer.encode(output, False, False)
308 -
309 - parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(parser_name)(
310 - mistral_tokenizer
311 - )
312 -
313 - reasoning, content = run_reasoning_extraction_mistral(
314 - parser, output_tokens, streaming=streaming
315 - )
316 -
317 - assert reasoning == param_dict["reasoning"]
318 - assert content == param_dict["content"]
319 -
320 - # Test is_reasoning_end
321 - is_reasoning_end = parser.is_reasoning_end(output_tokens)
322 - assert is_reasoning_end == param_dict["is_reasoning_end"]
323 -
324 - # Test extract_content
325 - if param_dict["content"] is not None:
326 - # Handle the case where there are tokens outputted before Thinking.
327 - # This should not occur if the model is well trained and prompted.
328 - if "[THINK]" in param_dict["output"] and not param_dict["output"].startswith(
329 - "[THINK]"
330 - ):
331 - before_content = param_dict["output"].split("[THINK]")[0]
332 - before_token_ids = mistral_tokenizer.tokenizer.encode(
333 - before_content, bos=False, eos=False
334 - )
335 - left_to_encode = param_dict["content"][len(before_content) :]
336 - # Normal situation.
337 - else:
338 - before_token_ids = []
339 - left_to_encode = param_dict["content"]
340 -
341 - content_tokens = parser.extract_content_ids(output_tokens)
342 - expected_token_ids = before_token_ids + mistral_tokenizer.tokenizer.encode(
343 - left_to_encode, bos=False, eos=False
344 - )
345 - assert content_tokens == expected_token_ids
346 - else:
347 - content = parser.extract_content_ids(output_tokens)
348 - assert content == []
tests/tokenizers_/test_mistral.py
+144 -2
@@ -8,11 +8,13 @@ import llguidance
8 import pytest
9 from mistral_common.exceptions import InvalidMessageStructureException
10 from mistral_common.guidance.grammar_factory import GrammarFactory
11 -from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy
11 +from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy, SpecialTokens
12
13 +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
14 from vllm.tokenizers.mistral import (
15 MistralTokenizer,
16 _validate_apply_chat_template_args,
17 + validate_request_params,
18 )
19
20
@@ -1247,7 +1249,9 @@ class TestMistralTokenizer:
1249
1250 expected_strings = (
1251 '[{"type": "function", "function": {"name": "get_weather", "description": "Gets the current weather in a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city name"}}, "required": ["city"]}}}] I am an AI\n\nHello world ![TOOL_CALLS][{"name": "get_weather", "arguments": {"city": "Paris"}, "id": "123456789"}] {"content": {"temperature": 20, "unit": "celsius"}, "call_id": "123456789"}', # noqa: E501
1250 - 'I am an AI[{"type": "function", "function": {"name": "get_weather", "description": "Gets the current weather in a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city name"}}, "required": ["city"]}}}]Hello world ![TOOL_CALLS]get_weather{"city": "Paris"}{"temperature": 20, "unit": "celsius"}', # noqa: E501
1252 + # v11+ tool-call decode emits the explicit [ARGS] separator between
1253 + # the function name and its JSON arguments (get_weather[ARGS]{...}).
1254 + 'I am an AI[{"type": "function", "function": {"name": "get_weather", "description": "Gets the current weather in a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city name"}}, "required": ["city"]}}}]Hello world ![TOOL_CALLS]get_weather[ARGS]{"city": "Paris"}{"temperature": 20, "unit": "celsius"}', # noqa: E501
1255 )
1256
1257 assert (
@@ -1498,6 +1502,7 @@ class TestMistralTokenizer:
1502 "get",
1503 "_",
1504 "weather",
1505 + "[ARGS]",
1506 '{"',
1507 "city",
1508 '":',
@@ -2234,3 +2239,140 @@ class TestMistralTokenizer:
2239 decoded = mistral_tokenizer.tokenizer.decode(output, SpecialTokenPolicy.KEEP)
2240
2241 assert "[THINK]2+2 equals 4[/THINK]" in decoded
2242 +
2243 +
2244 +def test_convert_ids_to_tokens_pre_args_tekken():
2245 + """convert_ids_to_tokens must not raise on pre-[ARGS] Tekken tokenizers.
2246 +
2247 + Tekken tokenizers before v11 (e.g. Ministral-8B-Instruct-2410) have no
2248 + [ARGS] special token. This guards the is_special([ARGS]) gate in
2249 + MistralTokenizer.convert_ids_to_tokens, which must skip [ARGS] rather than
2250 + call get_special_token(args) unconditionally (which raises on Tekken).
2251 + """
2252 + tokenizer = MistralTokenizer.from_pretrained("mistralai/Ministral-8B-Instruct-2410")
2253 + assert tokenizer.is_tekken
2254 + assert not tokenizer.tokenizer.is_special(SpecialTokens.args)
2255 +
2256 + ids = tokenizer.encode("Hello world !", add_special_tokens=False)
2257 + tokens = tokenizer.convert_ids_to_tokens(ids, skip_special_tokens=True)
2258 + assert len(tokens) > 0
2259 +
2260 +
2261 +# ---------------------------------------------------------------------------
2262 +# validate_request_params – reasoning_effort validation (no tokenizer needed)
2263 +# ---------------------------------------------------------------------------
2264 +
2265 +
2266 +@pytest.mark.parametrize("reasoning_effort", [None, "none", "high"])
2267 +def test_validate_request_params_valid_effort(reasoning_effort: str | None) -> None:
2268 + request = ChatCompletionRequest(
2269 + model="test-model",
2270 + messages=[],
2271 + reasoning_effort=reasoning_effort,
2272 + )
2273 + assert validate_request_params(request) is None
2274 +
2275 +
2276 +def test_validate_request_params_rejects_unsupported_effort() -> None:
2277 + request = ChatCompletionRequest(
2278 + model="test-model",
2279 + messages=[],
2280 + reasoning_effort="medium",
2281 + )
2282 + with pytest.raises(ValueError, match="reasoning_effort"):
2283 + validate_request_params(request)
2284 +
2285 +
2286 +# ---------------------------------------------------------------------------
2287 +# apply_chat_template reasoning_effort passthrough (v15 tokenizer)
2288 +# ---------------------------------------------------------------------------
2289 +
2290 +_PASSTHROUGH_MESSAGES = [{"role": "user", "content": "Hello"}]
2291 +
2292 +
2293 +@pytest.fixture(scope="module")
2294 +def v15_mistral_tokenizer() -> MistralTokenizer:
2295 + """Load the v15 Mistral tokenizer; skip if unavailable."""
2296 + try:
2297 + return MistralTokenizer.from_pretrained("mistralai/Mistral-Small-4-119B-2603")
2298 + except Exception:
2299 + pytest.skip("v15 tokenizer unavailable")
2300 +
2301 +
2302 +@pytest.fixture(scope="module")
2303 +def v13_mistral_tokenizer() -> MistralTokenizer:
2304 + """Load the v13 Mistral tokenizer; skip if unavailable."""
2305 + try:
2306 + return MistralTokenizer.from_pretrained("mistralai/Magistral-Small-2509")
2307 + except Exception:
2308 + pytest.skip("v13 tokenizer unavailable")
2309 +
2310 +
2311 +def test_v15_apply_chat_template_passes_reasoning_effort_high(
2312 + monkeypatch: pytest.MonkeyPatch,
2313 + v15_mistral_tokenizer: MistralTokenizer,
2314 +) -> None:
2315 + captured_kwargs: list[dict] = []
2316 +
2317 + def fake_apply_chat_template(**kwargs):
2318 + captured_kwargs.append(kwargs)
2319 + return [0]
2320 +
2321 + monkeypatch.setattr(
2322 + v15_mistral_tokenizer.transformers_tokenizer,
2323 + "apply_chat_template",
2324 + fake_apply_chat_template,
2325 + )
2326 +
2327 + v15_mistral_tokenizer.apply_chat_template(
2328 + messages=_PASSTHROUGH_MESSAGES,
2329 + reasoning_effort="high",
2330 + )
2331 +
2332 + assert captured_kwargs[-1]["reasoning_effort"] == "high"
2333 +
2334 +
2335 +def test_v15_apply_chat_template_passes_reasoning_effort_none_by_default(
2336 + monkeypatch: pytest.MonkeyPatch,
2337 + v15_mistral_tokenizer: MistralTokenizer,
2338 +) -> None:
2339 + captured_kwargs: list[dict] = []
2340 +
2341 + def fake_apply_chat_template(**kwargs):
2342 + captured_kwargs.append(kwargs)
2343 + return [0]
2344 +
2345 + monkeypatch.setattr(
2346 + v15_mistral_tokenizer.transformers_tokenizer,
2347 + "apply_chat_template",
2348 + fake_apply_chat_template,
2349 + )
2350 +
2351 + v15_mistral_tokenizer.apply_chat_template(messages=_PASSTHROUGH_MESSAGES)
2352 +
2353 + assert "reasoning_effort" in captured_kwargs[-1]
2354 + assert captured_kwargs[-1]["reasoning_effort"] is None
2355 +
2356 +
2357 +def test_pre_v15_apply_chat_template_omits_reasoning_effort(
2358 + monkeypatch: pytest.MonkeyPatch,
2359 + v13_mistral_tokenizer: MistralTokenizer,
2360 +) -> None:
2361 + captured_kwargs: list[dict] = []
2362 +
2363 + def fake_apply_chat_template(**kwargs):
2364 + captured_kwargs.append(kwargs)
2365 + return [0]
2366 +
2367 + monkeypatch.setattr(
2368 + v13_mistral_tokenizer.transformers_tokenizer,
2369 + "apply_chat_template",
2370 + fake_apply_chat_template,
2371 + )
2372 +
2373 + v13_mistral_tokenizer.apply_chat_template(
2374 + messages=_PASSTHROUGH_MESSAGES,
2375 + reasoning_effort="high",
2376 + )
2377 +
2378 + assert "reasoning_effort" not in captured_kwargs[-1]
tests/tool_use/mistral/test_mistral_tool_calls.py
+1 -1
@@ -219,7 +219,7 @@ _NOT_SET = object()
219 pytest.param(
220 [WEATHER_TOOL],
221 "required",
222 - 30,
222 + 9,
223 id="required",
224 ),
225 ],
vllm/entrypoints/openai/chat_completion/protocol.py
+2 -2
@@ -533,8 +533,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
533 msg["tool_calls"] = list(tool_calls)
534 return self
535
536 - _grammar_from_tool_parser: bool = PrivateAttr(default=False)
537 - """CAUTION: Should only be set by ``ToolParser.adjust_request``."""
536 + _grammar_from_parser: bool = PrivateAttr(default=False)
537 + """CAUTION: Should only be set by the parser-engine adapter's adjust_request."""
538
539 def build_chat_params(
540 self,
vllm/entrypoints/openai/chat_completion/serving.py
+1 -11
@@ -65,7 +65,6 @@ from vllm.renderers.online_renderer import OnlineRenderer
65 from vllm.sampling_params import BeamSearchParams, SamplingParams
66 from vllm.tokenizers import TokenizerLike
67 from vllm.utils.collection_utils import as_list
68 -from vllm.utils.mistral import is_mistral_tool_parser
68
69 logger = init_logger(__name__)
70
@@ -157,15 +156,6 @@ class OpenAIServingChat(GenerateBaseServing):
156 model_name=self.model_config.model,
157 is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
158 )
160 - if (
161 - self.parser_cls is not None
162 - and is_mistral_tool_parser(self.parser_cls.tool_parser_cls)
163 - and self.parser_cls.reasoning_parser_cls is not None
164 - ):
165 - from vllm.tool_parsers.mistral_tool_parser import MistralToolParser
166 -
167 - MistralToolParser.model_can_reason = True
168 -
159 self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
160
161 self.enable_prompt_tokens_details = enable_prompt_tokens_details
@@ -350,7 +340,7 @@ class OpenAIServingChat(GenerateBaseServing):
340 else:
341 if not request.include_reasoning:
342 reasoning_ended = True
353 - elif request._grammar_from_tool_parser:
343 + elif request._grammar_from_parser:
344 # The Mistral grammar already includes an optional
345 # `think?` rule that handles both reasoning and
346 # non-reasoning outputs.
vllm/entrypoints/scale_out/token_in_token_out/protocol.py
+1 -1
@@ -284,7 +284,7 @@ class DerenderChatRequest(BaseModel):
284
285 Required by the parsing so that tool/reasoning parsers can receive the full
286 request context they expect (request.tools, request.tool_choice,
287 - request._grammar_from_tool_parser, etc.).
287 + request._grammar_from_parser, etc.).
288 """
289 # --8<-- [end:derender-chat-request]
290
vllm/parser/engine/parser_engine.py
+7 -4
@@ -396,6 +396,9 @@ class ParserEngine(Parser):
396 return True
397 return find_tool_name(self._tools, name)
398
399 + def _accept_tool_name(self, name: str) -> bool:
400 + return bool(name) and self._is_valid_tool_name(name)
401 +
402 # ── Private helpers ─────────────────────────────────────────────
403
404 def _check_skip_tool_parsing(
@@ -807,7 +810,7 @@ class ParserEngine(Parser):
810 deltas: list[DeltaToolCall],
811 name: str | None,
812 ) -> None:
810 - if not name or not self._is_valid_tool_name(name):
813 + if name is None or not self._accept_tool_name(name):
814 return
815 slot = self._tool_slots[idx]
816 slot.name = name
@@ -866,8 +869,8 @@ class ParserEngine(Parser):
869 slot = self._tool_slots[idx]
870
871 if not slot.name_sent:
869 - name = slot.name or self._try_extract_name(idx)
870 - if name and self._is_valid_tool_name(name):
872 + name = slot.name or self._try_extract_name(idx) or ""
873 + if self._accept_tool_name(name):
874 slot.name = name
875 slot.name_sent = True
876 slot.string_keys = self._streamable_string_keys(
@@ -1042,7 +1045,7 @@ class ParserEngine(Parser):
1045 else:
1046 args_json = "{}"
1047
1045 - if name and self._is_valid_tool_name(name):
1048 + if self._accept_tool_name(name):
1049 self._ensure_tool_id(slot, name)
1050 args_json = self._fix_arg_types(args_json, name)
1051 tool_calls.append(
vllm/parser/engine/registered_adapters.py
+6
@@ -15,6 +15,7 @@ from vllm.parser.glm47_moe import Glm47MoeParser
15 from vllm.parser.inkling import InklingParser
16 from vllm.parser.kimi_k2 import KimiK2Parser
17 from vllm.parser.minimax_m2 import MinimaxM2Parser
18 +from vllm.parser.mistral import MistralParser
19 from vllm.parser.nemotron_v3 import NemotronV3Parser
20 from vllm.parser.qwen3 import Qwen3Parser
21 from vllm.parser.seed_oss import SeedOssParser
@@ -68,3 +69,8 @@ from vllm.parser.seed_oss import SeedOssParser
69 InklingParserReasoningAdapter,
70 InklingParserToolAdapter,
71 ) = make_adapters(InklingParser)
72 +
73 +(
74 + MistralParserReasoningAdapter,
75 + MistralParserToolAdapter,
76 +) = make_adapters(MistralParser)
vllm/parser/mistral.py
+946 -54
@@ -3,80 +3,972 @@
3
4 from __future__ import annotations
5
6 +import functools
7 +import json
8 from collections.abc import Sequence
7 -from typing import TYPE_CHECKING
9 +from enum import Enum, auto
10 +from random import choices
11 +from string import ascii_letters, digits
12 +from typing import TYPE_CHECKING, Any, Literal
13
9 -from vllm.entrypoints.openai.engine.protocol import DeltaMessage, FunctionCall
10 -from vllm.parser.abstract_parser import DelegatingParser
14 +import ijson
15 +import regex as re
16 +from mistral_common.protocol.instruct.tool_calls import (
17 + NamedToolChoice as MistralNamedToolChoice,
18 +)
19 +from mistral_common.protocol.instruct.tool_calls import (
20 + Tool as MistralTool,
21 +)
22 +from mistral_common.protocol.instruct.tool_calls import (
23 + ToolChoice as MistralToolChoice,
24 +)
25 +from mistral_common.protocol.instruct.tool_calls import (
26 + ToolChoiceEnum as MistralToolChoiceEnum,
27 +)
28 +from mistral_common.tokens.tokenizers.base import SpecialTokens
29 +from pydantic import Field
30 +
31 +from vllm.entrypoints.openai.chat_completion.protocol import (
32 + ChatCompletionNamedToolChoiceParam,
33 +)
34 +from vllm.entrypoints.openai.engine.protocol import (
35 + DeltaFunctionCall,
36 + DeltaMessage,
37 + DeltaToolCall,
38 + ExtractedToolCallInformation,
39 + FunctionCall,
40 + ToolCall,
41 +)
42 +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
43 +from vllm.logger import init_logger
44 +from vllm.parser.engine.events import EventType
45 +from vllm.parser.engine.parser_engine import ParserEngine
46 +from vllm.parser.engine.parser_engine_config import (
47 + ParserEngineConfig,
48 + ParserState,
49 + Transition,
50 +)
51 +from vllm.sampling_params import StructuredOutputsParams
52 +from vllm.utils.mistral import is_mistral_tokenizer
53
54 if TYPE_CHECKING:
55 from vllm.entrypoints.openai.chat_completion.protocol import (
56 ChatCompletionRequest,
57 )
16 - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
58 + from vllm.parser.engine.events import SemanticEvent
59 + from vllm.parser.engine.parser_engine import ToolCallSlot
60 + from vllm.tokenizers import TokenizerLike
61 + from vllm.tool_parsers.abstract_tool_parser import Tool
62 +
63 +logger = init_logger(__name__)
64 +
65 +_ALPHANUMERIC = ascii_letters + digits
66 +_DEFAULT_JSON_SCHEMA: dict[str, Any] = {
67 + "anyOf": [{"type": "object"}, {"type": "array"}]
68 +}
69 +
70 +# Special token strings as emitted/decoded by MistralTokenizer.
71 +_TOOL_CALLS = SpecialTokens.tool_calls.value
72 +_ARGS = SpecialTokens.args.value
73 +_THINK_START_SPECIAL = SpecialTokens.begin_think.value
74 +_THINK_END_SPECIAL = SpecialTokens.end_think.value
75 +
76 +# Plain-text reasoning markers used by v11 tokenizers.
77 +_THINK_START_TEXT = "<think>"
78 +_THINK_END_TEXT = "</think>"
79 +
80 +_OPEN_BRACE = "{"
81 +
82 +
83 +class StreamingState(Enum):
84 + """Streaming parsing state for pre-v11 tool call extraction."""
85 +
86 + WAITING_FOR_TOOL_START = auto()
87 + WAITING_FOR_TOOL_KEY = auto()
88 + PARSING_NAME = auto()
89 + PARSING_NAME_COMPLETED = auto()
90 + WAITING_FOR_ARGUMENTS_START = auto()
91 + PARSING_ARGUMENTS = auto()
92 + PARSING_ARGUMENTS_COMPLETED = auto()
93 + TOOL_COMPLETE = auto()
94 + ALL_TOOLS_COMPLETE = auto()
95 +
96 +
97 +class MistralToolCall(ToolCall):
98 + """ToolCall with a Mistral-compatible random alphanumeric id."""
99 +
100 + id: str = Field(default_factory=lambda: MistralToolCall.generate_random_id())
101 +
102 + @staticmethod
103 + def generate_random_id() -> str:
104 + # Mistral Tool Call Ids must be alphanumeric with a length of 9.
105 + # https://github.com/mistralai/mistral-common/blob/21ee9f6cee3441e9bb1e6ed2d10173f90bd9b94b/src/mistral_common/protocol/instruct/validator.py#L299
106 + return "".join(choices(_ALPHANUMERIC, k=9))
107 +
108 + @staticmethod
109 + def is_valid_id(id: str) -> bool:
110 + return id.isalnum() and len(id) == 9
111 +
112 +
113 +def _is_pre_v11_tokeniser(model_tokenizer: TokenizerLike) -> bool:
114 + if is_mistral_tokenizer(model_tokenizer):
115 + return model_tokenizer.version < 11
116 + vocab: dict[str, int] = getattr(model_tokenizer, "get_vocab", lambda: {})()
117 + return _ARGS not in vocab
118 +
119 +
120 +@functools.cache
121 +def mistral_config(
122 + *,
123 + reasoning_encoding: Literal["special_token", "text", "none"],
124 + name: str = "mistral",
125 +) -> ParserEngineConfig:
126 + """Return a :class:`ParserEngineConfig` for the Mistral output format.
127 +
128 + Args:
129 + reasoning_encoding: Reasoning token format.
130 + ``"special_token"`` – v13+ ``[THINK]``/``[/THINK]`` special
131 + tokens placed in both ``terminals`` and ``token_id_terminals``.
132 + ``"text"`` – v11 ``<think>``/``</think>`` plain text placed in
133 + ``terminals`` only; no ``token_id_terminals`` for think tokens.
134 + ``"none"`` – no reasoning support.
135 + name: Name embedded in the returned config (used for debugging).
136 +
137 + Returns:
138 + A frozen :class:`ParserEngineConfig` with ``initial_state=CONTENT``.
139 + """
140 + if reasoning_encoding == "special_token":
141 + think_start = _THINK_START_SPECIAL
142 + think_end = _THINK_END_SPECIAL
143 + reasoning_terminals: dict[str, str] = {
144 + "THINK_START": think_start,
145 + "THINK_END": think_end,
146 + }
147 + reasoning_token_id_terminals: dict[str, str] = {
148 + "THINK_START": think_start,
149 + "THINK_END": think_end,
150 + }
151 + elif reasoning_encoding == "text":
152 + think_start = _THINK_START_TEXT
153 + think_end = _THINK_END_TEXT
154 + reasoning_terminals = {
155 + "THINK_START": think_start,
156 + "THINK_END": think_end,
157 + }
158 + # Text think markers have no token-id terminals.
159 + reasoning_token_id_terminals = {}
160 + else:
161 + reasoning_terminals = {}
162 + reasoning_token_id_terminals = {}
163 +
164 + if reasoning_encoding != "none":
165 + reasoning_transitions: dict[tuple[ParserState, str], Transition] = {
166 + (ParserState.CONTENT, "THINK_START"): Transition(
167 + ParserState.REASONING,
168 + (),
169 + ),
170 + # Absorb a duplicate/re-emitted THINK_START inside reasoning.
171 + (ParserState.REASONING, "THINK_START"): Transition(
172 + ParserState.REASONING,
173 + (),
174 + ),
175 + (ParserState.REASONING, "THINK_END"): Transition(
176 + ParserState.CONTENT,
177 + (EventType.REASONING_END,),
178 + ),
179 + # Absorb stray THINK_END that arrives after reasoning ended.
180 + (ParserState.CONTENT, "THINK_END"): Transition(
181 + ParserState.CONTENT,
182 + (),
183 + ),
184 + # [TOOL_CALLS] directly from reasoning implicitly ends it.
185 + (ParserState.REASONING, "TOOL_CALLS"): Transition(
186 + ParserState.TOOL_NAME,
187 + (EventType.REASONING_END, EventType.TOOL_CALL_START),
188 + ),
189 + }
190 + else:
191 + reasoning_transitions = {}
192 +
193 + return ParserEngineConfig(
194 + name=name,
195 + initial_state=ParserState.CONTENT,
196 + terminals={
197 + **reasoning_terminals,
198 + "TOOL_CALLS": _TOOL_CALLS,
199 + "ARGS": _ARGS,
200 + "OPEN_BRACE": _OPEN_BRACE,
201 + },
202 + token_id_terminals={
203 + **reasoning_token_id_terminals,
204 + "TOOL_CALLS": _TOOL_CALLS,
205 + "ARGS": _ARGS,
206 + },
207 + transitions={
208 + **reasoning_transitions,
209 + # A tool call from content implicitly ends reasoning when enabled.
210 + (ParserState.CONTENT, "TOOL_CALLS"): Transition(
211 + ParserState.TOOL_NAME,
212 + (EventType.REASONING_END, EventType.TOOL_CALL_START)
213 + if reasoning_encoding != "none"
214 + else (EventType.TOOL_CALL_START,),
215 + ),
216 + # NAME→ARGS via explicit [ARGS] separator (v11+): consumed, no events.
217 + (ParserState.TOOL_NAME, "ARGS"): Transition(
218 + ParserState.TOOL_ARGS,
219 + (),
220 + ),
221 + # NAME→ARGS via "{": carried as ARG_VALUE_CHUNK so the opening
222 + # brace lands in the JSON argument buffer (fallback for name{args}).
223 + (ParserState.TOOL_NAME, "OPEN_BRACE"): Transition(
224 + ParserState.TOOL_ARGS,
225 + (EventType.ARG_VALUE_CHUNK,),
226 + ),
227 + # Parallel tool calls: next [TOOL_CALLS] ends current call.
228 + (ParserState.TOOL_ARGS, "TOOL_CALLS"): Transition(
229 + ParserState.TOOL_NAME,
230 + (EventType.TOOL_CALL_END, EventType.TOOL_CALL_START),
231 + ),
232 + },
233 + stream_arg_deltas=True,
234 + tool_args_json=True,
235 + strip_trailing_reasoning_whitespace=True,
236 + drop_whitespace_only_content_before_tools=True,
237 + )
238 +
239 +
240 +class MistralParser(ParserEngine):
241 + """Mistral parser: engine-based reasoning + ``[TOOL_CALLS]`` tool calls.
242
243 + Reasoning encoding is auto-detected from the tokenizer:
244
19 -class MistralParser(DelegatingParser):
20 - def __init__(self, tokenizer, tools=None, *args, **kwargs):
21 - super().__init__(tokenizer, tools, *args, **kwargs)
22 - from vllm.tool_parsers.mistral_tool_parser import MistralToolParser
245 + - ``"special_token"`` – ``[THINK]`` present in vocab (v13+).
246 + - ``"text"`` – tokenizer supports grammar but has no ``[THINK]`` (v11).
247 + - ``"none"`` – no grammar support; reasoning disabled.
248
24 - if not isinstance(self._tool_parser, MistralToolParser):
25 - raise ValueError(
26 - "MistralParser requires --tool-call-parser mistral, "
27 - f"got {self._tool_parser.__class__.__name__}."
249 + Tool calls use the ``[TOOL_CALLS]func_name{...}`` format. The opening
250 + ``{`` doubles as the NAME→ARGS separator and is included in the argument
251 + buffer via an ``ARG_VALUE_CHUNK`` event on the transition (mirrors
252 + kimi_k2, JSON args, `tool_args_json=True`).
253 +
254 + When the tokenizer does not support grammar (``_reasoning_encoding ==
255 + "none"``), the legacy ``[TOOL_CALLS]``-based extraction path is used
256 + instead of the declarative engine, handling both pre-v11 JSON-array and
257 + v11+ ``funcname{args}`` formats.
258 + """
259 +
260 + def __init__(
261 + self,
262 + tokenizer: TokenizerLike,
263 + tools: list[Tool] | None = None,
264 + **kwargs,
265 + ) -> None:
266 + vocab = tokenizer.get_vocab()
267 + self._reasoning_encoding: Literal["special_token", "text", "none"]
268 + if _THINK_START_SPECIAL in vocab:
269 + self._reasoning_encoding = "special_token"
270 + elif getattr(tokenizer, "supports_grammar", False):
271 + self._reasoning_encoding = "text"
272 + else:
273 + self._reasoning_encoding = "none"
274 +
275 + kwargs.setdefault(
276 + "parser_engine_config",
277 + mistral_config(reasoning_encoding=self._reasoning_encoding),
278 + )
279 + super().__init__(tokenizer, tools, **kwargs)
280 +
281 + self._tool_calls_token_id: int | None = self.vocab.get(_TOOL_CALLS)
282 +
283 + # Tool calls use the legacy parser for all tokenizer versions;
284 + # reasoning is handled by the engine.
285 + self.bot_token: str = _TOOL_CALLS
286 + self.bot_token_id: int | None = self._tool_calls_token_id
287 + if self.bot_token_id is None:
288 + raise RuntimeError(
289 + "Mistral parser could not locate the tool call token in the tokenizer!"
290 + )
291 +
292 + # Legacy tool-call streaming state.
293 + self.prev_tool_call_arr: list[dict[str, Any]] = []
294 + self.current_tool_id: int = -1
295 + self.streaming_state: StreamingState = StreamingState.WAITING_FOR_TOOL_START
296 + self.tool_call_started: bool = False
297 + self.current_tool_name: str | None = None
298 + self.current_tool_mistral_id: str | None = None
299 + self.starting_new_tool: bool = False
300 + self.streamed_args_for_tool: list[str] = []
301 + self._is_pre_v11: bool = _is_pre_v11_tokeniser(tokenizer)
302 + self.parse_coro = None
303 + if self._is_pre_v11:
304 + self.parse_coro = ijson.parse_coro(
305 + self.update_stream_state_pre_v11_tokenizer()
306 )
307 + self.tool_call_regex = re.compile(r"\[{.*}\]", re.DOTALL)
308
30 - def _maybe_force_auto_tool_parsing(
309 + def adjust_request(
310 self, request: ChatCompletionRequest | ResponsesRequest
32 - ) -> None:
33 - # When the Mistral grammar factory injected structured outputs,
34 - # the model emits v11+ format ([TOOL_CALLS]name{args}) that the
35 - # named/required parsers can't handle. Disable them so all
36 - # tool_choice modes fall back to auto tool parsing via
37 - # extract_tool_calls.
38 - if getattr(request, "_grammar_from_tool_parser", False):
39 - assert self._tool_parser is not None
40 - self._tool_parser.supports_required_and_named = False
41 -
42 - def parse(
311 + ) -> ChatCompletionRequest | ResponsesRequest:
312 + if not isinstance(request, ResponsesRequest) and request._grammar_from_parser:
313 + return request
314 + so_non_supported_attributes = [
315 + "regex",
316 + "choice",
317 + "grammar",
318 + # whitespace_pattern is not a constraint type but an option;
319 + # Mistral grammar factory does not support it.
320 + "whitespace_pattern",
321 + "structural_tag",
322 + ]
323 + any_so_non_supported_active = request.structured_outputs is not None and any(
324 + getattr(request.structured_outputs, attribute) is not None
325 + for attribute in so_non_supported_attributes
326 + )
327 + response_format_non_supported_active = (
328 + isinstance(request, ResponsesRequest)
329 + or request.response_format is not None
330 + and request.response_format.type == "structural_tag"
331 + )
332 +
333 + if (
334 + not is_mistral_tokenizer(self.model_tokenizer)
335 + or isinstance(request, ResponsesRequest)
336 + or not self.model_tokenizer.supports_grammar
337 + or any_so_non_supported_active
338 + or response_format_non_supported_active
339 + ):
340 + request = super().adjust_request(request)
341 + if request.tools and request.tool_choice != "none":
342 + # Keep special tokens so the [TOOL_CALLS] marker
343 + # survives for tool detection.
344 + request.skip_special_tokens = False
345 + # Inject a guided JSON schema for pre-v11 required/named tool choice
346 + # so the model emits a well-formed bare JSON array instead of
347 + # rambling. tool_choice forces a tool call, so a json_object /
348 + # json_schema response_format is cleared here (the tool schema is
349 + # the sole structured-output constraint), mirroring the base
350 + # ToolParser; a structural_tag response_format is left untouched.
351 + req_tool_choice = request.tool_choice
352 + is_required_or_named = req_tool_choice == "required" or isinstance(
353 + req_tool_choice, ChatCompletionNamedToolChoiceParam
354 + )
355 + response_format = getattr(request, "response_format", None)
356 + response_format_overridable = response_format is None or (
357 + response_format.type in ("text", "json_object", "json_schema")
358 + )
359 + # This runs only in the non-grammar (legacy) branch, so it covers
360 + # both pre-v11 Mistral tokenizers and non-Mistral (e.g. HF-mode)
361 + # tokenizers driving the Mistral tool parser. The remaining guards
362 + # keep it off when the user supplied their own structured output.
363 + if (
364 + not isinstance(request, ResponsesRequest)
365 + and request.tools
366 + and request.structured_outputs is None
367 + and is_required_or_named
368 + and response_format_overridable
369 + ):
370 + schema = self._build_guided_schema_pre_v11(request)
371 + if schema is not None:
372 + request.structured_outputs = StructuredOutputsParams(json=schema)
373 + request.response_format = None
374 + return request
375 +
376 + json_schema: dict[str, Any] | None = None
377 + if request.structured_outputs is not None:
378 + if request.structured_outputs.json_object is not None:
379 + json_schema = _DEFAULT_JSON_SCHEMA
380 + elif request.structured_outputs.json is not None:
381 + if isinstance(request.structured_outputs.json, str):
382 + json_schema = json.loads(request.structured_outputs.json)
383 + else:
384 + json_schema = request.structured_outputs.json
385 + else:
386 + raise ValueError(
387 + "Unsupported request.structured_outputs for MistralParser. "
388 + "Only `json` and `json_object` are supported."
389 + )
390 + elif (
391 + request.response_format is not None
392 + and request.response_format.type != "text"
393 + ):
394 + if request.response_format.type == "json_object":
395 + json_schema = _DEFAULT_JSON_SCHEMA
396 + elif request.response_format.type == "json_schema":
397 + if request.response_format.json_schema is not None:
398 + json_schema = request.response_format.json_schema.json_schema
399 + else:
400 + json_schema = _DEFAULT_JSON_SCHEMA
401 + else:
402 + raise ValueError(
403 + "MistralParser only accepts `text`, `json_object` or "
404 + f"`json_schema`, got {request.response_format=}"
405 + )
406 + request.response_format = None
407 +
408 + grammar_factory = self.model_tokenizer.grammar_factory
409 +
410 + # Rendering grammar is cached in mistral-common given tools, template and mode.
411 + template = grammar_factory.select_jinja_template()
412 +
413 + mistral_tools = (
414 + [MistralTool.from_openai(tool.model_dump()) for tool in request.tools]
415 + if request.tools is not None
416 + else None
417 + )
418 +
419 + tool_choice: MistralToolChoice
420 + match request.tool_choice:
421 + case "none" | "auto" | "required":
422 + tool_choice = MistralToolChoiceEnum(request.tool_choice)
423 + case None:
424 + tool_choice = MistralToolChoiceEnum.auto
425 + # _ == Named tool choice
426 + case _:
427 + tool_choice = MistralNamedToolChoice.model_validate(
428 + {
429 + "type": "function",
430 + "function": {"name": request.tool_choice.function.name},
431 + }
432 + )
433 +
434 + match tool_choice, json_schema is not None:
435 + case MistralToolChoiceEnum.none, True:
436 + lark_grammar = grammar_factory.get_lark_for_json_schema(
437 + template=template, json_schema=json_schema
438 + )
439 + case _, _:
440 + lark_grammar = grammar_factory.get_lark_from_jinja(
441 + template=template,
442 + mode=tool_choice,
443 + tools=mistral_tools,
444 + json_schema=json_schema,
445 + parallel_tool_calls=request.parallel_tool_calls,
446 + json_only=False,
447 + )
448 +
449 + request.structured_outputs = StructuredOutputsParams(grammar=lark_grammar)
450 + request._grammar_from_parser = True
451 + return request
452 +
453 + def _build_guided_schema_pre_v11(
454 + self,
455 + request: ChatCompletionRequest,
456 + ) -> dict[str, Any] | None:
457 + """Build a guided JSON schema for pre-v11 required/named tool choice.
458 +
459 + The schema enforces the Mistral-native array format
460 + ``[{"name": ..., "arguments": {...}}]`` so the model emits a parseable
461 + bare JSON array instead of free-form text.
462 +
463 + Args:
464 + request: The chat completion request carrying `tools` and
465 + `tool_choice`.
466 +
467 + Returns:
468 + A JSON Schema dict, or ``None`` if the named tool is not found.
469 + """
470 + tool_choice = request.tool_choice
471 + tools = request.tools or []
472 + extra: dict[str, Any] = {}
473 +
474 + if tool_choice == "required":
475 + applicable_tools = tools
476 + else:
477 + # Named tool choice — restrict to the single requested tool.
478 + assert isinstance(tool_choice, ChatCompletionNamedToolChoiceParam)
479 + chosen_name = tool_choice.function.name
480 + applicable_tools = [t for t in tools if t.function.name == chosen_name]
481 + if not applicable_tools:
482 + logger.warning(
483 + "Named tool %r not found in tools list; "
484 + "skipping guided schema injection.",
485 + chosen_name,
486 + )
487 + return None
488 + extra["maxItems"] = 1
489 +
490 + any_of = [
491 + {
492 + "type": "object",
493 + "properties": {
494 + "name": {"type": "string", "enum": [tool.function.name]},
495 + "arguments": tool.function.parameters or {"type": "object"},
496 + },
497 + "required": ["name", "arguments"],
498 + }
499 + for tool in applicable_tools
500 + ]
501 +
502 + return {
503 + "type": "array",
504 + "minItems": 1,
505 + **extra,
506 + "items": {
507 + "type": "object",
508 + "anyOf": any_of,
509 + },
510 + }
511 +
512 + def _ensure_tool_id(self, slot: ToolCallSlot, name: str) -> None:
513 + """Assign a Mistral-compatible 9-char alphanumeric id to `slot`."""
514 + if not slot.id:
515 + slot.id = MistralToolCall.generate_random_id()
516 +
517 + def extract_tool_calls_from_content(
518 + self,
519 + content: str,
520 + request: ChatCompletionRequest,
521 + ) -> ExtractedToolCallInformation:
522 + if self._is_pre_v11:
523 + return self._legacy_extract_tool_calls(content, request)
524 + return super().extract_tool_calls_from_content(content, request)
525 +
526 + def _legacy_extract_tool_calls(
527 self,
528 model_output: str,
529 + request: ChatCompletionRequest,
530 + ) -> ExtractedToolCallInformation:
531 + """Pre-v11 non-streaming extraction.
532 +
533 + Handles ``[TOOL_CALLS][{...}]`` and guided bare-array formats.
534 + """
535 + # tool_choice="none" with tools: never produce tool calls.
536 + if request.tool_choice == "none" and request.tools:
537 + return ExtractedToolCallInformation(
538 + tools_called=False, tool_calls=[], content=model_output
539 + )
540 +
541 + content: str | None = None
542 + if self.bot_token in model_output:
543 + content_and_raw_tool_calls = model_output.split(self.bot_token)
544 + content = content_and_raw_tool_calls[0]
545 + raw_tool_calls = content_and_raw_tool_calls[1:]
546 + # pre-v11: content[BOT] [{tool_call1},{tool_call2}]
547 + if len(raw_tool_calls) != 1:
548 + raise ValueError(
549 + "Only one BOT token should have been outputted, "
550 + f"but got {model_output}."
551 + )
552 + stringified_tool_calls = raw_tool_calls[0].strip()
553 + elif request.tool_choice == "required" or isinstance(
554 + request.tool_choice, ChatCompletionNamedToolChoiceParam
555 + ):
556 + # Guided bare-array output (no [TOOL_CALLS] marker).
557 + stringified_tool_calls = model_output.strip()
558 + else:
559 + return ExtractedToolCallInformation(
560 + tools_called=False, tool_calls=[], content=model_output
561 + )
562 +
563 + try:
564 + # Use raw_decode to parse the first valid JSON value,
565 + # ignoring trailing tokens the model may emit after
566 + # the tool call array.
567 + tool_calls, _ = json.JSONDecoder().raw_decode(stringified_tool_calls)
568 + except json.JSONDecodeError:
569 + try:
570 + raw_tool_call = self.tool_call_regex.findall(stringified_tool_calls)[0]
571 + tool_calls = json.loads(raw_tool_call)
572 + tool_calls = [
573 + {
574 + "name": tool_call["name"],
575 + "arguments": json.dumps(
576 + tool_call.get("arguments", {}),
577 + ensure_ascii=False,
578 + ),
579 + }
580 + for tool_call in tool_calls
581 + ]
582 + except (IndexError, json.JSONDecodeError):
583 + logger.exception("Error in extracting tool call from response.")
584 + return ExtractedToolCallInformation(
585 + tools_called=False,
586 + tool_calls=[],
587 + content=stringified_tool_calls,
588 + )
589 + else:
590 + tool_calls = [
591 + {
592 + "name": tool_call["name"],
593 + "arguments": json.dumps(
594 + tool_call.get("arguments", {}),
595 + ensure_ascii=False,
596 + ),
597 + }
598 + for tool_call in tool_calls
599 + ]
600 +
601 + mistral_tool_calls: list[MistralToolCall] = [
602 + MistralToolCall(
603 + type="function",
604 + function=FunctionCall(
605 + name=tool_call["name"],
606 + arguments=tool_call.get("arguments", "{}"),
607 + ),
608 + )
609 + for tool_call in tool_calls
610 + ]
611 +
612 + return ExtractedToolCallInformation(
613 + tools_called=True,
614 + tool_calls=mistral_tool_calls,
615 + content=content if content and content.strip() else None,
616 + )
617 +
618 + def extract_tool_calls_streaming(
619 + self,
620 + previous_text: str,
621 + current_text: str,
622 + delta_text: str,
623 + previous_token_ids: Sequence[int],
624 + current_token_ids: Sequence[int],
625 + delta_token_ids: Sequence[int],
626 request: ChatCompletionRequest | ResponsesRequest,
46 - enable_auto_tools: bool = False,
47 - model_output_token_ids: Sequence[int] = (),
48 - ) -> tuple[str | None, str | None, list[FunctionCall] | None]:
49 - self._maybe_force_auto_tool_parsing(request)
50 - reasoning, content, tool_calls = super().parse(
51 - model_output,
52 - request,
53 - enable_auto_tools,
54 - model_output_token_ids,
627 + ) -> DeltaMessage | None:
628 + if not self._is_pre_v11:
629 + return super().extract_tool_calls_streaming(
630 + previous_text,
631 + current_text,
632 + delta_text,
633 + previous_token_ids,
634 + current_token_ids,
635 + delta_token_ids,
636 + request,
637 + )
638 + # Pre-v11: latch on [TOOL_CALLS] or on the first content of a guided
639 + # required/named request (bare JSON array, no special token).
640 + if self.bot_token_id in delta_token_ids or self.bot_token in delta_text:
641 + self.tool_call_started = True
642 + elif not self.tool_call_started and delta_text:
643 + is_guided = request.tool_choice == "required" or isinstance(
644 + request.tool_choice, ChatCompletionNamedToolChoiceParam
645 + )
646 + if is_guided:
647 + self.tool_call_started = True
648 + if not self.tool_call_started:
649 + return DeltaMessage(content=delta_text)
650 + try:
651 + return self._extract_tool_calls_streaming_pre_v11_tokenizer(
652 + delta_text=delta_text,
653 + delta_token_ids=delta_token_ids,
654 + )
655 + except Exception:
656 + logger.exception("Error trying to handle streaming tool call.")
657 + return None
658 +
659 + @ijson.coroutine
660 + def update_stream_state_pre_v11_tokenizer(self):
661 + while True:
662 + (prefix, event, value) = yield
663 +
664 + if prefix == "item" and event == "start_map":
665 + self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY
666 + self.starting_new_tool = True
667 + if prefix == "item" and event == "map_key" and value == "name":
668 + self.streaming_state = StreamingState.PARSING_NAME
669 + if prefix == "item.name" and event == "string":
670 + self.current_tool_name = value
671 + self.streaming_state = StreamingState.PARSING_NAME_COMPLETED
672 + if prefix == "item" and event == "map_key" and value == "arguments":
673 + self.streaming_state = StreamingState.WAITING_FOR_ARGUMENTS_START
674 + if prefix == "item.arguments" and event == "start_map":
675 + self.streaming_state = StreamingState.PARSING_ARGUMENTS
676 + if prefix == "item.arguments" and event == "end_map":
677 + self.streaming_state = StreamingState.PARSING_ARGUMENTS_COMPLETED
678 + if prefix == "item" and event == "end_map":
679 + self.streaming_state = StreamingState.TOOL_COMPLETE
680 + if prefix == "" and event == "end_array":
681 + self.streaming_state = StreamingState.ALL_TOOLS_COMPLETE
682 +
683 + def _extract_tool_calls_streaming_pre_v11_tokenizer(
684 + self,
685 + delta_text: str,
686 + delta_token_ids: Sequence[int],
687 + ) -> DeltaMessage | None:
688 + """Extract tool calls for pre-v11 Mistral models.
689 +
690 + Handles ``[TOOL_CALLS][{"name": "add", "arguments":{"a": 3.5}}]``.
691 + """
692 + assert self.parse_coro is not None
693 + content = None
694 + delta_tool_calls: list[DeltaToolCall] = []
695 + current_tool_call: DeltaToolCall = DeltaToolCall(
696 + index=self.current_tool_id, type="function"
697 )
56 - if tool_calls:
57 - from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
698 + current_tool_call_modified = False
699 + if self.bot_token_id in delta_token_ids or self.bot_token in delta_text:
700 + # this is the first tool call
701 + if not delta_text.startswith(self.bot_token):
702 + content = delta_text.split(self.bot_token)[0]
703 + delta_text = "".join(delta_text.split(self.bot_token)[1:])
704 +
705 + # ijson gives no text index per event, so split the delta manually
706 + # to know where each event is emitted from.
707 + while len(delta_text) > 0:
708 + streaming_state_before_parse = self.streaming_state
709 +
710 + if self.streaming_state == StreamingState.WAITING_FOR_TOOL_START:
711 + delta_to_be_parsed, delta_text = self._split_delta(
712 + delta_text=delta_text,
713 + stop_after_opening_curly_braces=1,
714 + )
715 + elif self.streaming_state == StreamingState.WAITING_FOR_TOOL_KEY:
716 + delta_to_be_parsed, delta_text = self._split_delta(
717 + delta_text=delta_text,
718 + stop_after_colon=1,
719 + stop_after_opening_curly_braces=1,
720 + )
721 + elif self.streaming_state == StreamingState.PARSING_NAME:
722 + delta_to_be_parsed, delta_text = self._split_delta(
723 + delta_text=delta_text,
724 + stop_after_comma=1,
725 + stop_after_closing_brackets=1,
726 + )
727 + elif self.streaming_state == StreamingState.WAITING_FOR_ARGUMENTS_START:
728 + delta_to_be_parsed, delta_text = self._split_delta(
729 + delta_text=delta_text,
730 + stop_after_opening_curly_braces=1,
731 + )
732 + elif self.streaming_state == StreamingState.PARSING_ARGUMENTS:
733 + delta_to_be_parsed, delta_text = self._split_delta(
734 + delta_text=delta_text,
735 + stop_after_closing_curly_braces=1,
736 + )
737 + elif self.streaming_state in [
738 + StreamingState.PARSING_ARGUMENTS_COMPLETED,
739 + StreamingState.PARSING_NAME_COMPLETED,
740 + ]:
741 + delta_to_be_parsed, delta_text = self._split_delta(
742 + delta_text=delta_text,
743 + stop_after_closing_curly_braces=1,
744 + stop_after_closing_brackets=1,
745 + )
746 + elif self.streaming_state == StreamingState.TOOL_COMPLETE:
747 + delta_to_be_parsed, delta_text = self._split_delta(
748 + delta_text=delta_text,
749 + stop_after_opening_curly_braces=1,
750 + stop_after_closing_brackets=1,
751 + )
752 + elif self.streaming_state == StreamingState.ALL_TOOLS_COMPLETE:
753 + content = delta_text
754 + delta_text = ""
755 + else:
756 + delta_to_be_parsed = delta_text
757 + delta_text = ""
758 +
759 + if self.streaming_state != StreamingState.ALL_TOOLS_COMPLETE:
760 + self.parse_coro.send(delta_to_be_parsed.encode("utf-8"))
761 +
762 + # start_map is the authoritative new-tool signal and survives
763 + # batched deltas, unlike comparing pre/post streaming states.
764 + if self.starting_new_tool:
765 + self.starting_new_tool = False
766 + if current_tool_call_modified:
767 + if self.current_tool_mistral_id is not None:
768 + current_tool_call.id = self.current_tool_mistral_id
769 + self.current_tool_mistral_id = None
770 + self._track_streamed_args_pre_v11(current_tool_call)
771 + delta_tool_calls.append(current_tool_call)
772 + current_tool_call_modified = False
773 + self.current_tool_id += 1
774 + self.streamed_args_for_tool.append("")
775 + self.prev_tool_call_arr.append({})
776 + self.current_tool_mistral_id = MistralToolCall.generate_random_id()
777 + current_tool_call = DeltaToolCall(
778 + index=self.current_tool_id,
779 + type="function",
780 + )
781 + if current_tool_call.function is None:
782 + current_tool_call.function = DeltaFunctionCall()
783 +
784 + if self.current_tool_name is not None:
785 + current_tool_call_modified = True
786 + current_tool_call.function.name = self.current_tool_name
787 + self.prev_tool_call_arr[self.current_tool_id]["name"] = (
788 + self.current_tool_name
789 + )
790 + self.current_tool_name = None
791 + if self.streaming_state == StreamingState.PARSING_NAME_COMPLETED:
792 + self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY
793 + if self.streaming_state in [
794 + StreamingState.PARSING_ARGUMENTS,
795 + StreamingState.PARSING_ARGUMENTS_COMPLETED,
796 + ]:
797 + if self.streaming_state == StreamingState.PARSING_ARGUMENTS_COMPLETED:
798 + self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY
799 + current_tool_call_modified = True
800 + if current_tool_call.function.arguments is None:
801 + current_tool_call.function.arguments = delta_to_be_parsed
802 + else:
803 + current_tool_call.function.arguments += delta_to_be_parsed
804 + if streaming_state_before_parse != StreamingState.PARSING_ARGUMENTS:
805 + # It's the first chunk of arg. let's lstrip it
806 + current_tool_call.function.arguments = (
807 + current_tool_call.function.arguments.lstrip()
808 + )
809
59 - # Named/required tool_choice builds FunctionCalls without
60 - # ID, backfill with Mistral-format IDs.
61 - for tc in tool_calls:
62 - if not tc.id:
63 - tc.id = MistralToolCall.generate_random_id()
64 - return reasoning, content, tool_calls
810 + if current_tool_call_modified:
811 + if self.current_tool_mistral_id is not None:
812 + current_tool_call.id = self.current_tool_mistral_id
813 + self.current_tool_mistral_id = None
814 + self._track_streamed_args_pre_v11(current_tool_call)
815 + delta_tool_calls.append(current_tool_call)
816
66 - def parse_delta(
817 + if content or len(delta_tool_calls) > 0:
818 + delta_message = DeltaMessage()
819 + if content:
820 + delta_message.content = content
821 + if len(delta_tool_calls) > 0:
822 + delta_message.tool_calls = delta_tool_calls
823 + return delta_message
824 + else:
825 + if self.streaming_state == StreamingState.ALL_TOOLS_COMPLETE:
826 + return DeltaMessage()
827 + else:
828 + return None
829 +
830 + def _track_streamed_args_pre_v11(self, tool_call: DeltaToolCall) -> None:
831 + r"""Accumulate `tool_call` arguments into the streaming state."""
832 + if tool_call.function is not None and tool_call.function.arguments is not None:
833 + self.streamed_args_for_tool[self.current_tool_id] += (
834 + tool_call.function.arguments
835 + )
836 + self.prev_tool_call_arr[self.current_tool_id]["arguments"] = (
837 + self.streamed_args_for_tool[self.current_tool_id]
838 + )
839 +
840 + def _split_delta(
841 self,
842 delta_text: str,
69 - delta_token_ids: list[int],
843 + stop_after_quotes: int = -1,
844 + stop_after_opening_curly_braces: int = -1,
845 + stop_after_closing_curly_braces: int = -1,
846 + stop_after_closing_brackets: int = -1,
847 + stop_after_colon: int = -1,
848 + stop_after_comma: int = -1,
849 + ) -> tuple[str, str]:
850 + delta_to_be_parsed = ""
851 + for i, c in enumerate(delta_text):
852 + if c in ['"', "'"]:
853 + delta_to_be_parsed += c
854 + stop_after_quotes -= 1
855 + if stop_after_quotes == 0:
856 + return (delta_to_be_parsed, delta_text[i + 1 :])
857 + elif c == "{":
858 + delta_to_be_parsed += c
859 + stop_after_opening_curly_braces -= 1
860 + if stop_after_opening_curly_braces == 0:
861 + return (delta_to_be_parsed, delta_text[i + 1 :])
862 + elif c == "}":
863 + delta_to_be_parsed += c
864 + stop_after_closing_curly_braces -= 1
865 + if stop_after_closing_curly_braces == 0:
866 + return (delta_to_be_parsed, delta_text[i + 1 :])
867 + elif c == "]":
868 + delta_to_be_parsed += c
869 + stop_after_closing_brackets -= 1
870 + if stop_after_closing_brackets == 0:
871 + return (delta_to_be_parsed, delta_text[i + 1 :])
872 + elif c == ":":
873 + delta_to_be_parsed += c
874 + stop_after_colon -= 1
875 + if stop_after_colon == 0:
876 + return (delta_to_be_parsed, delta_text[i + 1 :])
877 + elif c == ",":
878 + delta_to_be_parsed += c
879 + stop_after_comma -= 1
880 + if stop_after_comma == 0:
881 + return (delta_to_be_parsed, delta_text[i + 1 :])
882 + else:
883 + delta_to_be_parsed += c
884 +
885 + return (delta_to_be_parsed, "")
886 +
887 + def is_reasoning_end(self, input_ids: list[int]) -> bool:
888 + if self._reasoning_encoding == "none":
889 + return True
890 + if super().is_reasoning_end(input_ids):
891 + return True
892 + # [TOOL_CALLS] acts as an implicit reasoning-end marker
893 + if self.bot_token_id is not None:
894 + reasoning_start_id = self._reasoning_start_token_id
895 + for i in range(len(input_ids) - 1, -1, -1):
896 + if (
897 + reasoning_start_id is not None
898 + and input_ids[i] == reasoning_start_id
899 + ):
900 + return False
901 + if input_ids[i] == self.bot_token_id:
902 + return True
903 + return False
904 +
905 + def extract_reasoning(
906 + self,
907 + model_output: str,
908 request: ChatCompletionRequest | ResponsesRequest,
71 - prompt_token_ids: list[int] | None = None,
72 - *,
73 - finished: bool,
74 - ) -> DeltaMessage | None:
75 - self._maybe_force_auto_tool_parsing(request)
76 - return super().parse_delta(
77 - delta_text,
78 - delta_token_ids,
79 - request,
80 - prompt_token_ids,
81 - finished=finished,
909 + ) -> tuple[str | None, str | None]:
910 + if self._reasoning_encoding == "none":
911 + return None, model_output
912 + return super().extract_reasoning(model_output, request)
913 +
914 + def _accept_tool_name(self, name: str) -> bool:
915 + # Once `[ARGS]` or the opening `{` has moved the slot past its name,
916 + # that name is final, empty included. Emitting the call with ""
917 + # surfaces the malformed generation instead of dropping it.
918 + return self._is_valid_tool_name(name)
919 +
920 + def _try_extract_name(self, idx: int) -> str | None:
921 + # No JSON-embedded "name" key in this format; the slot name is final.
922 + return self._tool_slots[idx].name
923 +
924 + def _extract_name_and_args(self, raw_body: str) -> tuple[str, str]:
925 + # Never a {"name": ..., "arguments": ...} envelope -- a literal "name"
926 + # argument key must not be mistaken for the tool name.
927 + return "", self._extract_args_json(raw_body, "")
928 +
929 + def _handle_arg_chunk(
930 + self,
931 + event: SemanticEvent,
932 + deltas: list[DeltaToolCall],
933 + ) -> None:
934 + """Emit the opening ``{`` as an arg delta when the name is first sent.
935 +
936 + When the TOOL_NAME→TOOL_ARGS transition fires, ``{`` arrives as an
937 + ARG_VALUE_CHUNK before ``name_sent`` is True. The parent emits the
938 + name delta but not the ``{`` arg delta. This override re-emits the
939 + current chunk so streaming clients receive a valid JSON prefix.
940 + """
941 + idx = event.tool_index
942 + name_sent_before = (
943 + 0 <= idx < len(self._tool_slots) and self._tool_slots[idx].name_sent
944 )
945 + super()._handle_arg_chunk(event, deltas)
946 + if (
947 + event.value
948 + and not name_sent_before
949 + and 0 <= idx < len(self._tool_slots)
950 + and self._tool_slots[idx].name_sent
951 + ):
952 + deltas.append(
953 + DeltaToolCall(
954 + index=idx,
955 + function=DeltaFunctionCall(arguments=event.value),
956 + )
957 + )
958 +
959 + def _extract_args_json(self, raw_args: str, func_name: str) -> str:
960 + """Return the first complete JSON value in ``raw_args``.
961 +
962 + v11+ tool calls are emitted as ``name{args}`` with no terminator, so a
963 + model may append ordinary text after the closing brace. Parse the first
964 + JSON value with ``raw_decode`` and drop any trailing output, mirroring
965 + the pre-v11 path (fixes gh#48975).
966 + """
967 + stripped = raw_args.strip()
968 + if not stripped:
969 + return "{}"
970 + try:
971 + _, end = json.JSONDecoder().raw_decode(stripped)
972 + except json.JSONDecodeError:
973 + return stripped
974 + return stripped[:end]
vllm/parser/parser_manager.py
-9
@@ -109,8 +109,6 @@ class ParserManager:
109 if reasoning_parser_cls is None and tool_parser_cls is None:
110 return None
111
112 - from vllm.utils.mistral import is_mistral_tool_parser
113 -
112 if is_harmony:
113 from vllm.parser.harmony import HarmonyParser
114
@@ -118,13 +116,6 @@ class ParserManager:
116 HarmonyParser.tool_parser_cls = tool_parser_cls
117 return HarmonyParser
118
121 - if is_mistral_tool_parser(tool_parser_cls):
122 - from vllm.parser.mistral import MistralParser
123 -
124 - MistralParser.reasoning_parser_cls = reasoning_parser_cls
125 - MistralParser.tool_parser_cls = tool_parser_cls
126 - return MistralParser
127 -
119 if reasoning_parser_name == "kimi_k3" or tool_parser_name == "kimi_k3":
120 from vllm.parser.kimi_k3 import KimiK3Parser
121
vllm/reasoning/__init__.py
+1 -1
@@ -106,7 +106,7 @@ _REASONING_PARSERS_TO_REGISTER = {
106 ),
107 "mistral": (
108 "mistral_reasoning_parser",
109 - "MistralReasoningParser",
109 + "MistralParserReasoningAdapter",
110 ),
111 "nemotron_v3": (
112 "nemotron_v3_engine_reasoning_parser",
vllm/reasoning/mistral_reasoning_parser.py
+2 -158
@@ -1,162 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -from collections.abc import Iterable, Sequence
5 -from functools import cached_property
6 -from typing import TYPE_CHECKING
4 +from vllm.parser.engine.registered_adapters import MistralParserReasoningAdapter
5
8 -from vllm.reasoning import ReasoningParser
9 -from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
10 -from vllm.tokenizers.mistral import MistralTokenizer
11 -
12 -if TYPE_CHECKING:
13 - from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
14 - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
15 -
16 -
17 -class MistralReasoningParser(BaseThinkingReasoningParser):
18 - """
19 - Reasoning parser for Mistral models.
20 -
21 - The Mistral models uses `[THINK]`...`[/THINK]` tokens to denote reasoning
22 - text. This parser extracts the reasoning content from the model output.
23 -
24 - A valid reasoning trace should always start with a `[THINK]` token and end with
25 - a `[/THINK]` token.
26 -
27 - If `[THINK]` token is not generated, then this parser only returns content.
28 - """
29 -
30 - def __init__(self, tokenizer: MistralTokenizer, *args, **kwargs):
31 - if not isinstance(tokenizer, MistralTokenizer):
32 - raise ValueError("The tokenizer must be an instance of MistralTokenizer.")
33 -
34 - ReasoningParser.__init__(self, tokenizer, *args, **kwargs)
35 -
36 - if not self.model_tokenizer:
37 - raise ValueError(
38 - "The model tokenizer must be passed to the ReasoningParser "
39 - "constructor during construction."
40 - )
41 -
42 - self.start_token_id = tokenizer.tokenizer.get_special_token(self.start_token)
43 - self.end_token_id = tokenizer.tokenizer.get_special_token(self.end_token)
44 -
45 - if self.start_token_id is None or self.end_token_id is None:
46 - raise RuntimeError(
47 - "Mistral reasoning parser could not locate think start/end "
48 - "tokens in the tokenizer!"
49 - )
50 -
51 - @cached_property
52 - def start_token(self) -> str:
53 - """The token that starts reasoning content."""
54 - from mistral_common.tokens.tokenizers.base import SpecialTokens
55 -
56 - return SpecialTokens.begin_think
57 -
58 - @cached_property
59 - def end_token(self) -> str:
60 - """The token that ends reasoning content."""
61 - from mistral_common.tokens.tokenizers.base import SpecialTokens
62 -
63 - return SpecialTokens.end_think
64 -
65 - def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
66 - has_eot_token = False
67 -
68 - for id in reversed(input_ids):
69 - if id == self.start_token_id:
70 - # Reasoning ends only if a BOT token is found before a EOT token.
71 - return has_eot_token
72 - elif id == self.end_token_id:
73 - has_eot_token = True
74 - return False
75 -
76 - def is_reasoning_end_streaming(
77 - self, input_ids: Sequence[int], delta_ids: Iterable[int]
78 - ) -> bool:
79 - if self.end_token_id in delta_ids:
80 - return True
81 - # Grammar's think? is optional — if [THINK] was never generated,
82 - # reasoning was skipped entirely.
83 - return self.start_token_id not in input_ids
84 -
85 - def extract_content_ids(self, input_ids: list[int]) -> list[int]:
86 - """
87 - Extract the content
88 - """
89 - has_bot_token = False
90 - has_eot_token = False
91 - bot_token_index = -1
92 - eot_token_index = -1
93 - # One for loop instead of multiple lookups
94 - for i, token_id in enumerate(input_ids):
95 - # We filter that we have multiple BOT tokens which should not
96 - # happen for a well prompted trained model
97 - if token_id == self.start_token_id and not has_bot_token:
98 - has_bot_token = True
99 - bot_token_index = i
100 - elif token_id == self.end_token_id:
101 - has_eot_token = True
102 - eot_token_index = i
103 - break
104 -
105 - # 1. Only BOT has been outputted
106 - if has_bot_token and not has_eot_token:
107 - # Should be = [] if model is well prompted and trained.
108 - return input_ids[:bot_token_index]
109 - # 2. Neither BOT or EOT have been outputted
110 - elif not has_bot_token and not has_eot_token:
111 - return input_ids
112 - # 3. Both BOT and EOT have been outputted.
113 - elif has_bot_token and has_eot_token:
114 - return input_ids[:bot_token_index] + input_ids[eot_token_index + 1 :]
115 - # 4. Only EOT has been outputted => this should not have occurred for a model
116 - # well prompted and trained.
117 - else:
118 - return input_ids[:eot_token_index] + input_ids[eot_token_index + 1 :]
119 -
120 - def extract_reasoning(
121 - self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest"
122 - ) -> tuple[str | None, str | None]:
123 - """
124 - Extract reasoning content from the model output.
125 - """
126 - if not model_output:
127 - return (None, "")
128 -
129 - # Check if the start token is present in the model output, remove it
130 - # if it is present.
131 - prev_bot_token, bot_token, post_bot_token = model_output.partition(
132 - self.start_token
133 - )
134 -
135 - has_bot_token = bool(bot_token)
136 - # Valid EOT tokens should follow BOT token
137 - has_valid_eot_token = has_bot_token and self.end_token in post_bot_token
138 -
139 - # 1. If there is BOT token followed by EOT token
140 - if has_bot_token and has_valid_eot_token:
141 - prev_eot_token, _, post_eot_token = post_bot_token.partition(self.end_token)
142 - # If model is well prompted and trained prev_bot_token should be ""
143 - content = prev_bot_token + post_eot_token
144 - return prev_eot_token, content if content else None
145 - # 2. Only BOT token
146 - elif has_bot_token:
147 - # If model is well prompted and trained prev_bot_token should be ""
148 - return post_bot_token, prev_bot_token if prev_bot_token else None
149 - # 3. EOT token has been outputted without BOT or neither has been outputted
150 - else:
151 - has_non_valid_eot_token = self.end_token in prev_bot_token
152 - # 3.a EOT token has been outputted without BOT
153 - # If model is well prompted and trained `has_non_valid_eot_token` should
154 - # be `False` and the parser outputs all tokens as 'content'
155 - if has_non_valid_eot_token:
156 - prev_eot_token, _, post_eot_token = prev_bot_token.partition(
157 - self.end_token
158 - )
159 - return None, prev_eot_token + post_eot_token
160 - # 3.b neither BOT or EOT have been outputted
161 - else:
162 - return None, prev_bot_token
6 +__all__ = ["MistralParserReasoningAdapter"]
vllm/tokenizers/mistral.py
+15 -6
@@ -21,7 +21,6 @@ from mistral_common.tokens.tokenizers.base import (
21 )
22 from mistral_common.tokens.tokenizers.instruct import (
23 InstructTokenizerBase,
24 - InstructTokenizerV13,
24 )
25 from mistral_common.tokens.tokenizers.mistral import (
26 MistralTokenizer as MistralCommonTokenizer,
@@ -461,6 +460,7 @@ class MistralTokenizer(TokenizerLike):
460 def convert_tokens_to_string(self, tokens: list[str]) -> str:
461 to_decode_special_tokens = {
462 SpecialTokens.tool_calls,
463 + SpecialTokens.args,
464 SpecialTokens.begin_think,
465 SpecialTokens.end_think,
466 }
@@ -517,11 +517,20 @@ class MistralTokenizer(TokenizerLike):
517 non_skip_special_tokens_ids = {
518 self.tokenizer.get_special_token(SpecialTokens.tool_calls),
519 }
520 - if isinstance(self.instruct, InstructTokenizerV13):
521 - if self.instruct.BEGIN_THINK:
522 - non_skip_special_tokens_ids.add(self.instruct.BEGIN_THINK)
523 - if self.instruct.END_THINK:
524 - non_skip_special_tokens_ids.add(self.instruct.END_THINK)
520 + # [ARGS] only exists in v11+ tool-call tokenizers; older tokenizers
521 + # raise (Tekken) or return unk (SPM) for it.
522 + if self.tokenizer.is_special(SpecialTokens.args):
523 + non_skip_special_tokens_ids.add(
524 + self.tokenizer.get_special_token(SpecialTokens.args)
525 + )
526 + # [THINK]/[/THINK] only exist in v13+ reasoning tokenizers; use the
527 + # same is_special gate as [ARGS] above so newer versions are covered
528 + # without an isinstance check on the instruct tokenizer.
529 + for think_token in (SpecialTokens.begin_think, SpecialTokens.end_think):
530 + if self.tokenizer.is_special(think_token):
531 + non_skip_special_tokens_ids.add(
532 + self.tokenizer.get_special_token(think_token)
533 + )
534
535 ids_kept = [
536 i
vllm/tool_parsers/mistral_tool_parser.py
+26 -756
@@ -3,767 +3,37 @@
3
4 from __future__ import annotations
5
6 -import json
7 -from collections.abc import Sequence
8 -from enum import Enum, auto
9 -from random import choices
10 -from string import ascii_letters, digits
11 -from typing import Any
6 +from typing import TYPE_CHECKING
7
13 -import ijson
14 -import regex as re
15 -from mistral_common.protocol.instruct.tool_calls import (
16 - NamedToolChoice as MistralNamedToolChoice,
17 -)
18 -from mistral_common.protocol.instruct.tool_calls import (
19 - Tool as MistralTool,
20 -)
21 -from mistral_common.protocol.instruct.tool_calls import (
22 - ToolChoice as MistralToolChoice,
23 -)
24 -from mistral_common.protocol.instruct.tool_calls import (
25 - ToolChoiceEnum as MistralToolChoiceEnum,
26 -)
27 -from pydantic import Field
8 +from vllm.parser.engine.registered_adapters import MistralParserToolAdapter
9
29 -from vllm.entrypoints.openai.chat_completion.protocol import (
30 - ChatCompletionRequest,
31 -)
32 -from vllm.entrypoints.openai.engine.protocol import (
33 - DeltaFunctionCall,
34 - DeltaMessage,
35 - DeltaToolCall,
36 - ExtractedToolCallInformation,
37 - FunctionCall,
38 - ToolCall,
39 -)
40 -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
41 -from vllm.logger import init_logger
42 -from vllm.sampling_params import StructuredOutputsParams
43 -from vllm.tokenizers import TokenizerLike
44 -from vllm.tool_parsers.abstract_tool_parser import (
45 - Tool,
46 - ToolParser,
47 -)
48 -from vllm.utils.mistral import is_mistral_tokenizer
10 +if TYPE_CHECKING:
11 + from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
12 + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
13
50 -logger = init_logger(__name__)
14
52 -ALPHANUMERIC = ascii_letters + digits
15 +class MistralToolParser(MistralParserToolAdapter): # type: ignore[valid-type, misc]
16 + # Marked so is_mistral_tool_parser() in vllm.utils.mistral recognises
17 + # this adapter and lets adjust_request() fire for tool_choice="none"
18 + # on grammar-capable Mistral tokenizers.
19 + IS_MISTRAL_TOOL_PARSER = True
20
54 -_DEFAULT_JSON_SCHEMA = {"anyOf": [{"type": "object"}, {"type": "array"}]}
55 -
56 -
57 -class StreamingState(Enum):
58 - """Enum for tracking the current streaming parsing state."""
59 -
60 - WAITING_FOR_TOOL_START = auto()
61 - WAITING_FOR_TOOL_KEY = (
62 - auto()
63 - ) # waiting for the "name" or "arguments" key to be complete
64 - PARSING_NAME = auto()
65 - PARSING_NAME_COMPLETED = auto()
66 - WAITING_FOR_ARGUMENTS_START = auto()
67 - PARSING_ARGUMENTS = auto()
68 - PARSING_ARGUMENTS_COMPLETED = auto()
69 - TOOL_COMPLETE = auto()
70 - ALL_TOOLS_COMPLETE = auto()
71 -
72 -
73 -class MistralToolCall(ToolCall):
74 - id: str = Field(default_factory=lambda: MistralToolCall.generate_random_id())
75 -
76 - @staticmethod
77 - def generate_random_id():
78 - # Mistral Tool Call Ids must be alphanumeric with a length of 9.
79 - # https://github.com/mistralai/mistral-common/blob/21ee9f6cee3441e9bb1e6ed2d10173f90bd9b94b/src/mistral_common/protocol/instruct/validator.py#L299
80 - return "".join(choices(ALPHANUMERIC, k=9))
81 -
82 - @staticmethod
83 - def is_valid_id(id: str) -> bool:
84 - return id.isalnum() and len(id) == 9
85 -
86 -
87 -def _is_pre_v11_tokeniser(model_tokenizer: TokenizerLike) -> bool:
88 - if is_mistral_tokenizer(model_tokenizer):
89 - return model_tokenizer.version < 11
90 - # For HF tokenizers, check if [ARGS] token exists in vocab
91 - # which indicates a v11+ equivalent tokenizer
92 - vocab: dict[str, int] = getattr(model_tokenizer, "get_vocab", lambda: {})()
93 - return "[ARGS]" not in vocab
94 -
95 -
96 -class MistralToolParser(ToolParser):
97 - r"""Tool call parser for Mistral models, intended for use with either:
98 -
99 - - `mistral_common <https://github.com/mistralai/mistral-common/>`_
100 - (recommended)
101 - - the `examples/tool_chat_template_mistral.jinja` template.
102 -
103 - Used when `--enable-auto-tool-choice --tool-call-parser mistral` are all
104 - set.
105 - """
106 -
107 - IS_MISTRAL_TOOL_PARSER = True # used by vllm.utils.mistral
108 -
109 - # Used to generate correct grammar in `adjust_request`
110 - model_can_reason: bool = False
111 -
112 - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
113 - super().__init__(tokenizer, tools)
114 -
115 - if not is_mistral_tokenizer(self.model_tokenizer):
116 - logger.info("Non-Mistral tokenizer detected when using a Mistral model...")
117 -
118 - # initialize properties used for state when parsing tool calls in
119 - # streaming mode
120 - self.prev_tool_call_arr: list[dict[str, Any]] = []
121 - self.current_tool_id: int = -1
122 - self.streaming_state: StreamingState = StreamingState.WAITING_FOR_TOOL_START
123 -
124 - # For streaming pre v11 tokenizer tool calls
125 - self.current_tool_name: str | None = None
126 - self.current_tool_mistral_id: str | None = None
127 - self.starting_new_tool = False
128 - self._is_pre_v11 = _is_pre_v11_tokeniser(self.model_tokenizer)
129 - if self._is_pre_v11:
130 - self.parse_coro = ijson.parse_coro(
131 - self.update_stream_state_pre_v11_tokenizer()
132 - )
133 -
134 - self.bot_token = "[TOOL_CALLS]"
135 - self.bot_token_id = self.vocab.get(self.bot_token)
136 - self.tool_call_regex = re.compile(r"\[{.*}\]", re.DOTALL)
137 -
138 - if self.bot_token_id is None:
139 - raise RuntimeError(
140 - "Mistral Tool Parser could not locate the tool call token in "
141 - "the tokenizer!"
142 - )
21 + # Mistral emits tool calls in the [TOOL_CALLS]name[ARGS]{...} format, not the
22 + # plain JSON array / bare-args that the serving layer's generic required and
23 + # named handlers expect. Route required/named through this parser's own
24 + # extraction (treated like "auto") so the [TOOL_CALLS] format is parsed
25 + # correctly instead of crashing or dumping the raw envelope into arguments.
26 + supports_required_and_named = False
27
28 def adjust_request(
145 - self, request: ChatCompletionRequest | ResponsesRequest
146 - ) -> ChatCompletionRequest | ResponsesRequest:
147 - so_non_supported_attributes = [
148 - "regex",
149 - "choice",
150 - "grammar",
151 - # whitespace_pattern is not a constraint type but an option;
152 - # Mistral grammar factory does not support it.
153 - "whitespace_pattern",
154 - "structural_tag",
155 - ]
156 - any_so_non_supported_active = request.structured_outputs is not None and any(
157 - getattr(request.structured_outputs, attribute) is not None
158 - for attribute in so_non_supported_attributes
159 - )
160 - response_format_non_supported_active = (
161 - isinstance(request, ResponsesRequest)
162 - or request.response_format is not None
163 - and request.response_format.type == "structural_tag"
164 - )
165 -
166 - if (
167 - not is_mistral_tokenizer(self.model_tokenizer)
168 - or isinstance(request, ResponsesRequest)
169 - or not self.model_tokenizer.supports_grammar
170 - or any_so_non_supported_active
171 - or response_format_non_supported_active
172 - ):
173 - request = super().adjust_request(request)
174 - if request.tools and request.tool_choice != "none":
175 - # Do not skip special tokens when using chat template
176 - # with Mistral parser as TOOL_CALL token is needed
177 - # for tool detection.
178 - # Note: we don't want skip_special_tokens=False
179 - # with MistralTokenizer as it is incompatible
180 - request.skip_special_tokens = False
181 - return request
182 -
183 - json_schema: dict[str, Any] | None = None
184 - if request.structured_outputs is not None:
185 - if request.structured_outputs.json_object is not None:
186 - json_schema = _DEFAULT_JSON_SCHEMA
187 - elif request.structured_outputs.json is not None:
188 - if isinstance(request.structured_outputs.json, str):
189 - json_schema = json.loads(request.structured_outputs.json)
190 - else:
191 - json_schema = request.structured_outputs.json
192 - else:
193 - raise ValueError(
194 - "Unsupported request.structured_outputs for MistralToolParser. "
195 - "Only `json` and `json_object` are supported."
196 - )
197 - elif (
198 - request.response_format is not None
199 - and request.response_format.type != "text"
200 - ):
201 - if request.response_format.type == "json_object":
202 - json_schema = _DEFAULT_JSON_SCHEMA
203 - elif request.response_format.type == "json_schema":
204 - if request.response_format.json_schema is not None:
205 - json_schema = request.response_format.json_schema.json_schema
206 - else:
207 - json_schema = _DEFAULT_JSON_SCHEMA
208 - else:
209 - raise ValueError(
210 - "MistralToolParser only accepts `text`, `json_object` or "
211 - f"`json_schema`, got {request.response_format=}"
212 - )
213 - # Structured Outputs will be defined.
214 - request.response_format = None
215 -
216 - grammar_factory = self.model_tokenizer.grammar_factory
217 -
218 - # TODO: Once unified parser, improve this.
219 - # The issue is figuring out when a model is a reasoning one or not.
220 - template = grammar_factory.select_jinja_template(
221 - reasoning=self.model_can_reason
222 - )
223 -
224 - mistral_tools = (
225 - [MistralTool.from_openai(tool.model_dump()) for tool in request.tools]
226 - if request.tools is not None
227 - else None
228 - )
229 -
230 - tool_choice: MistralToolChoice
231 - match request.tool_choice:
232 - case "none" | "auto" | "required":
233 - tool_choice = MistralToolChoiceEnum(request.tool_choice)
234 - case None:
235 - tool_choice = MistralToolChoiceEnum.auto
236 - # _ == Named tool choice
237 - case _:
238 - tool_choice = MistralNamedToolChoice.model_validate(
239 - {
240 - "type": "function",
241 - "function": {"name": request.tool_choice.function.name},
242 - }
243 - )
244 -
245 - # Rendering grammar is cached in mistral-common given tools, template and mode.
246 - match tool_choice, json_schema is not None:
247 - case MistralToolChoiceEnum.none, True:
248 - lark_grammar = grammar_factory.get_lark_for_json_schema(
249 - template=template, json_schema=json_schema
250 - )
251 - case _, _:
252 - lark_grammar = grammar_factory.get_lark_from_jinja(
253 - template=template,
254 - mode=tool_choice,
255 - tools=mistral_tools,
256 - json_schema=json_schema,
257 - parallel_tool_calls=request.parallel_tool_calls,
258 - json_only=False,
259 - )
260 -
261 - request.structured_outputs = StructuredOutputsParams(grammar=lark_grammar)
262 - request._grammar_from_tool_parser = True
263 - return request
264 -
265 - def extract_tool_calls(
266 - self,
267 - model_output: str,
268 - request: ChatCompletionRequest,
269 - ) -> ExtractedToolCallInformation:
270 - """
271 - Extract the tool calls from a complete model response.
272 -
273 - Content and tool calls formatting depends on the Mistral's tokenizer version
274 - used to train the model:
275 -
276 - - < v11: `content[BOT] [{tool_call1},{tool_call2}]`
277 - - >= v11: `content[BOT]tool_name1{args_call1}[BOT]tool_name2{args_call2}`
278 -
279 - with [BOT] the tool call token.
280 -
281 - Note:
282 - For tokenizer versions >= v11, tool calls with arguments wrongly formatted
283 - are still returned as tool calls. This is to allow the model to know it
284 - tried to make a tool call. It reduces chance of another failure and
285 - prevents that the context is filled with tool calls wrongly placed in
286 - assistant message contents.
287 - """
288 -
289 - # If the tool call token is not present, return a text response
290 - if self.bot_token not in model_output:
291 - return ExtractedToolCallInformation(
292 - tools_called=False, tool_calls=[], content=model_output
293 - )
294 -
295 - content_and_raw_tool_calls = model_output.split(self.bot_token)
296 - content = content_and_raw_tool_calls[0]
297 - raw_tool_calls = content_and_raw_tool_calls[1:]
298 -
299 - # >= v11: content[BOT]tool_name1{args_call1}[BOT]tool_name2{args_call2}
300 - if not self._is_pre_v11:
301 - tool_calls = []
302 - for raw_tool_call in raw_tool_calls:
303 - if "{" not in raw_tool_call:
304 - continue
305 -
306 - end_name = raw_tool_call.find("{")
307 - tool_name, args = (
308 - raw_tool_call[:end_name],
309 - raw_tool_call[end_name:],
310 - )
311 -
312 - # HF tokenizers may include [ARGS] in the text
313 - tool_name = tool_name.replace("[ARGS]", "")
314 - tool_calls.append({"name": tool_name, "arguments": args})
315 -
316 - # < v11: content[BOT] [{tool_call1},{tool_call2}]
317 - else:
318 - if len(raw_tool_calls) != 1:
319 - raise ValueError(
320 - "Only one BOT token should have been outputted, "
321 - f"but got {model_output}."
322 - )
323 - stringified_tool_calls = raw_tool_calls[0].strip()
324 - try:
325 - # Use raw_decode to parse the first valid JSON value,
326 - # ignoring trailing tokens the model may emit after
327 - # the tool call array.
328 - tool_calls, _ = json.JSONDecoder().raw_decode(stringified_tool_calls)
329 - except json.JSONDecodeError:
330 - try:
331 - raw_tool_call = self.tool_call_regex.findall(
332 - stringified_tool_calls
333 - )[0]
334 - tool_calls = json.loads(raw_tool_call)
335 - tool_calls = [
336 - {
337 - "name": tool_call["name"],
338 - "arguments": json.dumps(
339 - tool_call.get("arguments", {}),
340 - ensure_ascii=False,
341 - ),
342 - }
343 - for tool_call in tool_calls
344 - ]
345 - except (IndexError, json.JSONDecodeError):
346 - logger.exception("Error in extracting tool call from response.")
347 - return ExtractedToolCallInformation(
348 - tools_called=False,
349 - tool_calls=[],
350 - content=stringified_tool_calls,
351 - )
352 - else:
353 - tool_calls = [
354 - {
355 - "name": tool_call["name"],
356 - "arguments": json.dumps(
357 - tool_call.get("arguments", {}),
358 - ensure_ascii=False,
359 - ),
360 - }
361 - for tool_call in tool_calls
362 - ]
363 -
364 - mistral_tool_calls: list[MistralToolCall] = [
365 - MistralToolCall(
366 - type="function",
367 - function=FunctionCall(
368 - name=tool_call["name"],
369 - arguments=tool_call.get("arguments", "{}"),
370 - ),
371 - )
372 - for tool_call in tool_calls
373 - ]
374 -
375 - return ExtractedToolCallInformation(
376 - tools_called=True,
377 - tool_calls=mistral_tool_calls,
378 - content=content if content.strip() else None,
379 - )
380 -
381 - def extract_tool_calls_streaming(
382 - self,
383 - previous_text: str,
384 - current_text: str,
385 - delta_text: str,
386 - previous_token_ids: Sequence[int],
387 - current_token_ids: Sequence[int],
388 - delta_token_ids: Sequence[int],
389 - request: ChatCompletionRequest,
390 - ) -> DeltaMessage | None:
391 - has_bot_token = (
392 - self.bot_token_id in current_token_ids or self.bot_token in current_text
393 - )
394 - if not has_bot_token:
395 - # if the tool call token is not in the tokens generated so far,
396 - # append output to contents since it's not a tool
397 - return DeltaMessage(content=delta_text)
398 -
399 - # if the tool call token IS in the tokens generated so far, that
400 - # means we're parsing as tool calls now
401 - try:
402 - if self._is_pre_v11:
403 - return self._extract_tool_calls_streaming_pre_v11_tokenizer(
404 - delta_text=delta_text,
405 - delta_token_ids=delta_token_ids,
406 - )
407 - else:
408 - return self._extract_tool_calls_streaming(
409 - delta_text=delta_text, delta_token_ids=delta_token_ids
410 - )
411 - except Exception:
412 - logger.exception("Error trying to handle streaming tool call.")
413 - return None
414 -
415 - def _extract_tool_calls_streaming(
416 - self,
417 - delta_text: str,
418 - delta_token_ids: Sequence[int],
419 - ) -> DeltaMessage | None:
420 - """
421 - Extracts tool calls for Mistral models
422 - doing tool calls of the following format:
423 - `[TOOL_CALLS]add{"a": 3.5, "b": 4}`
424 - """
425 - additional_content: str = ""
426 - if self.streaming_state == StreamingState.WAITING_FOR_TOOL_START:
427 - # this is the first tool call
428 - if self.bot_token not in delta_text:
429 - return DeltaMessage(content=delta_text)
430 - if not delta_text.startswith(self.bot_token):
431 - additional_content += delta_text.split(self.bot_token)[0]
432 - delta_text = self.bot_token + "".join(
433 - delta_text.split(self.bot_token)[1:]
434 - )
435 -
436 - delta_tool_calls = self._generate_delta_tool_call(delta_text)
437 - if not additional_content and len(delta_tool_calls) == 0:
438 - if self.streaming_state in [
439 - StreamingState.PARSING_ARGUMENTS,
440 - StreamingState.PARSING_ARGUMENTS_COMPLETED,
441 - StreamingState.TOOL_COMPLETE,
442 - StreamingState.ALL_TOOLS_COMPLETE,
443 - ]:
444 - # Return an empty DeltaMessage once the tool calls are all done
445 - # so that finish_reason gets set.
446 - return DeltaMessage()
447 - else:
448 - # return None when the tool is not likely to be finished
449 - # This can occur when the name is being parsed for example
450 - # and we wait for the name to be complete
451 - # before sending the function name
452 - return None
453 -
454 - delta = DeltaMessage()
455 - if additional_content:
456 - delta.content = additional_content
457 - if len(delta_tool_calls) > 0:
458 - delta.tool_calls = delta_tool_calls
459 -
460 - return delta
461 -
462 - def _generate_delta_tool_call(self, delta_text: str) -> list[DeltaToolCall]:
463 - if delta_text == "" or delta_text is None:
464 - return []
465 - delta_function_name = None
466 - tool_id = None
467 - if self.streaming_state not in [
468 - StreamingState.PARSING_NAME,
469 - StreamingState.PARSING_ARGUMENTS,
470 - ] and delta_text.startswith(self.bot_token):
471 - self.current_tool_id += 1
472 - self.streamed_args_for_tool.append("")
473 - self.prev_tool_call_arr.append({})
474 - self.streaming_state = StreamingState.PARSING_NAME
475 - delta_text = delta_text.replace(self.bot_token, "", 1)
476 - if self.streaming_state == StreamingState.PARSING_NAME:
477 - if self.current_tool_name is None:
478 - self.current_tool_name = ""
479 - # The name stops where the arguments start
480 - # And the arguments start with the `{` char
481 - if "{" in delta_text:
482 - tool_id = MistralToolCall.generate_random_id()
483 - delta_function_name = delta_text.split("{")[0]
484 - self.current_tool_name += delta_function_name
485 - # HF tokenizers may include [ARGS] in the text
486 - self.current_tool_name = self.current_tool_name.replace("[ARGS]", "")
487 - self.prev_tool_call_arr[self.current_tool_id]["name"] = (
488 - self.current_tool_name
489 - )
490 - delta_text = delta_text[len(delta_function_name) :]
491 - self.streaming_state = StreamingState.PARSING_ARGUMENTS
492 - else:
493 - # we want to send the tool name once it's complete
494 - self.current_tool_name += delta_text
495 - return []
496 - if self.streaming_state == StreamingState.PARSING_ARGUMENTS:
497 - next_function_text = None
498 - if self.bot_token in delta_text:
499 - # current tool call is over
500 - delta_arguments = ""
501 - delta_arguments += delta_text.split(self.bot_token)[0]
502 - next_function_text = delta_text[len(delta_arguments) :]
503 - self.streaming_state = StreamingState.TOOL_COMPLETE
504 - else:
505 - delta_arguments = delta_text
506 - self.streamed_args_for_tool[self.current_tool_id] += delta_arguments
507 - self.prev_tool_call_arr[self.current_tool_id]["arguments"] = (
508 - self.streamed_args_for_tool[self.current_tool_id]
509 - )
510 - ret = []
511 - if self.current_tool_name or delta_arguments:
512 - ret += [
513 - DeltaToolCall(
514 - index=self.current_tool_id,
515 - type="function",
516 - id=tool_id,
517 - function=DeltaFunctionCall(
518 - name=self.current_tool_name, arguments=delta_arguments
519 - ).model_dump(exclude_none=True),
520 - )
521 - ]
522 - self.current_tool_name = None
523 - if next_function_text:
524 - ret += self._generate_delta_tool_call(next_function_text)
525 - return ret
526 - # Should not happen
527 - return []
528 -
529 - @ijson.coroutine
530 - def update_stream_state_pre_v11_tokenizer(self):
531 - while True:
532 - (prefix, event, value) = yield
533 -
534 - if prefix == "item" and event == "start_map":
535 - self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY
536 - self.starting_new_tool = True
537 - if prefix == "item" and event == "map_key" and value == "name":
538 - self.streaming_state = StreamingState.PARSING_NAME
539 - if prefix == "item.name" and event == "string":
540 - self.current_tool_name = value
541 - self.streaming_state = StreamingState.PARSING_NAME_COMPLETED
542 - if prefix == "item" and event == "map_key" and value == "arguments":
543 - self.streaming_state = StreamingState.WAITING_FOR_ARGUMENTS_START
544 - if prefix == "item.arguments" and event == "start_map":
545 - self.streaming_state = StreamingState.PARSING_ARGUMENTS
546 - if prefix == "item.arguments" and event == "end_map":
547 - self.streaming_state = StreamingState.PARSING_ARGUMENTS_COMPLETED
548 - if prefix == "item" and event == "end_map":
549 - self.streaming_state = StreamingState.TOOL_COMPLETE
550 - if prefix == "" and event == "end_array":
551 - self.streaming_state = StreamingState.ALL_TOOLS_COMPLETE
552 -
553 - def _extract_tool_calls_streaming_pre_v11_tokenizer(
29 self,
555 - delta_text: str,
556 - delta_token_ids: Sequence[int],
557 - ) -> DeltaMessage | None:
558 - """
559 - Extracts tool calls for Mistral models
560 - doing tool calls of the following format:
561 - `[TOOL_CALLS][{"name": "add", "arguments":{"a": 3.5, "b": 4}}`
562 - """
563 - assert self.parse_coro is not None
564 - content = None
565 - delta_tool_calls: list[DeltaToolCall] = []
566 - current_tool_call: DeltaToolCall = DeltaToolCall(
567 - index=self.current_tool_id, type="function"
568 - )
569 - current_tool_call_modified = False
570 - if self.bot_token_id in delta_token_ids or self.bot_token in delta_text:
571 - # this is the first tool call
572 - if not delta_text.startswith(self.bot_token):
573 - content = delta_text.split(self.bot_token)[0]
574 - delta_text = "".join(delta_text.split(self.bot_token)[1:])
575 -
576 - # Cut smartly the delta text to catch the ijson events
577 - # as ijson does not give us the index in the text at each event.
578 - # We need to cut so that we know
579 - # where in the text the events are emitted from.
580 - while len(delta_text) > 0:
581 - streaming_state_before_parse = self.streaming_state
582 -
583 - if self.streaming_state == StreamingState.WAITING_FOR_TOOL_START:
584 - delta_to_be_parsed, delta_text = self._split_delta(
585 - delta_text=delta_text,
586 - stop_after_opening_curly_braces=1,
587 - )
588 - elif self.streaming_state == StreamingState.WAITING_FOR_TOOL_KEY:
589 - # Wait until another key is sent
590 - # or the current tool is completed
591 - delta_to_be_parsed, delta_text = self._split_delta(
592 - delta_text=delta_text,
593 - stop_after_colon=1,
594 - stop_after_opening_curly_braces=1,
595 - # if the tool ends, we want to separate
596 - # at the start of the next tool
597 - )
598 - elif self.streaming_state == StreamingState.PARSING_NAME:
599 - delta_to_be_parsed, delta_text = self._split_delta(
600 - delta_text=delta_text,
601 - stop_after_comma=1,
602 - stop_after_closing_brackets=1,
603 - )
604 - elif self.streaming_state == StreamingState.WAITING_FOR_ARGUMENTS_START:
605 - delta_to_be_parsed, delta_text = self._split_delta(
606 - delta_text=delta_text,
607 - stop_after_opening_curly_braces=1,
608 - )
609 - elif self.streaming_state == StreamingState.PARSING_ARGUMENTS:
610 - delta_to_be_parsed, delta_text = self._split_delta(
611 - delta_text=delta_text,
612 - stop_after_closing_curly_braces=1,
613 - # we could be more clever
614 - # by listening to item.arguments.* start_map events
615 - # and know how many curly braces we can allow
616 - )
617 - elif self.streaming_state in [
618 - StreamingState.PARSING_ARGUMENTS_COMPLETED,
619 - StreamingState.PARSING_NAME_COMPLETED,
620 - ]:
621 - delta_to_be_parsed, delta_text = self._split_delta(
622 - delta_text=delta_text,
623 - stop_after_closing_curly_braces=1,
624 - stop_after_closing_brackets=1,
625 - )
626 - elif self.streaming_state == StreamingState.TOOL_COMPLETE:
627 - delta_to_be_parsed, delta_text = self._split_delta(
628 - delta_text=delta_text,
629 - stop_after_opening_curly_braces=1,
630 - stop_after_closing_brackets=1,
631 - )
632 - elif self.streaming_state == StreamingState.ALL_TOOLS_COMPLETE:
633 - content = delta_text
634 - delta_text = ""
635 - else:
636 - delta_to_be_parsed = delta_text
637 - delta_text = ""
638 -
639 - if self.streaming_state != StreamingState.ALL_TOOLS_COMPLETE:
640 - self.parse_coro.send(delta_to_be_parsed.encode("utf-8"))
641 -
642 - # Given the parsed text and the possible streaming state change,
643 - # let's add to the tool delta
644 - # start_map is the authoritative new-tool signal and survives
645 - # batched deltas, unlike comparing pre/post streaming states
646 - if self.starting_new_tool:
647 - self.starting_new_tool = False
648 - if current_tool_call_modified:
649 - if self.current_tool_mistral_id is not None:
650 - current_tool_call.id = self.current_tool_mistral_id
651 - self.current_tool_mistral_id = None
652 - self._track_streamed_args_pre_v11(current_tool_call)
653 - delta_tool_calls.append(current_tool_call)
654 - current_tool_call_modified = False
655 - self.current_tool_id += 1
656 - self.streamed_args_for_tool.append("")
657 - self.prev_tool_call_arr.append({})
658 - self.current_tool_mistral_id = MistralToolCall.generate_random_id()
659 - current_tool_call = DeltaToolCall(
660 - index=self.current_tool_id,
661 - type="function",
662 - )
663 - if current_tool_call.function is None:
664 - current_tool_call.function = DeltaFunctionCall()
665 -
666 - if self.current_tool_name is not None:
667 - # we have the complete tool name
668 - current_tool_call_modified = True
669 - current_tool_call.function.name = self.current_tool_name
670 - self.prev_tool_call_arr[self.current_tool_id]["name"] = (
671 - self.current_tool_name
672 - )
673 - self.current_tool_name = None
674 - if self.streaming_state == StreamingState.PARSING_NAME_COMPLETED:
675 - self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY
676 - if self.streaming_state in [
677 - StreamingState.PARSING_ARGUMENTS,
678 - StreamingState.PARSING_ARGUMENTS_COMPLETED,
679 - ]:
680 - if self.streaming_state == StreamingState.PARSING_ARGUMENTS_COMPLETED:
681 - self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY
682 - # the delta_to_be_parsed is part of arguments.
683 - current_tool_call_modified = True
684 - if current_tool_call.function.arguments is None:
685 - current_tool_call.function.arguments = delta_to_be_parsed
686 - else:
687 - current_tool_call.function.arguments += delta_to_be_parsed
688 - if streaming_state_before_parse != StreamingState.PARSING_ARGUMENTS:
689 - # It's the first chunk of arg. let's lstrip it
690 - current_tool_call.function.arguments = (
691 - current_tool_call.function.arguments.lstrip()
692 - )
693 -
694 - if current_tool_call_modified:
695 - if self.current_tool_mistral_id is not None:
696 - current_tool_call.id = self.current_tool_mistral_id
697 - self.current_tool_mistral_id = None
698 - self._track_streamed_args_pre_v11(current_tool_call)
699 - delta_tool_calls.append(current_tool_call)
700 -
701 - if content or len(delta_tool_calls) > 0:
702 - delta_message = DeltaMessage()
703 - if content:
704 - delta_message.content = content
705 - if len(delta_tool_calls) > 0:
706 - delta_message.tool_calls = delta_tool_calls
707 - return delta_message
708 - else:
709 - if self.streaming_state == StreamingState.ALL_TOOLS_COMPLETE:
710 - return DeltaMessage()
711 - else:
712 - return None
713 -
714 - def _track_streamed_args_pre_v11(self, tool_call: DeltaToolCall) -> None:
715 - r"""Accumulate `tool_call` arguments into the streaming state."""
716 - if tool_call.function is not None and tool_call.function.arguments is not None:
717 - self.streamed_args_for_tool[self.current_tool_id] += (
718 - tool_call.function.arguments
719 - )
720 - self.prev_tool_call_arr[self.current_tool_id]["arguments"] = (
721 - self.streamed_args_for_tool[self.current_tool_id]
722 - )
723 -
724 - def _split_delta(
725 - self,
726 - delta_text: str,
727 - stop_after_quotes: int = -1,
728 - stop_after_opening_curly_braces: int = -1,
729 - stop_after_closing_curly_braces: int = -1,
730 - stop_after_closing_brackets: int = -1,
731 - stop_after_colon: int = -1,
732 - stop_after_comma=-1,
733 - ) -> tuple[str, str]:
734 - delta_to_be_parsed = ""
735 - for i, c in enumerate(delta_text):
736 - if c in ['"', "'"]:
737 - delta_to_be_parsed += c
738 - stop_after_quotes -= 1
739 - if stop_after_quotes == 0:
740 - return (delta_to_be_parsed, delta_text[i + 1 :])
741 - elif c == "{":
742 - delta_to_be_parsed += c
743 - stop_after_opening_curly_braces -= 1
744 - if stop_after_opening_curly_braces == 0:
745 - return (delta_to_be_parsed, delta_text[i + 1 :])
746 - elif c == "}":
747 - delta_to_be_parsed += c
748 - stop_after_closing_curly_braces -= 1
749 - if stop_after_closing_curly_braces == 0:
750 - return (delta_to_be_parsed, delta_text[i + 1 :])
751 - elif c == "]":
752 - delta_to_be_parsed += c
753 - stop_after_closing_brackets -= 1
754 - if stop_after_closing_brackets == 0:
755 - return (delta_to_be_parsed, delta_text[i + 1 :])
756 - elif c == ":":
757 - delta_to_be_parsed += c
758 - stop_after_colon -= 1
759 - if stop_after_colon == 0:
760 - return (delta_to_be_parsed, delta_text[i + 1 :])
761 - elif c == ",":
762 - delta_to_be_parsed += c
763 - stop_after_comma -= 1
764 - if stop_after_comma == 0:
765 - return (delta_to_be_parsed, delta_text[i + 1 :])
766 - else:
767 - delta_to_be_parsed += c
768 -
769 - return (delta_to_be_parsed, "")
30 + request: ChatCompletionRequest | ResponsesRequest,
31 + ) -> ChatCompletionRequest | ResponsesRequest:
32 + # Skip the base ToolParser tool_choice -> structured_outputs
33 + # conversion. Mistral enforces tool_choice via the grammar ``mode``
34 + # (auto/none/required/named); forwarding a tool-derived json_schema
35 + # would be unioned into the grammar by mistral-common, allowing raw
36 + # JSON in place of a real [TOOL_CALLS] call. Genuine user-provided
37 + # structured output on the request is still honored by the engine's
38 + # adjust_request.
39 + return self._parser_engine.adjust_request(request)
vllm/tool_parsers/streaming.py
+1 -1
@@ -99,7 +99,7 @@ def extract_named_tool_call_streaming(
99 else:
100 if is_mistral_tokenizer(tokenizer):
101 # Import mistral_common only if we need it.
102 - from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
102 + from vllm.parser.mistral import MistralToolCall
103
104 tool_call_id = MistralToolCall.generate_random_id()
105 else:
vllm/utils/mistral.py
+5 -10
@@ -12,10 +12,8 @@ from vllm.utils.import_utils import LazyLoader
12 if TYPE_CHECKING:
13 # if type checking, eagerly import the module
14 import vllm.tokenizers.mistral as mt
15 - import vllm.tool_parsers.mistral_tool_parser as mtp
15 else:
16 mt = LazyLoader("mt", globals(), "vllm.tokenizers.mistral")
18 - mtp = LazyLoader("mtp", globals(), "vllm.tool_parsers.mistral_tool_parser")
17
18
19 def is_mistral_tokenizer(obj: TokenizerLike | None) -> TypeGuard[mt.MistralTokenizer]:
@@ -31,13 +29,10 @@ def is_mistral_tokenizer(obj: TokenizerLike | None) -> TypeGuard[mt.MistralToken
29
30
31 def is_mistral_tool_parser(cls: type | None) -> bool:
34 - """Return true if *cls* is (a subclass of) MistralToolParser.
32 + """Return true if *cls* carries the ``IS_MISTRAL_TOOL_PARSER`` marker.
33
36 - Uses a class attribute check so that importing
37 - ``vllm.tool_parsers.mistral_tool_parser`` — and transitively
38 - ``mistral_common`` — is not required.
34 + The marker is set on the engine-adapter subclass registered under the
35 + ``"mistral"`` tool-parser key. No import of the parser module is
36 + required.
37 """
40 - return bool(
41 - getattr(cls, "IS_MISTRAL_TOOL_PARSER", False)
42 - and issubclass(cls, mtp.MistralToolParser) # type: ignore[arg-type]
43 - )
38 + return bool(getattr(cls, "IS_MISTRAL_TOOL_PARSER", False))