A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 18:45 UTC
[Bugfix][Parser] Fix Mistral pre-v11 tool parser failing on trailing model output (#40531)
Signed-off-by: dougbtv <dosmith@redhat.com> Signed-off-by: Doug Smith <dougbtv@gmail.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
Doug Smith committed
Apr 22, 2026 at 16:35 UTC
cfa49213d778f56364ea1312fa6b6b61a6d386ef
2 files changed
+66
-18
tests/tool_parsers/test_mistral_tool_parser.py
+49
-9
@@ -24,7 +24,6 @@ from mistral_common.protocol.instruct.tool_calls import (
24
ToolChoiceEnum as MistralToolChoiceEnum,
25
)
26
from partial_json_parser.core.options import Allow
27
-from pydantic import ValidationError
27
28
from vllm.entrypoints.openai.chat_completion.protocol import (
29
ChatCompletionRequest,
@@ -250,6 +249,7 @@ def test_extract_tool_calls_no_tools(parser_fixture, request):
249
"argument_before_name_and_name_in_argument",
250
"multiple_tools",
251
"content_before_tool",
252
+ "trailing_data_after_json",
253
],
254
argnames=["model_output", "expected_tool_calls", "expected_content"],
255
argvalues=[
@@ -338,6 +338,24 @@ def test_extract_tool_calls_no_tools(parser_fixture, request):
338
],
339
"Hello",
340
),
341
+ (
342
+ """[TOOL_CALLS] [{"name": "get_current_weather", "arguments":{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}]\nextra trailing data""", # noqa: E501
343
+ [
344
+ ToolCall(
345
+ function=FunctionCall(
346
+ name="get_current_weather",
347
+ arguments=json.dumps(
348
+ {
349
+ "city": "Dallas",
350
+ "state": "TX",
351
+ "unit": "fahrenheit",
352
+ }
353
+ ),
354
+ )
355
+ )
356
+ ],
357
+ None,
358
+ ),
359
],
360
)
361
def test_extract_tool_calls_pre_v11_tokenizer(
@@ -366,19 +384,22 @@ def test_extract_tool_calls_pre_v11_multiple_bot_tokens_raises(
384
)
385
386
369
-def test_extract_tool_calls_pre_v11_regex_fallback_raises(
387
+def test_extract_tool_calls_pre_v11_regex_fallback(
388
mistral_pre_v11_tool_parser,
389
):
372
- """The regex fallback path finds valid JSON but does not re-serialize
373
- the `arguments` dict to a string, causing a Pydantic
374
- `ValidationError` when constructing `FunctionCall`."""
390
+ """The regex fallback path finds valid JSON via regex when the primary
391
+ raw_decode fails on leading junk. It should re-serialize arguments
392
+ and return a valid tool call."""
393
model_output = (
394
'[TOOL_CALLS] junk [{"name": "add", "arguments":{"a": 1, "b": 2}}] trail'
395
)
378
- with pytest.raises(ValidationError):
379
- mistral_pre_v11_tool_parser.extract_tool_calls(
380
- model_output, request=_DUMMY_REQUEST
381
- )
396
+ result = mistral_pre_v11_tool_parser.extract_tool_calls(
397
+ model_output, request=_DUMMY_REQUEST
398
+ )
399
+ assert result.tools_called
400
+ assert len(result.tool_calls) == 1
401
+ assert result.tool_calls[0].function.name == "add"
402
+ assert result.tool_calls[0].function.arguments == json.dumps({"a": 1, "b": 2})
403
404
405
def test_extract_tool_calls_pre_v11_regex_fallback_fails(
@@ -579,6 +600,7 @@ def _test_extract_tool_calls_streaming(
600
"argument_before_name",
601
"argument_before_name_and_name_in_argument",
602
"multiple_tools",
603
+ "trailing_data_after_json",
604
],
605
argnames=["model_output", "expected_tool_calls", "expected_content"],
606
argvalues=[
@@ -668,6 +690,24 @@ def _test_extract_tool_calls_streaming(
690
],
691
"",
692
),
693
+ (
694
+ """[TOOL_CALLS] [{"name": "get_current_weather", "arguments":{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}]\nextra trailing data""", # noqa: E501
695
+ [
696
+ ToolCall(
697
+ function=FunctionCall(
698
+ name="get_current_weather",
699
+ arguments=json.dumps(
700
+ {
701
+ "city": "Dallas",
702
+ "state": "TX",
703
+ "unit": "fahrenheit",
704
+ }
705
+ ),
706
+ )
707
+ )
708
+ ],
709
+ "\nextra trailing data",
710
+ ),
711
],
712
)
713
def test_extract_tool_calls_streaming_pre_v11_tokenizer(
vllm/tool_parsers/mistral_tool_parser.py
+17
-9
@@ -479,21 +479,28 @@ class MistralToolParser(ToolParser):
479
)
480
stringified_tool_calls = raw_tool_calls[0].strip()
481
try:
482
- tool_calls = json.loads(stringified_tool_calls)
482
+ # Use raw_decode to parse the first valid JSON value,
483
+ # ignoring trailing tokens the model may emit after
484
+ # the tool call array.
485
+ tool_calls, _ = json.JSONDecoder().raw_decode(stringified_tool_calls)
486
except json.JSONDecodeError:
484
- # use a regex to find the part corresponding to the tool call.
485
- # NOTE: This use case should not happen if the model is trained
486
- # correctly. It's an easy possible fix so it's included, but
487
- # can be brittle for very complex / highly nested tool calls
487
try:
488
raw_tool_call = self.tool_call_regex.findall(
489
stringified_tool_calls
490
)[0]
491
tool_calls = json.loads(raw_tool_call)
492
+ tool_calls = [
493
+ {
494
+ "name": tool_call["name"],
495
+ "arguments": json.dumps(
496
+ tool_call.get("arguments", {}),
497
+ ensure_ascii=False,
498
+ ),
499
+ }
500
+ for tool_call in tool_calls
501
+ ]
502
except (IndexError, json.JSONDecodeError):
503
logger.exception("Error in extracting tool call from response.")
495
- # If raw decoding and decoding post regex rule fails, then just
496
- # return content.
504
return ExtractedToolCallInformation(
505
tools_called=False,
506
tool_calls=[],
@@ -504,7 +511,8 @@ class MistralToolParser(ToolParser):
511
{
512
"name": tool_call["name"],
513
"arguments": json.dumps(
507
- tool_call["arguments"], ensure_ascii=False
514
+ tool_call.get("arguments", {}),
515
+ ensure_ascii=False,
516
),
517
}
518
for tool_call in tool_calls
@@ -515,7 +523,7 @@ class MistralToolParser(ToolParser):
523
type="function",
524
function=FunctionCall(
525
name=tool_call["name"],
518
- arguments=tool_call["arguments"],
526
+ arguments=tool_call.get("arguments", "{}"),
527
),
528
)
529
for tool_call in tool_calls