@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c0722f22de

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 5, 02:15 UTC

[Mistral Grammar] Fix tool and reasoning parsing (#39217)

Signed-off-by: juliendenize <julien.denize@mistral.ai>

Julien Denize committed Apr 16, 2026 at 06:05 UTC c0722f22de7159c7ed91fa8268bcecd87b35fe9e

10 files changed +1601 -266

tests/tool_parsers/test_mistral_tool_parser.py

+752 -180

@@ -3,6 +3,7 @@

4 import json

5 from collections.abc import Generator

6 +from typing import Any

7 from unittest.mock import MagicMock, patch

9 import partial_json_parser

@@ -23,24 +24,33 @@ from mistral_common.protocol.instruct.tool_calls import (

24 ToolChoiceEnum as MistralToolChoiceEnum,

25 )

26 from partial_json_parser.core.options import Allow

27 +from pydantic import ValidationError

29 from vllm.entrypoints.openai.chat_completion.protocol import (

30 ChatCompletionRequest,

31 )

32 from vllm.entrypoints.openai.engine.protocol import (

33 + DeltaFunctionCall,

34 DeltaMessage,

35 DeltaToolCall,

36 + ExtractedToolCallInformation,

37 StructuralTagResponseFormat,

38 )

39 +from vllm.entrypoints.openai.engine.protocol import FunctionCall as VllmFunctionCall

40 +from vllm.reasoning.mistral_reasoning_parser import MistralReasoningParser

41 from vllm.sampling_params import StructuredOutputsParams

42 from vllm.tokenizers import TokenizerLike, get_tokenizer

43 from vllm.tokenizers.detokenizer_utils import detokenize_incrementally

44 from vllm.tokenizers.mistral import MistralTokenizer

45 from vllm.tool_parsers.mistral_tool_parser import (

46 _DEFAULT_JSON_SCHEMA,

47 + MistralStreamingResult,

48 + MistralToolCall,

49 MistralToolParser,

50 )

52 +_DUMMY_REQUEST = ChatCompletionRequest(messages=[], model="test")

53 +

55 @pytest.fixture(scope="module")

56 def mistral_pre_v11_tokenizer():

@@ -205,7 +215,7 @@ def stream_delta_message_generator(

215 previous_token_ids,

216 current_token_ids,

217 delta_token_ids,

208 - request=None, # type: ignore[arg-type]

218 + request=_DUMMY_REQUEST,

219 )

220 if delta_message:

221 yield delta_message

@@ -218,14 +228,18 @@ def stream_delta_message_generator(

228 read_offset = new_read_offset

229

230

221 -def test_extract_tool_calls_no_tools(mistral_pre_v11_tool_parser):

231 +@pytest.mark.parametrize(

232 + "parser_fixture",

233 + ["mistral_pre_v11_tool_parser", "mistral_tool_parser"],

234 + ids=["pre_v11", "v11"],

235 +)

236 +def test_extract_tool_calls_no_tools(parser_fixture, request):

237 + parser = request.getfixturevalue(parser_fixture)

238 model_output = "This is a test"

223 - extracted_tool_calls = mistral_pre_v11_tool_parser.extract_tool_calls(

224 - model_output, request=None

225 - ) # type: ignore[arg-type]

226 - assert not extracted_tool_calls.tools_called

227 - assert extracted_tool_calls.tool_calls == []

228 - assert extracted_tool_calls.content == model_output

239 + result = parser.extract_tool_calls(model_output, request=_DUMMY_REQUEST)

240 + assert result == ExtractedToolCallInformation(

241 + tools_called=False, tool_calls=[], content=model_output

242 + )

243

244

245 @pytest.mark.parametrize(

@@ -234,6 +248,8 @@ def test_extract_tool_calls_no_tools(mistral_pre_v11_tool_parser):

248 "single_tool_weather",

249 "argument_before_name",

250 "argument_before_name_and_name_in_argument",

251 + "multiple_tools",

252 + "content_before_tool",

253 ],

254 argnames=["model_output", "expected_tool_calls", "expected_content"],

255 argvalues=[

@@ -292,14 +308,44 @@ def test_extract_tool_calls_no_tools(mistral_pre_v11_tool_parser):

308 ],

309 None,

310 ),

311 + (

312 + """[TOOL_CALLS] [{"name": "add", "arguments": {"a": 3.5, "b": 4}}, {"name": "get_current_weather", "arguments":{"city": "San Francisco", "state": "CA", "unit": "celsius"}}]""", # noqa: E501

313 + [

314 + ToolCall(

315 + function=FunctionCall(

316 + name="add", arguments=json.dumps({"a": 3.5, "b": 4})

317 + )

318 + ),

319 + ToolCall(

320 + function=FunctionCall(

321 + name="get_current_weather",

322 + arguments=json.dumps(

323 + {"city": "San Francisco", "state": "CA", "unit": "celsius"}

324 + ),

325 + )

326 + ),

327 + ],

328 + None,

329 + ),

330 + (

331 + """Hello[TOOL_CALLS] [{"name": "add", "arguments":{"a": 1, "b": 2}}]""", # noqa: E501

332 + [

333 + ToolCall(

334 + function=FunctionCall(

335 + name="add", arguments=json.dumps({"a": 1, "b": 2})

336 + )

337 + )

338 + ],

339 + "Hello",

340 + ),

341 ],

342 )

343 def test_extract_tool_calls_pre_v11_tokenizer(

344 mistral_pre_v11_tool_parser, model_output, expected_tool_calls, expected_content

345 ):

346 extracted_tool_calls = mistral_pre_v11_tool_parser.extract_tool_calls(

301 - model_output, request=None

302 - ) # type: ignore[arg-type]

347 + model_output, request=_DUMMY_REQUEST

348 + )

349 assert extracted_tool_calls.tools_called

350

351 assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls)

@@ -307,6 +353,46 @@ def test_extract_tool_calls_pre_v11_tokenizer(

353 assert extracted_tool_calls.content == expected_content

354

355

356 +def test_extract_tool_calls_pre_v11_multiple_bot_tokens_raises(

357 + mistral_pre_v11_tool_parser,

358 +):

359 + model_output = (

360 + '[TOOL_CALLS] [{"name": "add", "arguments":{"a": 1}}]'

361 + '[TOOL_CALLS] [{"name": "sub", "arguments":{"b": 2}}]'

362 + )

363 + with pytest.raises(ValueError, match="Only one BOT token"):

364 + mistral_pre_v11_tool_parser.extract_tool_calls(

365 + model_output, request=_DUMMY_REQUEST

366 + )

367 +

368 +

369 +def test_extract_tool_calls_pre_v11_regex_fallback_raises(

370 + mistral_pre_v11_tool_parser,

371 +):

372 + """The regex fallback path finds valid JSON but does not re-serialize

373 + the `arguments` dict to a string, causing a Pydantic

374 + `ValidationError` when constructing `FunctionCall`."""

375 + model_output = (

376 + '[TOOL_CALLS] junk [{"name": "add", "arguments":{"a": 1, "b": 2}}] trail'

377 + )

378 + with pytest.raises(ValidationError):

379 + mistral_pre_v11_tool_parser.extract_tool_calls(

380 + model_output, request=_DUMMY_REQUEST

381 + )

382 +

383 +

384 +def test_extract_tool_calls_pre_v11_regex_fallback_fails(

385 + mistral_pre_v11_tool_parser,

386 +):

387 + model_output = "[TOOL_CALLS] not json at all"

388 + result = mistral_pre_v11_tool_parser.extract_tool_calls(

389 + model_output, request=_DUMMY_REQUEST

390 + )

391 + assert result == ExtractedToolCallInformation(

392 + tools_called=False, tool_calls=[], content="not json at all"

393 + )

394 +

395 +

396 @pytest.mark.parametrize(

397 ids=[

398 "single_tool_add",

@@ -395,8 +481,8 @@ def test_extract_tool_calls(

481 mistral_tool_parser, model_output, expected_tool_calls, expected_content

482 ):

483 extracted_tool_calls = mistral_tool_parser.extract_tool_calls(

398 - model_output, request=None

399 - ) # type: ignore[arg-type]

484 + model_output, request=_DUMMY_REQUEST

485 + )

486 assert extracted_tool_calls.tools_called

487

488 assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls)

@@ -404,6 +490,16 @@ def test_extract_tool_calls(

490 assert extracted_tool_calls.content == expected_content

491

492

493 +def test_extract_tool_calls_v11_without_args_skipped(mistral_tool_parser):

494 + model_output = "[TOOL_CALLS]toolname_no_args"

495 + result = mistral_tool_parser.extract_tool_calls(

496 + model_output, request=_DUMMY_REQUEST

497 + )

498 + assert result == ExtractedToolCallInformation(

499 + tools_called=True, tool_calls=[], content=None

500 + )

501 +

502 +

503 def _test_extract_tool_calls_streaming(

504 tool_parser, tokenizer, model_output, tools, expected_tool_calls, expected_content

505 ):

@@ -669,17 +765,65 @@ def test_extract_tool_calls_streaming(

765 )

766

767

768 +def test_extract_tool_calls_streaming_v11_no_tools(

769 + mistral_tool_parser, mistral_tokenizer

770 +):

771 + model_output = "This is a test"

772 + if isinstance(mistral_tokenizer, MistralTokenizer):

773 + all_token_ids = mistral_tokenizer.encode(model_output)

774 + else:

775 + all_token_ids = mistral_tokenizer.encode(model_output, add_special_tokens=False)

776 + skip_special = isinstance(mistral_tokenizer, MistralTokenizer)

777 + collected_content = ""

778 + previous_text = ""

779 + previous_tokens = None

780 + prefix_offset = 0

781 + read_offset = 0

782 + for i in range(len(all_token_ids)):

783 + current_token_ids = all_token_ids[: i + 1]

784 + previous_token_ids = all_token_ids[:i]

785 + delta_token_ids = [all_token_ids[i]]

786 +

787 + new_tokens, delta_text, prefix_offset, read_offset = detokenize_incrementally(

788 + tokenizer=mistral_tokenizer,

789 + all_input_ids=current_token_ids,

790 + prev_tokens=previous_tokens,

791 + prefix_offset=prefix_offset,

792 + read_offset=read_offset,

793 + skip_special_tokens=skip_special,

794 + spaces_between_special_tokens=True,

795 + )

796 + current_text = previous_text + delta_text

797 + previous_tokens = (

798 + previous_tokens + new_tokens if previous_tokens else new_tokens

799 + )

800 +

801 + delta_message = mistral_tool_parser.extract_tool_calls_streaming(

802 + previous_text=previous_text,

803 + current_text=current_text,

804 + delta_text=delta_text,

805 + previous_token_ids=previous_token_ids,

806 + current_token_ids=current_token_ids,

807 + delta_token_ids=delta_token_ids,

808 + request=_DUMMY_REQUEST,

809 + )

810 + if delta_message and delta_message.content:

811 + collected_content += delta_message.content

812 + if delta_message:

813 + assert not delta_message.tool_calls

814 +

815 + previous_text = current_text

816 +

817 + assert collected_content == model_output

818 +

819 +

820 @pytest.mark.parametrize(

673 - ids=[

674 - "single_tool_add",

675 - "single_tool_weather",

676 - "multiple_tool_calls",

677 - "content_before_tool",

678 - "complex",

679 - ],

680 - argnames=["model_output", "expected_tool_calls", "expected_content"],

681 - argvalues=[

682 - (

821 + "parser_fixture, tokenizer_fixture, model_output,"

822 + " expected_tool_calls, expected_content",

823 + [

824 + pytest.param(

825 + "mistral_tool_parser",

826 + "mistral_tokenizer",

827 """[TOOL_CALLS]add_this_and_that{"a": 3.5, "b": 4}""", # noqa: E501

828 [

829 ToolCall(

@@ -690,8 +834,11 @@ def test_extract_tool_calls_streaming(

834 )

835 ],

836 "",

837 + id="v11-single_tool_add",

838 ),

694 - (

839 + pytest.param(

840 + "mistral_tool_parser",

841 + "mistral_tokenizer",

842 """[TOOL_CALLS]get_current_weather{"city": "San Francisco", "state": "CA", "unit": "celsius"}""", # noqa: E501

843 [

844 ToolCall(

@@ -704,8 +851,11 @@ def test_extract_tool_calls_streaming(

851 )

852 ],

853 "",

854 + id="v11-single_tool_weather",

855 ),

708 - (

856 + pytest.param(

857 + "mistral_tool_parser",

858 + "mistral_tokenizer",

859 """[TOOL_CALLS]add{"a": 3.5, "b": 4}[TOOL_CALLS]multiply{"a": 3, "b": 6}""", # noqa: E501

860 [

861 ToolCall(

@@ -720,9 +870,11 @@ def test_extract_tool_calls_streaming(

870 ),

871 ],

872 "",

873 + id="v11-multiple_tool_calls",

874 ),

724 - (

725 - # Additional content should not be after the tool calls

875 + pytest.param(

876 + "mistral_tool_parser",

877 + "mistral_tokenizer",

878 """bla[TOOL_CALLS]add_this_and_that{"a": 3.5, "b": 4}""", # noqa: E501

879 [

880 ToolCall(

@@ -733,9 +885,11 @@ def test_extract_tool_calls_streaming(

885 )

886 ],

887 "bla",

888 + id="v11-content_before_tool",

889 ),

737 - (

738 - # Complex

890 + pytest.param(

891 + "mistral_tool_parser",

892 + "mistral_tokenizer",

893 """hi{hi[TOOL_CALLS]bash{"command": "print(\\"hello world!\\")\\nre.compile(r\'{}\')"}""", # noqa: E501

894 [

895 ToolCall(

@@ -748,58 +902,19 @@ def test_extract_tool_calls_streaming(

902 )

903 ],

904 "hi{hi",

905 + id="v11-complex",

906 ),

752 - ],

753 -)

754 -def test_extract_tool_calls_streaming_one_chunk(

755 - mistral_tool_parser,

756 - mistral_tokenizer,

757 - model_output,

758 - expected_tool_calls,

759 - expected_content,

760 -):

761 - if isinstance(mistral_tokenizer, MistralTokenizer):

762 - all_token_ids = mistral_tokenizer.encode(model_output)

763 - else:

764 - all_token_ids = mistral_tokenizer.encode(model_output, add_special_tokens=False)

765 - all_token_ids = fix_tool_call_tokenization(

766 - all_token_ids, mistral_tool_parser, mistral_tokenizer

767 - )

768 -

769 - delta_message = mistral_tool_parser.extract_tool_calls_streaming(

770 - previous_text="",

771 - current_text=model_output,

772 - delta_text=model_output,

773 - previous_token_ids=[],

774 - current_token_ids=all_token_ids,

775 - delta_token_ids=all_token_ids,

776 - request=None,

777 - ) # type: ignore[arg-type]

778 - assert isinstance(delta_message, DeltaMessage)

779 - assert len(delta_message.tool_calls) == len(expected_tool_calls)

780 -

781 - assert_tool_calls(delta_message.tool_calls, expected_tool_calls)

782 -

783 - if delta_message.content is None:

784 - assert expected_content == ""

785 - else:

786 - assert delta_message.content == expected_content

787 -

788 -

789 -@pytest.mark.parametrize(

790 - ids=[

791 - "no_tools",

792 - "single_tool_add",

793 - "single_tool_add_strings",

794 - "single_tool_weather",

795 - "argument_before_name",

796 - "argument_before_name_and_name_in_argument",

797 - "multiple_tools",

798 - ],

799 - argnames=["model_output", "expected_tool_calls", "expected_content"],

800 - argvalues=[

801 - ("""This is a test""", [], """This is a test"""),

802 - (

907 + pytest.param(

908 + "mistral_pre_v11_tool_parser",

909 + "mistral_pre_v11_tokenizer",

910 + """This is a test""",

911 + [],

912 + """This is a test""",

913 + id="pre_v11-no_tools",

914 + ),

915 + pytest.param(

916 + "mistral_pre_v11_tool_parser",

917 + "mistral_pre_v11_tokenizer",

918 """[TOOL_CALLS] [ {"name":"add" , "arguments" : {"a": 3, "b": 4} } ]""", # noqa: E501

919 [

920 ToolCall(

@@ -809,8 +924,11 @@ def test_extract_tool_calls_streaming_one_chunk(

924 )

925 ],

926 "",

927 + id="pre_v11-single_tool_add",

928 ),

813 - (

929 + pytest.param(

930 + "mistral_pre_v11_tool_parser",

931 + "mistral_pre_v11_tokenizer",

932 """[TOOL_CALLS] [{"name": "add", "arguments":{"a": "3", "b": "4"}}]""", # noqa: E501

933 [

934 ToolCall(

@@ -820,8 +938,11 @@ def test_extract_tool_calls_streaming_one_chunk(

938 )

939 ],

940 "",

941 + id="pre_v11-single_tool_add_strings",

942 ),

824 - (

943 + pytest.param(

944 + "mistral_pre_v11_tool_parser",

945 + "mistral_pre_v11_tokenizer",

946 """[TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"city": "San Francisco", "state": "CA", "unit": "celsius"}}]""", # noqa: E501

947 [

948 ToolCall(

@@ -834,8 +955,11 @@ def test_extract_tool_calls_streaming_one_chunk(

955 )

956 ],

957 "",

958 + id="pre_v11-single_tool_weather",

959 ),

838 - (

960 + pytest.param(

961 + "mistral_pre_v11_tool_parser",

962 + "mistral_pre_v11_tokenizer",

963 """[TOOL_CALLS] [{"arguments": {"city": "San Francisco", "state": "CA", "unit": "celsius"}, "name": "get_current_weather"}]""", # noqa: E501

964 [

965 ToolCall(

@@ -848,8 +972,11 @@ def test_extract_tool_calls_streaming_one_chunk(

972 )

973 ],

974 "",

975 + id="pre_v11-argument_before_name",

976 ),

852 - (

977 + pytest.param(

978 + "mistral_pre_v11_tool_parser",

979 + "mistral_pre_v11_tokenizer",

980 """[TOOL_CALLS] [{"arguments": {"name": "John Doe"}, "name": "get_age"}]""", # noqa: E501

981 [

982 ToolCall(

@@ -864,8 +991,11 @@ def test_extract_tool_calls_streaming_one_chunk(

991 )

992 ],

993 "",

994 + id="pre_v11-argument_before_name_and_name_in_argument",

995 ),

868 - (

996 + pytest.param(

997 + "mistral_pre_v11_tool_parser",

998 + "mistral_pre_v11_tokenizer",

999 """[TOOL_CALLS] [{"arguments": {"a": 3.5, "b": 4}, "name": "add"}, {"arguments":{"city": "San Francisco", "state": "CA", "unit": "celsius"}, "name": "get_current_weather"}]""", # noqa: E501

1000 [

1001 ToolCall(

@@ -883,35 +1013,50 @@ def test_extract_tool_calls_streaming_one_chunk(

1013 ),

1014 ],

1015 "",

1016 + id="pre_v11-multiple_tools",

1017 + ),

1018 + pytest.param(

1019 + "mistral_pre_v11_tool_parser",

1020 + "mistral_pre_v11_tokenizer",

1021 + """Some text[TOOL_CALLS] [{"name": "add", "arguments":{"a": 1, "b": 2}}]""", # noqa: E501

1022 + [

1023 + ToolCall(

1024 + function=FunctionCall(

1025 + name="add", arguments=json.dumps({"a": 1, "b": 2})

1026 + )

1027 + )

1028 + ],

1029 + "Some text",

1030 + id="pre_v11-content_before_tool",

1031 ),

1032 ],

1033 )

889 -def test_extract_tool_calls_streaming_pre_v11_tokenizer_one_chunk(

890 - mistral_pre_v11_tool_parser,

891 - mistral_pre_v11_tokenizer,

1034 +def test_extract_tool_calls_streaming_one_chunk(

1035 + parser_fixture,

1036 + tokenizer_fixture,

1037 model_output,

1038 expected_tool_calls,

1039 expected_content,

1040 + request,

1041 ):

896 - if isinstance(mistral_pre_v11_tokenizer, MistralTokenizer):

897 - all_token_ids = mistral_pre_v11_tokenizer.encode(model_output)

1042 + tool_parser = request.getfixturevalue(parser_fixture)

1043 + tokenizer = request.getfixturevalue(tokenizer_fixture)

1044 +

1045 + if isinstance(tokenizer, MistralTokenizer):

1046 + all_token_ids = tokenizer.encode(model_output)

1047 else:

899 - all_token_ids = mistral_pre_v11_tokenizer.encode(

900 - model_output, add_special_tokens=False

901 - )

902 - all_token_ids = fix_tool_call_tokenization(

903 - all_token_ids, mistral_pre_v11_tool_parser, mistral_pre_v11_tokenizer

904 - )

1048 + all_token_ids = tokenizer.encode(model_output, add_special_tokens=False)

1049 + all_token_ids = fix_tool_call_tokenization(all_token_ids, tool_parser, tokenizer)

1050

906 - delta_message = mistral_pre_v11_tool_parser.extract_tool_calls_streaming(

1051 + delta_message = tool_parser.extract_tool_calls_streaming(

1052 previous_text="",

1053 current_text=model_output,

1054 delta_text=model_output,

1055 previous_token_ids=[],

1056 current_token_ids=all_token_ids,

1057 delta_token_ids=all_token_ids,

913 - request=None,

914 - ) # type: ignore[arg-type]

1058 + request=_DUMMY_REQUEST,

1059 + )

1060 assert isinstance(delta_message, DeltaMessage)

1061 assert len(delta_message.tool_calls) == len(expected_tool_calls)

1062

@@ -923,65 +1068,105 @@ def test_extract_tool_calls_streaming_pre_v11_tokenizer_one_chunk(

1068 assert delta_message.content == expected_content

1069

1070

926 -def test_fast_detokenization_text_detection(mistral_tool_parser):

1071 +@pytest.mark.parametrize(

1072 + "parser_fixture, model_output, fake_count, two_phase",

1073 + [

1074 + pytest.param(

1075 + "mistral_tool_parser",

1076 + '[TOOL_CALLS]add{"a": 1, "b": 2}',

1077 + 20,

1078 + True,

1079 + id="v11",

1080 + ),

1081 + pytest.param(

1082 + "mistral_pre_v11_tool_parser",

1083 + '[TOOL_CALLS] [{"name": "add", "arguments":{"a": 1, "b": 2}}]',

1084 + 30,

1085 + False,

1086 + id="pre_v11",

1087 + ),

1088 + ],

1089 +)

1090 +def test_fast_detokenization_text_detection(

1091 + parser_fixture, model_output, fake_count, two_phase, request

1092 +):

1093 """Regression: bot_token in text but not token_ids (PR #37209)."""

928 - model_output = '[TOOL_CALLS]add{"a": 1, "b": 2}'

1094 + parser = request.getfixturevalue(parser_fixture)

1095 # Token IDs that do NOT contain bot_token_id.

930 - fake_token_ids = list(range(99, 99 + 20))

931 -

932 - # First delta: pure content, no bot token yet

933 - delta_message_before = mistral_tool_parser.extract_tool_calls_streaming(

934 - previous_text="",

935 - current_text="Hello",

936 - delta_text="Hello",

937 - previous_token_ids=[],

938 - current_token_ids=[99],

939 - delta_token_ids=[99],

940 - request=None,

941 - )

942 - assert delta_message_before is not None

943 - assert delta_message_before.content == "Hello"

944 - assert not delta_message_before.tool_calls

945 -

946 - # Second delta: bot token in text but NOT in token_ids

947 - delta_message = mistral_tool_parser.extract_tool_calls_streaming(

948 - previous_text="Hello",

949 - current_text="Hello" + model_output,

1096 + fake_token_ids = list(range(99, 99 + fake_count))

1097 +

1098 + if two_phase:

1099 + # First delta: pure content, no bot token yet

1100 + delta_message_before = parser.extract_tool_calls_streaming(

1101 + previous_text="",

1102 + current_text="Hello",

1103 + delta_text="Hello",

1104 + previous_token_ids=[],

1105 + current_token_ids=[99],

1106 + delta_token_ids=[99],

1107 + request=_DUMMY_REQUEST,

1108 + )

1109 + assert delta_message_before is not None

1110 + assert delta_message_before.content == "Hello"

1111 + assert not delta_message_before.tool_calls

1112 +

1113 + previous_text = "Hello"

1114 + current_text = "Hello" + model_output

1115 + previous_token_ids = [99]

1116 + delta_token_ids = fake_token_ids[1:]

1117 + else:

1118 + previous_text = ""

1119 + current_text = model_output

1120 + previous_token_ids = []

1121 + delta_token_ids = fake_token_ids

1122 +

1123 + delta_message = parser.extract_tool_calls_streaming(

1124 + previous_text=previous_text,

1125 + current_text=current_text,

1126 delta_text=model_output,

951 - previous_token_ids=[99],

1127 + previous_token_ids=previous_token_ids,

1128 current_token_ids=fake_token_ids,

953 - delta_token_ids=fake_token_ids[1:],

954 - request=None,

1129 + delta_token_ids=delta_token_ids,

1130 + request=_DUMMY_REQUEST,

1131 )

1132 assert delta_message is not None

1133 assert delta_message.tool_calls is not None

958 - assert len(delta_message.tool_calls) > 0

1134 + assert len(delta_message.tool_calls) == 1

1135 assert delta_message.tool_calls[0].function is not None

1136 assert delta_message.tool_calls[0].function.name == "add"

1137

1138

963 -def test_fast_detokenization_text_detection_pre_v11(

964 - mistral_pre_v11_tool_parser,

1139 +@pytest.mark.parametrize(

1140 + "parser_fixture, patched_method, current_text",

1141 + [

1142 + (

1143 + "mistral_tool_parser",

1144 + "_extract_tool_calls_streaming",

1145 + "[TOOL_CALLS]add{}",

1146 + ),

1147 + (

1148 + "mistral_pre_v11_tool_parser",

1149 + "_extract_tool_calls_streaming_pre_v11_tokenizer",

1150 + '[TOOL_CALLS] [{"name":"a","arguments":{}}]',

1151 + ),

1152 + ],

1153 + ids=["v11", "pre_v11"],

1154 +)

1155 +def test_extract_tool_calls_streaming_exception_returns_none(

1156 + parser_fixture, patched_method, current_text, request

1157 ):

966 - """Regression: bot_token text detection for pre-v11 tokenizer (PR #37209)."""

967 - model_output = '[TOOL_CALLS] [{"name": "add", "arguments":{"a": 1, "b": 2}}]'

968 -

969 - fake_token_ids = list(range(99, 99 + 30))

970 -

971 - delta_message = mistral_pre_v11_tool_parser.extract_tool_calls_streaming(

972 - previous_text="",

973 - current_text=model_output,

974 - delta_text=model_output,

975 - previous_token_ids=[],

976 - current_token_ids=fake_token_ids,

977 - delta_token_ids=fake_token_ids,

978 - request=None,

979 - )

980 - assert delta_message is not None

981 - assert delta_message.tool_calls is not None

982 - assert len(delta_message.tool_calls) > 0

983 - assert delta_message.tool_calls[0].function is not None

984 - assert delta_message.tool_calls[0].function.name == "add"

1158 + parser = request.getfixturevalue(parser_fixture)

1159 + with patch.object(parser, patched_method, side_effect=RuntimeError("boom")):

1160 + result = parser.extract_tool_calls_streaming(

1161 + previous_text="",

1162 + current_text=current_text,

1163 + delta_text=current_text,

1164 + previous_token_ids=[],

1165 + current_token_ids=[parser.bot_token_id],

1166 + delta_token_ids=[parser.bot_token_id],

1167 + request=_DUMMY_REQUEST,

1168 + )

1169 + assert result is None

1170

1171

1172 SAMPLE_TOOLS_DICTS = [

@@ -1238,57 +1423,444 @@ def test_adjust_request_response_format_generates_grammar(

1423 assert len(result.structured_outputs.grammar) > 0

1424

1425

1241 -def test_adjust_request_tool_choice_none_with_json_schema_uses_json_schema_factory(

1426 +@pytest.mark.parametrize(

1427 + "tool_choice, expected_method, not_called_method",

1428 + [

1429 + ("none", "get_lark_for_json_schema", None),

1430 + ("auto", "get_lark_from_jinja", "get_lark_for_json_schema"),

1431 + ],

1432 + ids=["none_uses_json_schema_factory", "auto_uses_jinja_factory"],

1433 +)

1434 +def test_adjust_request_tool_choice_with_json_schema_factory_routing(

1435 mistral_tool_parser: MistralToolParser,

1436 + tool_choice: str,

1437 + expected_method: str,

1438 + not_called_method: str | None,

1439 ) -> None:

1440 request = _make_request(

1245 - tool_choice="none",

1441 + tool_choice=tool_choice,

1442 structured_outputs=StructuredOutputsParams(json='{"type": "object"}'),

1443 )

1444 factory = mistral_tool_parser.model_tokenizer.grammar_factory

1445

1250 - with patch.object(

1251 - factory,

1252 - "get_lark_for_json_schema",

1253 - wraps=factory.get_lark_for_json_schema,

1254 - ) as mock_json_schema:

1255 - result = mistral_tool_parser.adjust_request(request)

1446 + patches = {

1447 + expected_method: patch.object(

1448 + factory,

1449 + expected_method,

1450 + wraps=getattr(factory, expected_method),

1451 + ),

1452 + }

1453 + if not_called_method:

1454 + patches[not_called_method] = patch.object(

1455 + factory,

1456 + not_called_method,

1457 + wraps=getattr(factory, not_called_method),

1458 + )

1459 +

1460 + with patches[expected_method] as mock_expected:

1461 + ctx = patches[not_called_method] if not_called_method else None

1462 + if ctx:

1463 + with ctx as mock_not_called:

1464 + result = mistral_tool_parser.adjust_request(request)

1465 + mock_not_called.assert_not_called()

1466 + else:

1467 + result = mistral_tool_parser.adjust_request(request)

1468

1257 - mock_json_schema.assert_called_once()

1258 - assert mock_json_schema.call_args.kwargs["json_schema"] == {"type": "object"}

1469 + mock_expected.assert_called_once()

1470 + assert mock_expected.call_args.kwargs["json_schema"] == {"type": "object"}

1471

1472 assert result.structured_outputs is not None

1473 assert isinstance(result.structured_outputs.grammar, str)

1474 assert len(result.structured_outputs.grammar) > 0

1475

1476

1265 -def test_adjust_request_tool_choice_auto_with_json_schema_uses_jinja_factory(

1477 +def test_grammar_from_tool_parser_default_false() -> None:

1478 + request = _make_request()

1479 + assert request._grammar_from_tool_parser is False

1480 +

1481 +

1482 +def test_grammar_from_tool_parser_set_by_adjust_request(

1483 mistral_tool_parser: MistralToolParser,

1484 ) -> None:

1268 - request = _make_request(

1269 - tool_choice="auto",

1270 - structured_outputs=StructuredOutputsParams(json='{"type": "object"}'),

1271 - )

1272 - factory = mistral_tool_parser.model_tokenizer.grammar_factory

1485 + request = _make_request()

1486 + result = mistral_tool_parser.adjust_request(request)

1487 + assert result._grammar_from_tool_parser is True

1488

1274 - with (

1275 - patch.object(

1276 - factory,

1277 - "get_lark_for_json_schema",

1278 - wraps=factory.get_lark_for_json_schema,

1279 - ) as mock_json_schema,

1280 - patch.object(

1281 - factory,

1282 - "get_lark_from_jinja",

1283 - wraps=factory.get_lark_from_jinja,

1284 - ) as mock_jinja,

1285 - ):

1286 - result = mistral_tool_parser.adjust_request(request)

1489

1288 - mock_jinja.assert_called_once()

1289 - assert mock_jinja.call_args.kwargs["json_schema"] == {"type": "object"}

1290 - mock_json_schema.assert_not_called()

1490 +@pytest.mark.parametrize(

1491 + "tool_calls, expected_len",

1492 + [

1493 + (None, 0),

1494 + ([], 0),

1495 + ([VllmFunctionCall(id="abc123xyz", name="f", arguments="{}")], 1),

1496 + ([VllmFunctionCall(name="f", arguments="{}")], 1),

1497 + (

1498 + [

1499 + VllmFunctionCall(id="fixed1234", name="a", arguments='{"x": 1}'),

1500 + VllmFunctionCall(name="b", arguments='{"y": 2}'),

1501 + ],

1502 + 2,

1503 + ),

1504 + ],

1505 + ids=["none", "empty", "with_id", "without_id", "mixed"],

1506 +)

1507 +def test_build_non_streaming_tool_calls(

1508 + tool_calls: list[VllmFunctionCall] | None,

1509 + expected_len: int,

1510 +) -> None:

1511 + result = MistralToolParser.build_non_streaming_tool_calls(tool_calls)

1512 + assert len(result) == expected_len

1513

1292 - assert result.structured_outputs is not None

1293 - assert isinstance(result.structured_outputs.grammar, str)

1294 - assert len(result.structured_outputs.grammar) > 0

1514 + if tool_calls is None:

1515 + return

1516 +

1517 + for i, tc in enumerate(result):

1518 + assert isinstance(tc, MistralToolCall)

1519 + assert tc.type == "function"

1520 +

1521 + input_tc = tool_calls[i]

1522 + if input_tc.id:

1523 + assert tc.id == input_tc.id

1524 + else:

1525 + assert len(tc.id) == 9

1526 + assert tc.id.isalnum()

1527 +

1528 + assert tc.function.name == input_tc.name

1529 + assert tc.function.arguments == input_tc.arguments

1530 +

1531 +

1532 +class TestExtractMaybeReasoningAndToolStreaming:

1533 + r"""Tests for `MistralToolParser.extract_maybe_reasoning_and_tool_streaming`."""

1534 +

1535 + @pytest.fixture

1536 + def parser(self) -> MistralToolParser:

1537 + mock_tokenizer = MagicMock()

1538 + mock_tokenizer.get_vocab.return_value = {"[TOOL_CALLS]": 1}

1539 + return MistralToolParser(mock_tokenizer)

1540 +

1541 + @pytest.fixture

1542 + def request_obj(self) -> ChatCompletionRequest:

1543 + return _make_request()

1544 +

1545 + @staticmethod

1546 + def _call(

1547 + parser: MistralToolParser,

1548 + request: ChatCompletionRequest,

1549 + *,

1550 + reasoning_parser: Any = None,

1551 + previous_text: str = "",

1552 + current_text: str = "hello",

1553 + delta_text: str = "hello",

1554 + previous_token_ids: list[int] | None = None,

1555 + current_token_ids: list[int] | None = None,

1556 + output_token_ids: list[int] | None = None,

1557 + reasoning_ended: bool = False,

1558 + prompt_is_reasoning_end: bool | None = None,

1559 + ) -> MistralStreamingResult:

1560 + return parser.extract_maybe_reasoning_and_tool_streaming(

1561 + reasoning_parser=reasoning_parser,

1562 + previous_text=previous_text,

1563 + current_text=current_text,

1564 + delta_text=delta_text,

1565 + previous_token_ids=previous_token_ids or [],

1566 + current_token_ids=current_token_ids or [1, 2, 3],

1567 + output_token_ids=output_token_ids or [1, 2, 3],

1568 + reasoning_ended=reasoning_ended,

1569 + prompt_is_reasoning_end=prompt_is_reasoning_end,

1570 + request=request,

1571 + )

1572 +

1573 + def test_no_reasoning_tools_called(

1574 + self, parser: MistralToolParser, request_obj: ChatCompletionRequest

1575 + ) -> None:

1576 + tool_delta = DeltaMessage(

1577 + tool_calls=[

1578 + DeltaToolCall(

1579 + index=0,

1580 + function=DeltaFunctionCall(name="f", arguments="{}"),

1581 + )

1582 + ]

1583 + )

1584 + with patch.object(

1585 + parser, "extract_tool_calls_streaming", return_value=tool_delta

1586 + ):

1587 + result = self._call(parser, request_obj, reasoning_parser=None)

1588 +

1589 + assert result == MistralStreamingResult(

1590 + delta_message=tool_delta,

1591 + reasoning_ended=False,

1592 + tools_called=True,

1593 + current_text="hello",

1594 + current_token_ids=[1, 2, 3],

1595 + )

1596 +

1597 + def test_no_reasoning_no_tools(

1598 + self, parser: MistralToolParser, request_obj: ChatCompletionRequest

1599 + ) -> None:

1600 + content_delta = DeltaMessage(content="hello")

1601 + with patch.object(

1602 + parser, "extract_tool_calls_streaming", return_value=content_delta

1603 + ):

1604 + result = self._call(parser, request_obj, reasoning_parser=None)

1605 +

1606 + assert result == MistralStreamingResult(

1607 + delta_message=content_delta,

1608 + reasoning_ended=False,

1609 + tools_called=False,

1610 + current_text="hello",

1611 + current_token_ids=[1, 2, 3],

1612 + )

1613 +

1614 + def test_mistral_reasoning_parser_no_think_token(

1615 + self, parser: MistralToolParser, request_obj: ChatCompletionRequest

1616 + ) -> None:

1617 + mock_rp = MagicMock(spec=MistralReasoningParser)

1618 + mock_rp.start_token_id = 999

1619 + content_delta = DeltaMessage(content="direct")

1620 + with patch.object(

1621 + parser, "extract_tool_calls_streaming", return_value=content_delta

1622 + ):

1623 + result = self._call(

1624 + parser,

1625 + request_obj,

1626 + reasoning_parser=mock_rp,

1627 + reasoning_ended=False,

1628 + current_token_ids=[1, 2, 3],

1629 + )

1630 +

1631 + mock_rp.extract_reasoning_streaming.assert_not_called()

1632 + assert result == MistralStreamingResult(

1633 + delta_message=content_delta,

1634 + reasoning_ended=False,

1635 + tools_called=False,

1636 + current_text="hello",

1637 + current_token_ids=[1, 2, 3],

1638 + )

1639 +

1640 + def test_mistral_reasoning_parser_with_think_token(

1641 + self, parser: MistralToolParser, request_obj: ChatCompletionRequest

1642 + ) -> None:

1643 + mock_rp = MagicMock(spec=MistralReasoningParser)

1644 + mock_rp.start_token_id = 999

1645 + mock_rp.extract_reasoning_streaming.return_value = DeltaMessage(

1646 + reasoning="thinking..."

1647 + )

1648 + mock_rp.is_reasoning_end_streaming.return_value = False

1649 +

1650 + result = self._call(

1651 + parser,

1652 + request_obj,

1653 + reasoning_parser=mock_rp,

1654 + reasoning_ended=False,

1655 + current_token_ids=[1, 999, 3],

1656 + )

1657 +

1658 + mock_rp.extract_reasoning_streaming.assert_called_once()

1659 + assert result == MistralStreamingResult(

1660 + delta_message=DeltaMessage(reasoning="thinking..."),

1661 + reasoning_ended=False,

1662 + tools_called=False,

1663 + current_text="hello",

1664 + current_token_ids=[1, 999, 3],

1665 + )

1666 +

1667 + def test_non_mistral_reasoning_parser_always_expects_thinking(

1668 + self, parser: MistralToolParser, request_obj: ChatCompletionRequest

1669 + ) -> None:

1670 + mock_rp = MagicMock()

1671 + mock_rp.start_token_id = 999

1672 + mock_rp.extract_reasoning_streaming.return_value = DeltaMessage(

1673 + reasoning="thinking..."

1674 + )

1675 + mock_rp.is_reasoning_end_streaming.return_value = False

1676 +

1677 + result = self._call(

1678 + parser,

1679 + request_obj,

1680 + reasoning_parser=mock_rp,

1681 + reasoning_ended=False,

1682 + current_token_ids=[1, 2, 3],

1683 + )

1684 +

1685 + mock_rp.extract_reasoning_streaming.assert_called_once()

1686 + assert result == MistralStreamingResult(

1687 + delta_message=DeltaMessage(reasoning="thinking..."),

1688 + reasoning_ended=False,

1689 + tools_called=False,

1690 + current_text="hello",

1691 + current_token_ids=[1, 2, 3],

1692 + )

1693 +

1694 + def test_reasoning_already_ended_no_reset(

1695 + self, parser: MistralToolParser, request_obj: ChatCompletionRequest

1696 + ) -> None:

1697 + content_delta = DeltaMessage(content="content")

1698 + with patch.object(

1699 + parser, "extract_tool_calls_streaming", return_value=content_delta

1700 + ) as mock_extract:

1701 + result = self._call(

1702 + parser,

1703 + request_obj,

1704 + reasoning_parser=MagicMock(),

1705 + reasoning_ended=True,

1706 + previous_text="prior_tool_text",

1707 + previous_token_ids=[10, 20],

1708 + current_text="prior_tool_texthello",

1709 + current_token_ids=[10, 20, 1, 2, 3],

1710 + )

1711 +

1712 + _, call_kwargs = mock_extract.call_args

1713 + assert call_kwargs["previous_text"] == "prior_tool_text"

1714 + assert call_kwargs["previous_token_ids"] == [10, 20]

1715 +

1716 + assert result == MistralStreamingResult(

1717 + delta_message=content_delta,

1718 + reasoning_ended=True,

1719 + tools_called=False,

1720 + current_text="prior_tool_texthello",

1721 + current_token_ids=[10, 20, 1, 2, 3],

1722 + )

1723 +

1724 + def test_pre_v15_ignores_prompt_reasoning_end(

1725 + self, parser: MistralToolParser, request_obj: ChatCompletionRequest

1726 + ) -> None:

1727 + mock_tokenizer = MagicMock(spec=MistralTokenizer)

1728 + mock_tokenizer.version = 13

1729 + parser.model_tokenizer = mock_tokenizer

1730 +

1731 + mock_rp = MagicMock(spec=MistralReasoningParser)

1732 + mock_rp.start_token_id = 999

1733 + mock_rp.extract_reasoning_streaming.return_value = DeltaMessage(

1734 + reasoning="thinking..."

1735 + )

1736 + mock_rp.is_reasoning_end_streaming.return_value = False

1737 +

1738 + result = self._call(

1739 + parser,

1740 + request_obj,

1741 + reasoning_parser=mock_rp,

1742 + reasoning_ended=False,

1743 + prompt_is_reasoning_end=True,

1744 + current_token_ids=[999, 1, 2],

1745 + )

1746 +

1747 + mock_rp.extract_reasoning_streaming.assert_called_once()

1748 + assert result == MistralStreamingResult(

1749 + delta_message=DeltaMessage(reasoning="thinking..."),

1750 + reasoning_ended=False,

1751 + tools_called=False,

1752 + current_text="hello",

1753 + current_token_ids=[999, 1, 2],

1754 + )

1755 +

1756 + def test_non_pre_v15_prompt_reasoning_end(

1757 + self, parser: MistralToolParser, request_obj: ChatCompletionRequest

1758 + ) -> None:

1759 + mock_tokenizer = MagicMock(spec=MistralTokenizer)

1760 + mock_tokenizer.version = 15

1761 + parser.model_tokenizer = mock_tokenizer

1762 +

1763 + mock_rp = MagicMock(spec=MistralReasoningParser)

1764 + mock_rp.start_token_id = 999

1765 +

1766 + content_delta = DeltaMessage(content="after reasoning")

1767 + with patch.object(

1768 + parser, "extract_tool_calls_streaming", return_value=content_delta

1769 + ):

1770 + result = self._call(

1771 + parser,

1772 + request_obj,

1773 + reasoning_parser=mock_rp,

1774 + reasoning_ended=False,

1775 + prompt_is_reasoning_end=True,

1776 + current_token_ids=[999, 1, 2],

1777 + output_token_ids=[10, 20, 30],

1778 + )

1779 +

1780 + mock_rp.extract_reasoning_streaming.assert_not_called()

1781 + assert result == MistralStreamingResult(

1782 + delta_message=content_delta,

1783 + reasoning_ended=True,

1784 + tools_called=False,

1785 + current_text="hello",

1786 + current_token_ids=[10, 20, 30],

1787 + )

1788 +

1789 + def test_reasoning_end_transition_with_content(

1790 + self, parser: MistralToolParser, request_obj: ChatCompletionRequest

1791 + ) -> None:

1792 + """When reasoning ends and the delta has content, that content is

1793 + cleared from delta_message and used as current_text for tool parsing."""

1794 + mock_rp = MagicMock()

1795 + mock_rp.start_token_id = 999

1796 + mock_rp.extract_reasoning_streaming.return_value = DeltaMessage(

1797 + reasoning="think", content="leftover"

1798 + )

1799 + mock_rp.is_reasoning_end_streaming.return_value = True

1800 + mock_rp.extract_content_ids.return_value = [50, 51]

1801 +

1802 + content_delta = DeltaMessage(content="leftover")

1803 + with patch.object(

1804 + parser, "extract_tool_calls_streaming", return_value=content_delta

1805 + ) as mock_extract:

1806 + result = self._call(

1807 + parser,

1808 + request_obj,

1809 + reasoning_parser=mock_rp,

1810 + reasoning_ended=False,

1811 + current_token_ids=[999, 1, 2],

1812 + output_token_ids=[10, 20, 30],

1813 + )

1814 +

1815 + mock_rp.extract_content_ids.assert_called_once_with([10, 20, 30])

1816 + _, call_kwargs = mock_extract.call_args

1817 + assert call_kwargs["previous_text"] == ""

1818 + assert call_kwargs["previous_token_ids"] == []

1819 + assert call_kwargs["delta_text"] == "leftover"

1820 + assert call_kwargs["current_token_ids"] == [50, 51]

1821 +

1822 + assert result == MistralStreamingResult(

1823 + delta_message=content_delta,

1824 + reasoning_ended=True,

1825 + tools_called=False,

1826 + current_text="leftover",

1827 + current_token_ids=[50, 51],

1828 + )

1829 +

1830 + def test_reasoning_end_transition_without_content(

1831 + self, parser: MistralToolParser, request_obj: ChatCompletionRequest

1832 + ) -> None:

1833 + """When reasoning ends but the delta has no content, current_text

1834 + is set to empty string."""

1835 + mock_rp = MagicMock()

1836 + mock_rp.start_token_id = 999

1837 + mock_rp.extract_reasoning_streaming.return_value = DeltaMessage(

1838 + reasoning="think"

1839 + )

1840 + mock_rp.is_reasoning_end_streaming.return_value = True

1841 + mock_rp.extract_content_ids.return_value = [50, 51]

1842 +

1843 + empty_delta = DeltaMessage(content="")

1844 + with patch.object(

1845 + parser, "extract_tool_calls_streaming", return_value=empty_delta

1846 + ) as mock_extract:

1847 + result = self._call(

1848 + parser,

1849 + request_obj,

1850 + reasoning_parser=mock_rp,

1851 + reasoning_ended=False,

1852 + current_token_ids=[999, 1, 2],

1853 + output_token_ids=[10, 20, 30],

1854 + )

1855 +

1856 + _, call_kwargs = mock_extract.call_args

1857 + assert call_kwargs["delta_text"] == ""

1858 + assert call_kwargs["current_token_ids"] == [50, 51]

1859 +

1860 + assert result == MistralStreamingResult(

1861 + delta_message=empty_delta,

1862 + reasoning_ended=True,

1863 + tools_called=False,

1864 + current_text="",

1865 + current_token_ids=[50, 51],

1866 + )

tests/tool_use/mistral/test_mistral_tool_calls.py

+480 -3

@@ -1,25 +1,198 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 +import json

5 +from dataclasses import dataclass, field

6 +

7 import openai

8 import pytest

7 -from tests.tool_use.utils import MESSAGES_ASKING_FOR_TOOLS, WEATHER_TOOL

10 +from tests.tool_use.utils import (

11 + MESSAGES_ASKING_FOR_PARALLEL_TOOLS,

12 + MESSAGES_ASKING_FOR_TOOLS,

13 + MESSAGES_WITH_TOOL_RESPONSE,

14 + MESSAGES_WITHOUT_TOOLS,

15 + SEARCH_TOOL,

16 + SEED,

17 + WEATHER_TOOL,

18 + ensure_system_prompt,

19 +)

20 +

21 +from .utils import ServerConfig

22 +

23 +

24 +def _requires_tool_parser(server_config: ServerConfig) -> None:

25 + r"""Skip test if server was not started with --tool-call-parser."""

26 + if "--tool-call-parser" not in server_config.get("arguments", []):

27 + pytest.skip(

28 + f"Skipping: {server_config['model']} not configured with --tool-call-parser"

29 + )

30 +

31 +

32 +def _is_pre_v11(server_config: ServerConfig) -> bool:

33 + r"""Pre-v11 Mistral models lack grammar-based tool call enforcement."""

34 + return "7B" in server_config.get("model", "")

35 +

36 +

37 +@dataclass

38 +class StreamedToolCallResult:

39 + r"""Accumulated result from streaming a single tool call."""

40 +

41 + function_name: str | None = None

42 + function_args_str: str = ""

43 + tool_call_id: str | None = None

44 + role_name: str | None = None

45 + finish_reason_count: int = 0

46 + finish_reason: str | None = None

47 +

48 +

49 +async def _collect_streamed_tool_call(

50 + stream: openai.AsyncStream,

51 + *,

52 + expected_finish_reason: str = "tool_calls",

53 +) -> StreamedToolCallResult:

54 + result = StreamedToolCallResult()

55 +

56 + async for chunk in stream:

57 + if chunk.choices[0].finish_reason:

58 + result.finish_reason_count += 1

59 + result.finish_reason = chunk.choices[0].finish_reason

60 + assert chunk.choices[0].finish_reason == expected_finish_reason

61 +

62 + if chunk.choices[0].delta.role:

63 + assert not result.role_name or result.role_name == "assistant"

64 + result.role_name = "assistant"

65 +

66 + streamed_tool_calls = chunk.choices[0].delta.tool_calls

67 + if streamed_tool_calls and len(streamed_tool_calls) > 0:

68 + assert len(streamed_tool_calls) == 1

69 + tool_call = streamed_tool_calls[0]

70 +

71 + if tool_call.id:

72 + assert not result.tool_call_id

73 + result.tool_call_id = tool_call.id

74 +

75 + if tool_call.function:

76 + if tool_call.function.name:

77 + assert result.function_name is None

78 + result.function_name = tool_call.function.name

79 + if tool_call.function.arguments:

80 + result.function_args_str += tool_call.function.arguments

81 +

82 + return result

83 +

84 +

85 +@dataclass

86 +class StreamedContentResult:

87 + r"""Accumulated result from streaming a content-only response."""

88 +

89 + chunks: list[str] = field(default_factory=list)

90 + finish_reason_count: int = 0

91 + finish_reason: str | None = None

92 + role_sent: bool = False

93 +

94 +

95 +async def _collect_streamed_content(

96 + stream: openai.AsyncStream,

97 + *,

98 + expected_finish_reason: str | None = None,

99 + no_tool_calls: bool = True,

100 +) -> StreamedContentResult:

101 + r"""Consume a streaming response and collect text content."""

102 + result = StreamedContentResult()

103 +

104 + async for chunk in stream:

105 + delta = chunk.choices[0].delta

106 +

107 + if delta.role:

108 + assert not result.role_sent

109 + assert delta.role == "assistant"

110 + result.role_sent = True

111 +

112 + if delta.content:

113 + result.chunks.append(delta.content)

114 +

115 + if chunk.choices[0].finish_reason is not None:

116 + result.finish_reason_count += 1

117 + result.finish_reason = chunk.choices[0].finish_reason

118 + if expected_finish_reason is not None:

119 + assert result.finish_reason == expected_finish_reason

120 +

121 + if no_tool_calls:

122 + assert not delta.tool_calls or len(delta.tool_calls) == 0

123 +

124 + return result

125 +

126 +

127 +@dataclass

128 +class StreamedParallelToolCallResult:

129 + r"""Accumulated result from streaming parallel tool calls."""

130 +

131 + function_names: list[str] = field(default_factory=list)

132 + function_args_strs: list[str] = field(default_factory=list)

133 + tool_call_ids: list[str] = field(default_factory=list)

134 + role_name: str | None = None

135 + finish_reason_count: int = 0

136 +

137 +

138 +async def _collect_streamed_parallel_tool_calls(

139 + stream: openai.AsyncStream,

140 +) -> StreamedParallelToolCallResult:

141 + r"""Consume a streaming response and collect parallel tool calls."""

142 + result = StreamedParallelToolCallResult()

143 + tool_call_idx: int = -1

144 +

145 + async for chunk in stream:

146 + if chunk.choices[0].finish_reason:

147 + result.finish_reason_count += 1

148 + assert chunk.choices[0].finish_reason == "tool_calls"

149 +

150 + if chunk.choices[0].delta.role:

151 + assert not result.role_name or result.role_name == "assistant"

152 + result.role_name = "assistant"

153 +

154 + streamed_tool_calls = chunk.choices[0].delta.tool_calls

155 + if streamed_tool_calls and len(streamed_tool_calls) > 0:

156 + assert len(streamed_tool_calls) == 1

157 + tool_call = streamed_tool_calls[0]

158 +

159 + if tool_call.index != tool_call_idx:

160 + tool_call_idx = tool_call.index

161 + result.function_args_strs.append("")

162 + result.tool_call_ids.append("")

163 +

164 + if tool_call.id:

165 + result.tool_call_ids[tool_call.index] = tool_call.id

166 +

167 + if tool_call.function:

168 + if tool_call.function.name:

169 + result.function_names.append(tool_call.function.name)

170 + if tool_call.function.arguments:

171 + result.function_args_strs[tool_call.index] += (

172 + tool_call.function.arguments

173 + )

174 +

175 + return result

176

177

178 # test: a tool_choice with mistral-tokenizer results in an ID of length 9

179 @pytest.mark.asyncio

12 -async def test_tool_call_with_tool_choice(client: openai.AsyncOpenAI):

180 +async def test_tool_call_with_tool_choice(

181 + client: openai.AsyncOpenAI, server_config: ServerConfig

182 +) -> None:

183 + _requires_tool_parser(server_config)

184 +

185 models = await client.models.list()

186 model_name: str = models.data[0].id

187 chat_completion = await client.chat.completions.create(

16 - messages=MESSAGES_ASKING_FOR_TOOLS,

188 + messages=ensure_system_prompt(MESSAGES_ASKING_FOR_TOOLS, server_config),

189 temperature=0,

190 max_completion_tokens=100,

191 model=model_name,

192 tools=[WEATHER_TOOL],

193 tool_choice=WEATHER_TOOL,

194 logprobs=False,

195 + seed=SEED,

196 )

197

198 choice = chat_completion.choices[0]

@@ -28,3 +201,307 @@ async def test_tool_call_with_tool_choice(client: openai.AsyncOpenAI):

201 assert choice.message.role == "assistant"

202 assert choice.message.tool_calls is None or len(choice.message.tool_calls) == 1

203 assert len(choice.message.tool_calls[0].id) == 9 # length of 9 for mistral

204 +

205 +

206 +_NOT_SET = object()

207 +

208 +

209 +@pytest.mark.asyncio

210 +@pytest.mark.parametrize(

211 + "tools, tool_choice, streaming_id_len_pre_v11",

212 + [

213 + pytest.param(

214 + [WEATHER_TOOL, SEARCH_TOOL],

215 + _NOT_SET,

216 + 9,

217 + id="auto",

218 + ),

219 + pytest.param(

220 + [WEATHER_TOOL],

221 + "required",

222 + 30,

223 + id="required",

224 + ),

225 + ],

226 +)

227 +async def test_tool_call_auto_or_required(

228 + client: openai.AsyncOpenAI,

229 + server_config: ServerConfig,

230 + tools: list,

231 + tool_choice: object,

232 + streaming_id_len_pre_v11: int,

233 +) -> None:

234 + _requires_tool_parser(server_config)

235 +

236 + models = await client.models.list()

237 + model_name: str = models.data[0].id

238 +

239 + create_kwargs: dict = {

240 + "messages": ensure_system_prompt(MESSAGES_ASKING_FOR_TOOLS, server_config),

241 + "temperature": 0,

242 + "max_completion_tokens": 100,

243 + "model": model_name,

244 + "tools": tools,

245 + "logprobs": False,

246 + "seed": SEED,

247 + }

248 + if tool_choice is not _NOT_SET:

249 + create_kwargs["tool_choice"] = tool_choice

250 +

251 + # --- non-streaming ---

252 + chat_completion = await client.chat.completions.create(**create_kwargs)

253 +

254 + choice = chat_completion.choices[0]

255 + tool_calls = choice.message.tool_calls

256 +

257 + assert choice.finish_reason == "tool_calls"

258 + assert tool_calls is not None and len(tool_calls) >= 1

259 + assert tool_calls[0].function.name == "get_current_weather"

260 + parsed_arguments = json.loads(tool_calls[0].function.arguments)

261 + assert "city" in parsed_arguments

262 + assert len(tool_calls[0].id) == 9

263 +

264 + # --- streaming ---

265 + stream = await client.chat.completions.create(**create_kwargs, stream=True)

266 +

267 + result = await _collect_streamed_tool_call(stream)

268 +

269 + assert result.finish_reason_count == 1

270 + assert result.role_name == "assistant"

271 + assert result.function_name == "get_current_weather"

272 + streamed_args = json.loads(result.function_args_str)

273 + assert isinstance(result.tool_call_id, str)

274 + if _is_pre_v11(server_config):

275 + assert len(result.tool_call_id) == streaming_id_len_pre_v11

276 + else:

277 + assert len(result.tool_call_id) == 9

278 + assert parsed_arguments == streamed_args

279 +

280 +

281 +@pytest.mark.asyncio

282 +async def test_tool_call_none_with_tools(

283 + client: openai.AsyncOpenAI, server_config: ServerConfig

284 +) -> None:

285 + _requires_tool_parser(server_config)

286 +

287 + models = await client.models.list()

288 + model_name: str = models.data[0].id

289 +

290 + # --- non-streaming ---

291 + chat_completion = await client.chat.completions.create(

292 + messages=ensure_system_prompt(MESSAGES_ASKING_FOR_TOOLS, server_config),

293 + temperature=0,

294 + max_completion_tokens=100,

295 + model=model_name,

296 + tools=[WEATHER_TOOL],

297 + tool_choice="none",

298 + logprobs=False,

299 + seed=SEED,

300 + )

301 +

302 + choice = chat_completion.choices[0]

303 +

304 + assert choice.finish_reason != "tool_calls"

305 + assert choice.message.tool_calls is None or len(choice.message.tool_calls) == 0

306 + assert choice.message.content is not None

307 + # Without grammar enforcement, pre-v11 models may still emit [TOOL_CALLS]

308 + if not _is_pre_v11(server_config):

309 + assert "[TOOL_CALLS]" not in choice.message.content

310 +

311 + non_streaming_content = choice.message.content

312 +

313 + # --- streaming ---

314 + stream = await client.chat.completions.create(

315 + messages=ensure_system_prompt(MESSAGES_ASKING_FOR_TOOLS, server_config),

316 + temperature=0,

317 + max_completion_tokens=100,

318 + model=model_name,

319 + tools=[WEATHER_TOOL],

320 + tool_choice="none",

321 + logprobs=False,

322 + seed=SEED,

323 + stream=True,

324 + )

325 +

326 + # Pre-v11 models lack grammar enforcement, so the model may still

327 + # emit tool calls even with tool_choice="none".

328 + pre_v11 = _is_pre_v11(server_config)

329 + result = await _collect_streamed_content(stream, no_tool_calls=not pre_v11)

330 +

331 + assert result.finish_reason_count == 1

332 + if not pre_v11:

333 + assert result.finish_reason != "tool_calls"

334 + streamed_content = "".join(result.chunks)

335 + if not pre_v11:

336 + assert "[TOOL_CALLS]" not in streamed_content

337 + assert streamed_content == non_streaming_content

338 +

339 +

340 +@pytest.mark.asyncio

341 +async def test_chat_without_tools(

342 + client: openai.AsyncOpenAI, server_config: ServerConfig

343 +) -> None:

344 + models = await client.models.list()

345 + model_name: str = models.data[0].id

346 +

347 + # --- non-streaming ---

348 + chat_completion = await client.chat.completions.create(

349 + messages=ensure_system_prompt(MESSAGES_WITHOUT_TOOLS, server_config),

350 + temperature=0,

351 + max_completion_tokens=150,

352 + model=model_name,

353 + logprobs=False,

354 + seed=SEED,

355 + )

356 +

357 + choice = chat_completion.choices[0]

358 + output_text = choice.message.content

359 +

360 + assert output_text is not None and len(output_text) > 0

361 + assert choice.finish_reason != "tool_calls"

362 + assert choice.message.tool_calls is None or len(choice.message.tool_calls) == 0

363 +

364 + # --- streaming ---

365 + stream = await client.chat.completions.create(

366 + messages=ensure_system_prompt(MESSAGES_WITHOUT_TOOLS, server_config),

367 + temperature=0,

368 + max_completion_tokens=150,

369 + model=model_name,

370 + logprobs=False,

371 + seed=SEED,

372 + stream=True,

373 + )

374 +

375 + result = await _collect_streamed_content(

376 + stream, expected_finish_reason=choice.finish_reason

377 + )

378 +

379 + assert result.role_sent

380 + assert result.finish_reason_count == 1

381 + assert len(result.chunks)

382 + assert "".join(result.chunks) == output_text

383 +

384 +

385 +@pytest.mark.asyncio

386 +async def test_tool_call_with_results(

387 + client: openai.AsyncOpenAI, server_config: ServerConfig

388 +) -> None:

389 + _requires_tool_parser(server_config)

390 +

391 + models = await client.models.list()

392 + model_name: str = models.data[0].id

393 +

394 + # --- non-streaming ---

395 + chat_completion = await client.chat.completions.create(

396 + messages=ensure_system_prompt(MESSAGES_WITH_TOOL_RESPONSE, server_config),

397 + temperature=0,

398 + max_completion_tokens=100,

399 + model=model_name,

400 + tools=[WEATHER_TOOL, SEARCH_TOOL],

401 + logprobs=False,

402 + seed=SEED,

403 + )

404 +

405 + choice = chat_completion.choices[0]

406 +

407 + assert choice.finish_reason != "tool_calls"

408 + assert choice.message.tool_calls is None or len(choice.message.tool_calls) == 0

409 + assert choice.message.content is not None

410 + assert "98" in choice.message.content

411 +

412 + # --- streaming ---

413 + stream = await client.chat.completions.create(

414 + messages=ensure_system_prompt(MESSAGES_WITH_TOOL_RESPONSE, server_config),

415 + temperature=0,

416 + max_completion_tokens=100,

417 + model=model_name,

418 + tools=[WEATHER_TOOL, SEARCH_TOOL],

419 + logprobs=False,

420 + seed=SEED,

421 + stream=True,

422 + )

423 +

424 + result = await _collect_streamed_content(

425 + stream, expected_finish_reason=choice.finish_reason

426 + )

427 +

428 + assert result.role_sent

429 + assert result.finish_reason_count == 1

430 + assert len(result.chunks)

431 + assert "".join(result.chunks) == choice.message.content

432 +

433 +

434 +def _requires_parallel(server_config: ServerConfig) -> None:

435 + r"""Skip test if the model does not support parallel tool calls."""

436 + if not server_config.get("supports_parallel"):

437 + pytest.skip(

438 + f"Skipping: {server_config['model']} does not support parallel tool calls"

439 + )

440 +

441 +

442 +@pytest.mark.asyncio

443 +async def test_tool_call_parallel(

444 + client: openai.AsyncOpenAI, server_config: ServerConfig

445 +) -> None:

446 + _requires_tool_parser(server_config)

447 + _requires_parallel(server_config)

448 +

449 + models = await client.models.list()

450 + model_name: str = models.data[0].id

451 +

452 + # --- non-streaming ---

453 + chat_completion = await client.chat.completions.create(

454 + messages=ensure_system_prompt(

455 + MESSAGES_ASKING_FOR_PARALLEL_TOOLS, server_config

456 + ),

457 + temperature=0,

458 + max_completion_tokens=200,

459 + model=model_name,

460 + tools=[WEATHER_TOOL],

461 + logprobs=False,

462 + seed=SEED,

463 + )

464 +

465 + choice = chat_completion.choices[0]

466 + tool_calls = choice.message.tool_calls

467 +

468 + assert choice.finish_reason == "tool_calls"

469 + assert tool_calls is not None and len(tool_calls) >= 2

470 + for tc in tool_calls:

471 + assert tc.type == "function"

472 + assert tc.function.name == "get_current_weather"

473 + assert isinstance(tc.function.arguments, str)

474 + parsed = json.loads(tc.function.arguments)

475 + assert "city" in parsed

476 + assert len(tc.id) == 9

477 +

478 + non_streaming_tool_calls = tool_calls

479 +

480 + # --- streaming ---

481 + stream = await client.chat.completions.create(

482 + messages=ensure_system_prompt(

483 + MESSAGES_ASKING_FOR_PARALLEL_TOOLS, server_config

484 + ),

485 + temperature=0,

486 + max_completion_tokens=200,

487 + model=model_name,

488 + tools=[WEATHER_TOOL],

489 + logprobs=False,

490 + seed=SEED,

491 + stream=True,

492 + )

493 +

494 + result = await _collect_streamed_parallel_tool_calls(stream)

495 +

496 + assert result.finish_reason_count == 1

497 + assert result.role_name == "assistant"

498 + assert len(result.function_names) >= 2

499 + assert all(name == "get_current_weather" for name in result.function_names)

500 + assert len(result.tool_call_ids) >= 2

501 + assert all(isinstance(tid, str) and len(tid) == 9 for tid in result.tool_call_ids)

502 +

503 + for args_str in result.function_args_strs:

504 + streamed_args = json.loads(args_str)

505 + assert "city" in streamed_args

506 +

507 + assert len(result.function_names) == len(non_streaming_tool_calls)

tests/tool_use/mistral/utils.py

+24 -10

@@ -2,16 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

5 -from typing_extensions import TypedDict

6 -

7 -

8 -class ServerConfig(TypedDict, total=False):

9 - model: str

10 - arguments: list[str]

11 - system_prompt: str | None

12 - supports_parallel: bool | None

13 - supports_rocm: bool | None

14 -

5 +from tests.tool_use.utils import ServerConfig

7 ARGS: list[str] = ["--max-model-len", "1024"]

@@ -21,6 +12,11 @@ CONFIGS: dict[str, ServerConfig] = {

12 "arguments": [

13 "--tokenizer-mode",

14 "mistral",

15 + "--tool-call-parser",

16 + "mistral",

17 + "--enable-auto-tool-choice",

18 + "--enforce-eager",

19 + "--no-enable-prefix-caching",

20 '--ignore-patterns="consolidated.safetensors"',

21 ],

22 "system_prompt": "You are a helpful assistant with access to tools. If a tool"

@@ -29,4 +25,22 @@ CONFIGS: dict[str, ServerConfig] = {

25 "without calling a tool. DO NOT CALL A TOOL THAT IS IRRELEVANT "

26 "to the user's question - just respond to it normally.",

27 },

28 + "ministral-3b": {

29 + "model": "mistralai/Ministral-3-3B-Instruct-2512",

30 + "arguments": [

31 + "--tokenizer-mode",

32 + "mistral",

33 + "--tool-call-parser",

34 + "mistral",

35 + "--enable-auto-tool-choice",

36 + "--enforce-eager",

37 + "--no-enable-prefix-caching",

38 + ],

39 + "system_prompt": "You are a helpful assistant with access to tools. If a tool"

40 + " that you have would be helpful to answer a user query, "

41 + "call the tool. Otherwise, answer the user's query directly "

42 + "without calling a tool. DO NOT CALL A TOOL THAT IS IRRELEVANT "

43 + "to the user's question - just respond to it normally.",

44 + "supports_parallel": True,

45 + },

46 }

vllm/entrypoints/openai/chat_completion/protocol.py

+4 -8

@@ -11,7 +11,7 @@ from openai.types.chat.chat_completion_audio import (

11 ChatCompletionAudio as OpenAIChatCompletionAudio,

12 )

13 from openai.types.chat.chat_completion_message import Annotation as OpenAIAnnotation

14 -from pydantic import Field, model_validator

14 +from pydantic import Field, PrivateAttr, model_validator

16 from vllm.config import ModelConfig

17 from vllm.config.utils import replace

@@ -398,6 +398,9 @@ class ChatCompletionRequest(OpenAIBaseModel):

398 msg["tool_calls"] = list(tool_calls)

399 return self

400

401 + _grammar_from_tool_parser: bool = PrivateAttr(default=False)

402 + """CAUTION: Should only be set by ``ToolParser.adjust_request``."""

403 +

404 def build_chat_params(

405 self,

406 default_template: str | None,

@@ -822,13 +825,6 @@ class ChatCompletionRequest(OpenAIBaseModel):

825

826 return data

827

825 - @model_validator(mode="before")

826 - @classmethod

827 - def set_include_reasoning_for_none_effort(cls, data: Any) -> Any:

828 - if data.get("reasoning_effort") == "none":

829 - data["include_reasoning"] = False

830 - return data

831 -

828

829 class BatchChatCompletionRequest(OpenAIBaseModel):

830 """Request model for the /v1/chat/completions/batch endpoint.

vllm/entrypoints/openai/chat_completion/serving.py

+64 -5

@@ -73,7 +73,10 @@ from vllm.reasoning import ReasoningParser

73 from vllm.renderers import ChatParams

74 from vllm.sampling_params import BeamSearchParams, SamplingParams

75 from vllm.tokenizers import TokenizerLike

76 -from vllm.tool_parsers.mistral_tool_parser import MistralToolCall

76 +from vllm.tool_parsers.mistral_tool_parser import (

77 + MistralToolCall,

78 + MistralToolParser,

79 +)

80 from vllm.tool_parsers.utils import partial_json_loads

81 from vllm.utils.collection_utils import as_list

82 from vllm.utils.mistral import is_mistral_tokenizer

@@ -140,6 +143,12 @@ class OpenAIServingChat(OpenAIServing):

143 enable_auto_tools=enable_auto_tools,

144 model_name=self.model_config.model,

145 )

146 + _is_mistral_tool_parser = self.tool_parser is not None and issubclass(

147 + self.tool_parser, MistralToolParser

148 + )

149 + if _is_mistral_tool_parser and self.reasoning_parser_cls is not None:

150 + MistralToolParser.model_can_reason = True

151 +

152 self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none

153

154 self.enable_prompt_tokens_details = enable_prompt_tokens_details

@@ -310,6 +319,11 @@ class OpenAIServingChat(OpenAIServing):

319 else:

320 if not request.include_reasoning:

321 reasoning_ended = True

322 + elif request._grammar_from_tool_parser:

323 + # The Mistral grammar already includes an optional

324 + # `think?` rule that handles both reasoning and

325 + # non-reasoning outputs.

326 + reasoning_ended = True

327 elif reasoning_parser:

328 reasoning_ended = reasoning_parser.is_reasoning_end(

329 prompt_token_ids or []

@@ -530,6 +544,8 @@ class OpenAIServingChat(OpenAIServing):

544 harmony_tools_streamed = [False] * num_choices

545 tools_streamed = [False] * num_choices

546

547 + is_mistral_grammar_path = request._grammar_from_tool_parser

548 +

549 if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):

550 tool_choice_function_name = request.tool_choice.function.name

551 else:

@@ -553,7 +569,7 @@ class OpenAIServingChat(OpenAIServing):

569

570 # Only one of these will be used, thus previous_texts and

571 # all_previous_token_ids will not be used twice in the same iteration.

556 - if tool_choice_auto or reasoning_parser:

572 + if is_mistral_grammar_path or tool_choice_auto or reasoning_parser:

573 # These are only required in "auto" tool choice case

574 all_previous_token_ids = [[] for _ in range(num_choices)]

575 reasoning_end_arr = [False] * num_choices

@@ -748,7 +764,7 @@ class OpenAIServingChat(OpenAIServing):

764 delta_message: DeltaMessage | None

765

766 # just update previous_texts and previous_token_ids

751 - if tool_choice_auto or reasoning_parser:

767 + if is_mistral_grammar_path or tool_choice_auto or reasoning_parser:

768 assert previous_texts is not None

769 assert all_previous_token_ids is not None

770 previous_text = previous_texts[i]

@@ -772,6 +788,30 @@ class OpenAIServingChat(OpenAIServing):

788 )

789 )

790 harmony_tools_streamed[i] |= tools_streamed_flag

791 + # Mistral grammar path: combined reasoning + tool streaming

792 + elif is_mistral_grammar_path:

793 + assert tool_parser is not None

794 + assert isinstance(tool_parser, MistralToolParser)

795 + assert reasoning_end_arr is not None

796 + output_token_ids = as_list(output.token_ids)

797 + result = tool_parser.extract_maybe_reasoning_and_tool_streaming(

798 + reasoning_parser=reasoning_parser,

799 + previous_text=previous_text,

800 + current_text=current_text,

801 + delta_text=delta_text,

802 + previous_token_ids=previous_token_ids,

803 + current_token_ids=current_token_ids,

804 + output_token_ids=output_token_ids,

805 + reasoning_ended=reasoning_end_arr[i],

806 + prompt_is_reasoning_end=(prompt_is_reasoning_end_arr[i]),

807 + request=request,

808 + )

809 + delta_message = result.delta_message

810 + reasoning_end_arr[i] = result.reasoning_ended

811 + current_text = result.current_text

812 + current_token_ids = result.current_token_ids

813 + if result.tools_called:

814 + tools_streamed[i] = True

815 # handle streaming deltas for tools with named tool_choice

816 elif tool_choice_function_name:

817 # When encountering think end id in prompt_token_ids

@@ -925,7 +965,9 @@ class OpenAIServingChat(OpenAIServing):

965 delta_message = DeltaMessage(content=delta_text)

966

967 # update the previous values for the next iteration

928 - if (tool_choice_auto or reasoning_parser) and not self.use_harmony:

968 + if (

969 + is_mistral_grammar_path or tool_choice_auto or reasoning_parser

970 + ) and not self.use_harmony:

971 assert previous_texts is not None

972 assert all_previous_token_ids is not None

973 previous_texts[i] = current_text

@@ -1312,7 +1354,24 @@ class OpenAIServingChat(OpenAIServing):

1354 tool_call_class = (

1355 MistralToolCall if is_mistral_tokenizer(tokenizer) else ToolCall

1356 )

1315 - if (not self.enable_auto_tools or not self.tool_parser) and (

1357 +

1358 + use_mistral_tool_parser = request._grammar_from_tool_parser

1359 + if use_mistral_tool_parser:

1360 + tool_call_items = MistralToolParser.build_non_streaming_tool_calls(

1361 + tool_calls

1362 + )

1363 + if tool_call_items:

1364 + auto_tools_called = (

1365 + request.tool_choice is None or request.tool_choice == "auto"

1366 + )

1367 + message = ChatMessage(

1368 + role=role,

1369 + reasoning=reasoning,

1370 + content=content,

1371 + tool_calls=tool_call_items,

1372 + )

1373 +

1374 + elif (not self.enable_auto_tools or not self.tool_parser) and (

1375 not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)

1376 and request.tool_choice != "required"

1377 ):

vllm/entrypoints/openai/engine/serving.py

+26 -8

@@ -65,6 +65,7 @@ from vllm.renderers.inputs.preprocess import (

65 from vllm.sampling_params import BeamSearchParams, SamplingParams

66 from vllm.tokenizers import TokenizerLike

67 from vllm.tool_parsers import ToolParser

68 +from vllm.tool_parsers.mistral_tool_parser import MistralToolParser

69 from vllm.tracing import (

70 contains_trace_headers,

71 extract_trace_headers,

@@ -610,16 +611,31 @@ class OpenAIServing:

611 tool_parser_cls: type[ToolParser] | None,

612 content: str | None = None,

613 ) -> tuple[list[FunctionCall] | None, str | None]:

614 + # When the Mistral grammar factory injected structured outputs,

615 + # let the parser handle the output.

616 + use_mistral_tool_parser = (

617 + isinstance(request, ChatCompletionRequest)

618 + and tool_parser_cls is not None

619 + and issubclass(tool_parser_cls, MistralToolParser)

620 + and request._grammar_from_tool_parser

621 + )

622 +

623 function_calls = list[FunctionCall]()

614 - if request.tool_choice and isinstance(request.tool_choice, ToolChoiceFunction):

624 + if (

625 + not use_mistral_tool_parser

626 + and request.tool_choice

627 + and isinstance(request.tool_choice, ToolChoiceFunction)

628 + ):

629 assert content is not None

630 # Forced Function Call

631 function_calls.append(

632 FunctionCall(name=request.tool_choice.name, arguments=content)

633 )

634 content = None # Clear content since tool is called.

621 - elif request.tool_choice and isinstance(

622 - request.tool_choice, ChatCompletionNamedToolChoiceParam

635 + elif (

636 + not use_mistral_tool_parser

637 + and request.tool_choice

638 + and isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)

639 ):

640 assert content is not None

641 # Forced Function Call

@@ -627,7 +643,7 @@ class OpenAIServing:

643 FunctionCall(name=request.tool_choice.function.name, arguments=content)

644 )

645 content = None # Clear content since tool is called.

630 - elif request.tool_choice == "required":

646 + elif not use_mistral_tool_parser and request.tool_choice == "required":

647 tool_calls = []

648 with contextlib.suppress(ValidationError):

649 content = content or ""

@@ -642,10 +658,12 @@ class OpenAIServing:

658 )

659 )

660 content = None # Clear content since tool is called.

645 - elif (

646 - tool_parser_cls

647 - and enable_auto_tools

648 - and (request.tool_choice == "auto" or request.tool_choice is None)

661 + elif tool_parser_cls and (

662 + use_mistral_tool_parser

663 + or (

664 + enable_auto_tools

665 + and (request.tool_choice == "auto" or request.tool_choice is None)

666 + )

667 ):

668 if tokenizer is None:

669 raise ValueError(

vllm/entrypoints/serve/render/serving.py

+12 -2

@@ -53,6 +53,7 @@ from vllm.renderers.inputs.preprocess import (

53 prompt_to_seq,

54 )

55 from vllm.tool_parsers import ToolParser

56 +from vllm.tool_parsers.mistral_tool_parser import MistralToolParser

57 from vllm.utils import random_uuid

58 from vllm.utils.mistral import is_mistral_tokenizer

59 from vllm.utils.mistral import mt as _mt

@@ -555,9 +556,19 @@ class OpenAIServingRender:

556 # tool parsing is done only if a tool_parser has been set and if

557 # tool_choice is not "none" (if tool_choice is "none" but a tool_parser

558 # is set, we want to prevent parsing a tool_call hallucinated by the LLM

559 + #

560 + # Exception: Mistral grammar-capable tokenizers always call

561 + # adjust_request — even for tool_choice="none" — so that the grammar

562 + # factory can prevent special-token leakage.

563 if tool_parser is not None:

564 tool_choice = getattr(request, "tool_choice", "none")

560 - if tool_choice != "none":

565 + tokenizer = renderer.get_tokenizer()

566 + is_mistral_grammar_eligible = (

567 + issubclass(tool_parser, MistralToolParser)

568 + and is_mistral_tokenizer(tokenizer)

569 + and tokenizer.supports_grammar

570 + )

571 + if tool_choice != "none" or is_mistral_grammar_eligible:

572 if not isinstance(request, ChatCompletionRequest | ResponsesRequest):

573 msg = (

574 "Tool usage is only supported "

@@ -565,7 +576,6 @@ class OpenAIServingRender:

576 f"but got {type(request).__name__}"

577 )

578 raise NotImplementedError(msg)

568 - tokenizer = renderer.get_tokenizer()

579 request = tool_parser(tokenizer, request.tools).adjust_request(

580 request=request

581 )

vllm/sampling_params.py

+12 -2

@@ -157,6 +157,10 @@ def _is_non_tekken_mistral(tokenizer: TokenizerLike) -> bool:

157 return is_mistral_tokenizer(tokenizer) and not tokenizer.is_tekken

158

159

160 +def _get_llg_tokenizer(tokenizer: TokenizerLike) -> Any:

161 + return tokenizer.llg_tokenizer if is_mistral_tokenizer(tokenizer) else None

162 +

163 +

164 class SamplingParams(

165 PydanticMsgspecMixin,

166 msgspec.Struct,

@@ -816,7 +820,10 @@ class SamplingParams(

820 # allows <|special_token|> and similar, see

821 # https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens

822 # Without tokenizer these are disallowed in grammars.

819 - validate_guidance_grammar(self, tokenizer=None)

823 + validate_guidance_grammar(

824 + self,

825 + tokenizer=_get_llg_tokenizer(tokenizer),

826 + )

827 elif backend == "outlines":

828 # outlines backend

829 validate_structured_output_request_outlines(self)

@@ -862,7 +869,10 @@ class SamplingParams(

869 self.structured_outputs._backend = "outlines"

870 else:

871 # Fall back to guidance by default.

865 - validate_guidance_grammar(self, tokenizer=None)

872 + validate_guidance_grammar(

873 + self,

874 + tokenizer=_get_llg_tokenizer(tokenizer),

875 + )

876 self.structured_outputs._backend = "guidance"

877 # Remember that this backend was set automatically

878 self.structured_outputs._backend_was_auto = True

vllm/tokenizers/mistral.py

+49 -38

@@ -54,6 +54,50 @@ if TYPE_CHECKING:

54 logger = init_logger(__name__)

57 +def _pop_unallowed_keys_and_warn(

58 + dictionary: dict[str, Any], allowed_keys: set[str], err_dict_name: str

59 +):

60 + keys = list(dictionary.keys())

61 + for key in keys:

62 + if key not in allowed_keys:

63 + dictionary.pop(key)

64 + logger.warning_once(

65 + f"'{key=}' is not supported by mistral-common "

66 + f"for {err_dict_name}. It has been popped from the "

67 + "object."

68 + )

69 +

70 +

71 +# TODO(juliendenize): remove this once OpenAI API is better supported by

72 +# `mistral-common`.

73 +def adapt_inplace_to_mistral_tool(

74 + tool: dict[str, Any],

75 +) -> dict[str, Any]:

76 + tools_fields = set(Tool.model_fields.keys())

77 + function_fields = set(Function.model_fields.keys())

78 +

79 + # The Mistral client, in comparison to the OpenAI client, requires the

80 + # "parameters" dict and the "description" string to be present

81 + # even if they are empty.

82 + if function := tool.get("function"):

83 + if function.get("parameters") is None:

84 + function["parameters"] = {}

85 + if function.get("description") is None:

86 + function["description"] = ""

87 +

88 + _pop_unallowed_keys_and_warn(

89 + dictionary=function,

90 + allowed_keys=function_fields,

91 + err_dict_name="function",

92 + )

93 +

94 + _pop_unallowed_keys_and_warn(

95 + dictionary=tool, allowed_keys=tools_fields, err_dict_name="tools"

96 + )

97 +

98 + return tool

99 +

100 +

101 def maybe_serialize_tool_calls(request: "MistralChatCompletionRequest"):

102 # SEE: https://github.com/vllm-project/vllm/pull/9951

103 # Credits go to: @gcalmettes

@@ -159,44 +203,11 @@ def _prepare_apply_chat_template_tools_and_messages(

203 # Remove reasoning as unsupported by Mistral

204 _ = message.pop("reasoning", None) # type: ignore

205

162 - # The Mistral client, in comparison to the OpenAI client, requires the

163 - # "parameters" dict and the "description" string to be present

164 - # even if they are empty.

165 - if tools:

166 - for function in [

167 - tool["function"] for tool in tools if tool["type"] == "function"

168 - ]:

169 - if function.get("parameters") is None:

170 - function["parameters"] = {}

171 - if function.get("description") is None:

172 - function["description"] = ""

173 -

174 - # We filter not supported arguments to avoid throwing an error.

175 - # TODO(juliendenize): remove this once OpenAI API is better supported by

176 - # `mistral-common`.

177 - tools_fields = set(Tool.model_fields.keys())

178 - function_fields = set(Function.model_fields.keys())

179 - for tool in tools:

180 - tool_keys = list(tool.keys())

181 - for tool_key in tool_keys:

182 - if tool_key not in tools_fields:

183 - tool.pop(tool_key)

184 - logger.warning_once(

185 - f"'{tool_key}' is not supported by mistral-common for tools. "

186 - "It has been popped from the tool definition."

187 - )

188 - if tool["type"] == "function":

189 - function_keys = list(tool["function"].keys())

190 - for function_key in function_keys:

191 - if function_key not in function_fields:

192 - tool["function"].pop(function_key)

193 - logger.warning_once(

194 - f"'{function_key}' is not supported by mistral-common "

195 - "for function tools. It has been popped from the "

196 - "function definition."

197 - )

198 - else:

199 - raise ValueError("mistral-common only supports function tools.")

206 + tools = (

207 + [adapt_inplace_to_mistral_tool(tool=tool) for tool in tools]

208 + if tools is not None

209 + else None

210 + )

211

212 return messages, tools

213

vllm/tool_parsers/mistral_tool_parser.py

+178 -10

@@ -1,12 +1,15 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 +from __future__ import annotations

5 +

6 import json

7 from collections.abc import Sequence

8 +from dataclasses import dataclass

9 from enum import Enum, auto

10 from random import choices

11 from string import ascii_letters, digits

9 -from typing import Any

12 +from typing import TYPE_CHECKING, Any

14 import ijson

15 import regex as re

@@ -37,14 +40,19 @@ from vllm.entrypoints.openai.engine.protocol import (

40 )

41 from vllm.entrypoints.openai.responses.protocol import ResponsesRequest

42 from vllm.logger import init_logger

43 +from vllm.reasoning.mistral_reasoning_parser import MistralReasoningParser

44 from vllm.sampling_params import StructuredOutputsParams

45 from vllm.tokenizers import TokenizerLike

46 +from vllm.tokenizers.mistral import MistralTokenizer, adapt_inplace_to_mistral_tool

47 from vllm.tool_parsers.abstract_tool_parser import (

48 Tool,

49 ToolParser,

50 )

51 from vllm.utils.mistral import is_mistral_tokenizer

53 +if TYPE_CHECKING:

54 + from vllm.reasoning import ReasoningParser

55 +

56 logger = init_logger(__name__)

58 ALPHANUMERIC = ascii_letters + digits

@@ -86,13 +94,28 @@ def _is_pre_v11_tokeniser(model_tokenizer: TokenizerLike) -> bool:

94 return not (is_mistral_tokenizer(model_tokenizer) and model_tokenizer.version >= 11)

89 -class MistralToolParser(ToolParser):

97 +@dataclass

98 +class MistralStreamingResult:

99 + r"""Encapsulates the mutable state returned from

100 + `MistralToolParser.extract_maybe_reasoning_and_tool_streaming`.

101 """

91 - Tool call parser for Mistral 7B Instruct v0.3, intended for use with

92 - - [`mistral_common`](https://github.com/mistralai/mistral-common/)

93 - - the examples/tool_chat_template_mistral.jinja template.

102

95 - Used when --enable-auto-tool-choice --tool-call-parser mistral are all set

103 + delta_message: DeltaMessage | None

104 + reasoning_ended: bool

105 + tools_called: bool

106 + current_text: str

107 + current_token_ids: list[int]

108 +

109 +

110 +class MistralToolParser(ToolParser):

111 + r"""Tool call parser for Mistral models, intended for use with either:

112 +

113 + - `mistral_common <https://github.com/mistralai/mistral-common/>`_

114 + (recommended)

115 + - the `examples/tool_chat_template_mistral.jinja` template.

116 +

117 + Used when `--enable-auto-tool-choice --tool-call-parser mistral` are all

118 + set.

119 """

120

121 # Used to generate correct grammar in `adjust_request`

@@ -210,9 +233,11 @@ class MistralToolParser(ToolParser):

233 reasoning=self.model_can_reason

234 )

235

213 - tools = (

236 + mistral_tools = (

237 [

215 - MistralTool.from_openai(openai_tool=tool.model_dump())

238 + MistralTool.model_validate(

239 + adapt_inplace_to_mistral_tool(tool.model_dump())

240 + )

241 for tool in request.tools

242 ]

243 if request.tools is not None

@@ -244,15 +269,158 @@ class MistralToolParser(ToolParser):

269 lark_grammar = grammar_factory.get_lark_from_jinja(

270 template=template,

271 mode=tool_choice,

247 - tools=tools,

272 + tools=mistral_tools,

273 json_schema=json_schema,

274 parallel_tool_calls=request.parallel_tool_calls,

275 json_only=False,

276 )

277

278 request.structured_outputs = StructuredOutputsParams(grammar=lark_grammar)

279 + request._grammar_from_tool_parser = True

280 return request

281

282 + def extract_maybe_reasoning_and_tool_streaming(

283 + self,

284 + *,

285 + reasoning_parser: ReasoningParser | None,

286 + previous_text: str,

287 + current_text: str,

288 + delta_text: str,

289 + previous_token_ids: list[int],

290 + current_token_ids: list[int],

291 + output_token_ids: Sequence[int],

292 + reasoning_ended: bool,

293 + prompt_is_reasoning_end: bool | None,

294 + request: ChatCompletionRequest,

295 + ) -> MistralStreamingResult:

296 + r"""Streaming extraction with reasoning followed by tool-call parsing.

297 +

298 + This method encapsulates the combined reasoning extraction and

299 + tool-call streaming logic so that the serving layer only needs a

300 + thin routing branch.

301 +

302 + The flow is:

303 +

304 + 1. If a *reasoning_parser* is present and reasoning has **not** ended,

305 + extract reasoning tokens. Pre-v15 models may have pre-filled

306 + `[THINK]...[/THINK]` in system prompts, so we skip the

307 + prompt-level reasoning-end check for those.

308 + 2. Once reasoning ends (or if there is no reasoning parser), delegate

309 + to `extract_tool_calls_streaming` and track whether tools were

310 + called.

311 +

312 + Args:

313 + reasoning_parser: Optional reasoning parser instance.

314 + previous_text: Accumulated text from prior chunks.

315 + current_text: Full accumulated text including current chunk.

316 + delta_text: New text in this chunk.

317 + previous_token_ids: Token ids from prior chunks.

318 + current_token_ids: Full token ids including current chunk.

319 + output_token_ids: Raw output token ids from the engine.

320 + reasoning_ended: Whether reasoning has already ended.

321 + prompt_is_reasoning_end: Whether the prompt itself ends reasoning.

322 + request: The originating chat completion request.

323 + """

324 + delta_message: DeltaMessage | None = None

325 + tools_called = False

326 + reasoning_ended_at_entry = reasoning_ended

327 +

328 + # For MistralReasoningParser, only enter the reasoning block when

329 + # the model has actually emitted a [THINK] token. Other reasoning

330 + # parsers always expect thinking to be present.

331 + expect_thinking = (

332 + not isinstance(reasoning_parser, MistralReasoningParser)

333 + or reasoning_parser.start_token_id in current_token_ids

334 + )

335 + if reasoning_parser is not None and not reasoning_ended and expect_thinking:

336 + # Pre-v15 models may have pre-filled [THINK]...[/THINK] in

337 + # system prompts, so skip the prompt-level reasoning-end

338 + # check and wait for the output's own end-of-think.

339 + is_pre_v15 = (

340 + isinstance(self.model_tokenizer, MistralTokenizer)

341 + and self.model_tokenizer.version < 15

342 + )

343 +

344 + if not is_pre_v15 and prompt_is_reasoning_end:

345 + reasoning_ended = True

346 + current_token_ids = list(output_token_ids)

347 + else:

348 + delta_message = reasoning_parser.extract_reasoning_streaming(

349 + previous_text,

350 + current_text,

351 + delta_text,

352 + previous_token_ids,

353 + current_token_ids,

354 + output_token_ids,

355 + )

356 + if reasoning_parser.is_reasoning_end_streaming(

357 + current_token_ids, output_token_ids

358 + ):

359 + reasoning_ended = True

360 + current_token_ids = reasoning_parser.extract_content_ids(

361 + list(output_token_ids)

362 + )

363 + if delta_message and delta_message.content:

364 + current_text = delta_message.content

365 + delta_message.content = None

366 + else:

367 + current_text = ""

368 +

369 + if not reasoning_ended:

370 + return MistralStreamingResult(

371 + delta_message=delta_message,

372 + reasoning_ended=False,

373 + tools_called=False,

374 + current_text=current_text,

375 + current_token_ids=current_token_ids,

376 + )

377 +

378 + delta_token_ids = list(output_token_ids)

379 +

380 + # On the iteration where reasoning just ended, reset the text/token

381 + # state so the tool parser sees a clean history instead of the

382 + # accumulated reasoning text.

383 + if not reasoning_ended_at_entry and reasoning_ended:

384 + previous_text = ""

385 + previous_token_ids = []

386 + delta_text = current_text

387 + delta_token_ids = current_token_ids

388 +

389 + delta_message = self.extract_tool_calls_streaming(

390 + previous_text=previous_text,

391 + current_text=current_text,

392 + delta_text=delta_text,

393 + previous_token_ids=previous_token_ids,

394 + current_token_ids=current_token_ids,

395 + delta_token_ids=delta_token_ids,

396 + request=request,

397 + )

398 + if delta_message and delta_message.tool_calls:

399 + tools_called = True

400 +

401 + return MistralStreamingResult(

402 + delta_message=delta_message,

403 + reasoning_ended=reasoning_ended,

404 + tools_called=tools_called,

405 + current_text=current_text,

406 + current_token_ids=current_token_ids,

407 + )

408 +

409 + @staticmethod

410 + def build_non_streaming_tool_calls(

411 + tool_calls: list[FunctionCall] | None,

412 + ) -> list[ToolCall]:

413 + r"""Build `MistralToolCall` items for non-streaming responses."""

414 + if not tool_calls:

415 + return []

416 +

417 + return [

418 + MistralToolCall(id=tc.id, function=tc)

419 + if tc.id

420 + else MistralToolCall(function=tc)

421 + for tc in tool_calls

422 + ]

423 +

424 def extract_tool_calls(

425 self,

426 model_output: str,

@@ -323,7 +491,7 @@ class MistralToolParser(ToolParser):

491 )[0]

492 tool_calls = json.loads(raw_tool_call)

493 except (IndexError, json.JSONDecodeError):

326 - logger.exception("Error in extracting tool call from response: {e}")

494 + logger.exception("Error in extracting tool call from response.")

495 # If raw decoding and decoding post regex rule fails, then just

496 # return content.

497 return ExtractedToolCallInformation(