@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d82ac00923
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

[Refactor][Mistral] Extract parsing logic into MistralParser (#44596)

Signed-off-by: sfeng33 <4florafeng@gmail.com>

Flora Feng committed Jun 9, 2026 at 20:12 UTC d82ac0092392f84b1c0f65d2f8846d320cf0fd23
7 files changed +136 -729
tests/tool_parsers/test_mistral_tool_parser.py
-385
@@ -3,7 +3,6 @@
3
4 import json
5 from collections.abc import Generator
6 -from typing import Any
6 from unittest.mock import MagicMock, patch
7
8 import partial_json_parser
@@ -29,22 +28,17 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
28 ChatCompletionRequest,
29 )
30 from vllm.entrypoints.openai.engine.protocol import (
32 - DeltaFunctionCall,
31 DeltaMessage,
32 DeltaToolCall,
33 ExtractedToolCallInformation,
34 StructuralTagResponseFormat,
35 )
38 -from vllm.entrypoints.openai.engine.protocol import FunctionCall as VllmFunctionCall
39 -from vllm.reasoning.mistral_reasoning_parser import MistralReasoningParser
36 from vllm.sampling_params import StructuredOutputsParams
37 from vllm.tokenizers import TokenizerLike, get_tokenizer
38 from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
39 from vllm.tokenizers.mistral import MistralTokenizer
40 from vllm.tool_parsers.mistral_tool_parser import (
41 _DEFAULT_JSON_SCHEMA,
46 - MistralStreamingResult,
47 - MistralToolCall,
42 MistralToolParser,
43 )
44
@@ -1578,382 +1572,3 @@ def test_grammar_from_tool_parser_set_by_adjust_request(
1572 request = _make_request()
1573 result = mistral_tool_parser.adjust_request(request)
1574 assert result._grammar_from_tool_parser is True
1581 -
1582 -
1583 -@pytest.mark.parametrize(
1584 - "tool_calls, expected_len",
1585 - [
1586 - (None, 0),
1587 - ([], 0),
1588 - ([VllmFunctionCall(id="abc123xyz", name="f", arguments="{}")], 1),
1589 - ([VllmFunctionCall(name="f", arguments="{}")], 1),
1590 - (
1591 - [
1592 - VllmFunctionCall(id="fixed1234", name="a", arguments='{"x": 1}'),
1593 - VllmFunctionCall(name="b", arguments='{"y": 2}'),
1594 - ],
1595 - 2,
1596 - ),
1597 - ],
1598 - ids=["none", "empty", "with_id", "without_id", "mixed"],
1599 -)
1600 -def test_build_non_streaming_tool_calls(
1601 - tool_calls: list[VllmFunctionCall] | None,
1602 - expected_len: int,
1603 -) -> None:
1604 - result = MistralToolParser.build_non_streaming_tool_calls(tool_calls)
1605 - assert len(result) == expected_len
1606 -
1607 - if tool_calls is None:
1608 - return
1609 -
1610 - for i, tc in enumerate(result):
1611 - assert isinstance(tc, MistralToolCall)
1612 - assert tc.type == "function"
1613 -
1614 - input_tc = tool_calls[i]
1615 - if input_tc.id:
1616 - assert tc.id == input_tc.id
1617 - else:
1618 - assert len(tc.id) == 9
1619 - assert tc.id.isalnum()
1620 -
1621 - assert tc.function.name == input_tc.name
1622 - assert tc.function.arguments == input_tc.arguments
1623 -
1624 -
1625 -class TestExtractMaybeReasoningAndToolStreaming:
1626 - r"""Tests for `MistralToolParser.extract_maybe_reasoning_and_tool_streaming`."""
1627 -
1628 - @pytest.fixture
1629 - def parser(self) -> MistralToolParser:
1630 - mock_tokenizer = MagicMock()
1631 - mock_tokenizer.get_vocab.return_value = {"[TOOL_CALLS]": 1}
1632 - return MistralToolParser(mock_tokenizer)
1633 -
1634 - @pytest.fixture
1635 - def request_obj(self) -> ChatCompletionRequest:
1636 - return _make_request()
1637 -
1638 - @staticmethod
1639 - def _call(
1640 - parser: MistralToolParser,
1641 - request: ChatCompletionRequest,
1642 - *,
1643 - reasoning_parser: Any = None,
1644 - previous_text: str = "",
1645 - current_text: str = "hello",
1646 - delta_text: str = "hello",
1647 - previous_token_ids: list[int] | None = None,
1648 - current_token_ids: list[int] | None = None,
1649 - output_token_ids: list[int] | None = None,
1650 - reasoning_ended: bool = False,
1651 - prompt_is_reasoning_end: bool | None = None,
1652 - ) -> MistralStreamingResult:
1653 - return parser.extract_maybe_reasoning_and_tool_streaming(
1654 - reasoning_parser=reasoning_parser,
1655 - previous_text=previous_text,
1656 - current_text=current_text,
1657 - delta_text=delta_text,
1658 - previous_token_ids=previous_token_ids or [],
1659 - current_token_ids=current_token_ids or [1, 2, 3],
1660 - output_token_ids=output_token_ids or [1, 2, 3],
1661 - reasoning_ended=reasoning_ended,
1662 - prompt_is_reasoning_end=prompt_is_reasoning_end,
1663 - request=request,
1664 - )
1665 -
1666 - def test_no_reasoning_tools_called(
1667 - self, parser: MistralToolParser, request_obj: ChatCompletionRequest
1668 - ) -> None:
1669 - tool_delta = DeltaMessage(
1670 - tool_calls=[
1671 - DeltaToolCall(
1672 - index=0,
1673 - function=DeltaFunctionCall(name="f", arguments="{}"),
1674 - )
1675 - ]
1676 - )
1677 - with patch.object(
1678 - parser, "extract_tool_calls_streaming", return_value=tool_delta
1679 - ):
1680 - result = self._call(parser, request_obj, reasoning_parser=None)
1681 -
1682 - assert result == MistralStreamingResult(
1683 - delta_message=tool_delta,
1684 - reasoning_ended=False,
1685 - tools_called=True,
1686 - current_text="hello",
1687 - current_token_ids=[1, 2, 3],
1688 - )
1689 -
1690 - def test_no_reasoning_no_tools(
1691 - self, parser: MistralToolParser, request_obj: ChatCompletionRequest
1692 - ) -> None:
1693 - content_delta = DeltaMessage(content="hello")
1694 - with patch.object(
1695 - parser, "extract_tool_calls_streaming", return_value=content_delta
1696 - ):
1697 - result = self._call(parser, request_obj, reasoning_parser=None)
1698 -
1699 - assert result == MistralStreamingResult(
1700 - delta_message=content_delta,
1701 - reasoning_ended=False,
1702 - tools_called=False,
1703 - current_text="hello",
1704 - current_token_ids=[1, 2, 3],
1705 - )
1706 -
1707 - def test_mistral_reasoning_parser_no_think_token(
1708 - self, parser: MistralToolParser, request_obj: ChatCompletionRequest
1709 - ) -> None:
1710 - mock_rp = MagicMock(spec=MistralReasoningParser)
1711 - mock_rp.start_token_id = 999
1712 - content_delta = DeltaMessage(content="direct")
1713 - with patch.object(
1714 - parser, "extract_tool_calls_streaming", return_value=content_delta
1715 - ):
1716 - result = self._call(
1717 - parser,
1718 - request_obj,
1719 - reasoning_parser=mock_rp,
1720 - reasoning_ended=False,
1721 - current_token_ids=[1, 2, 3],
1722 - )
1723 -
1724 - mock_rp.extract_reasoning_streaming.assert_not_called()
1725 - assert result == MistralStreamingResult(
1726 - delta_message=content_delta,
1727 - reasoning_ended=False,
1728 - tools_called=False,
1729 - current_text="hello",
1730 - current_token_ids=[1, 2, 3],
1731 - )
1732 -
1733 - def test_mistral_reasoning_parser_with_think_token(
1734 - self, parser: MistralToolParser, request_obj: ChatCompletionRequest
1735 - ) -> None:
1736 - mock_rp = MagicMock(spec=MistralReasoningParser)
1737 - mock_rp.start_token_id = 999
1738 - mock_rp.extract_reasoning_streaming.return_value = DeltaMessage(
1739 - reasoning="thinking..."
1740 - )
1741 - mock_rp.is_reasoning_end_streaming.return_value = False
1742 -
1743 - result = self._call(
1744 - parser,
1745 - request_obj,
1746 - reasoning_parser=mock_rp,
1747 - reasoning_ended=False,
1748 - current_token_ids=[1, 999, 3],
1749 - )
1750 -
1751 - mock_rp.extract_reasoning_streaming.assert_called_once()
1752 - assert result == MistralStreamingResult(
1753 - delta_message=DeltaMessage(reasoning="thinking..."),
1754 - reasoning_ended=False,
1755 - tools_called=False,
1756 - current_text="hello",
1757 - current_token_ids=[1, 999, 3],
1758 - )
1759 -
1760 - def test_non_mistral_reasoning_parser_always_expects_thinking(
1761 - self, parser: MistralToolParser, request_obj: ChatCompletionRequest
1762 - ) -> None:
1763 - mock_rp = MagicMock()
1764 - mock_rp.start_token_id = 999
1765 - mock_rp.extract_reasoning_streaming.return_value = DeltaMessage(
1766 - reasoning="thinking..."
1767 - )
1768 - mock_rp.is_reasoning_end_streaming.return_value = False
1769 -
1770 - result = self._call(
1771 - parser,
1772 - request_obj,
1773 - reasoning_parser=mock_rp,
1774 - reasoning_ended=False,
1775 - current_token_ids=[1, 2, 3],
1776 - )
1777 -
1778 - mock_rp.extract_reasoning_streaming.assert_called_once()
1779 - assert result == MistralStreamingResult(
1780 - delta_message=DeltaMessage(reasoning="thinking..."),
1781 - reasoning_ended=False,
1782 - tools_called=False,
1783 - current_text="hello",
1784 - current_token_ids=[1, 2, 3],
1785 - )
1786 -
1787 - def test_reasoning_already_ended_no_reset(
1788 - self, parser: MistralToolParser, request_obj: ChatCompletionRequest
1789 - ) -> None:
1790 - content_delta = DeltaMessage(content="content")
1791 - with patch.object(
1792 - parser, "extract_tool_calls_streaming", return_value=content_delta
1793 - ) as mock_extract:
1794 - result = self._call(
1795 - parser,
1796 - request_obj,
1797 - reasoning_parser=MagicMock(),
1798 - reasoning_ended=True,
1799 - previous_text="prior_tool_text",
1800 - previous_token_ids=[10, 20],
1801 - current_text="prior_tool_texthello",
1802 - current_token_ids=[10, 20, 1, 2, 3],
1803 - )
1804 -
1805 - _, call_kwargs = mock_extract.call_args
1806 - assert call_kwargs["previous_text"] == "prior_tool_text"
1807 - assert call_kwargs["previous_token_ids"] == [10, 20]
1808 -
1809 - assert result == MistralStreamingResult(
1810 - delta_message=content_delta,
1811 - reasoning_ended=True,
1812 - tools_called=False,
1813 - current_text="prior_tool_texthello",
1814 - current_token_ids=[10, 20, 1, 2, 3],
1815 - )
1816 -
1817 - def test_pre_v15_ignores_prompt_reasoning_end(
1818 - self, parser: MistralToolParser, request_obj: ChatCompletionRequest
1819 - ) -> None:
1820 - mock_tokenizer = MagicMock(spec=MistralTokenizer)
1821 - mock_tokenizer.version = 13
1822 - parser.model_tokenizer = mock_tokenizer
1823 -
1824 - mock_rp = MagicMock(spec=MistralReasoningParser)
1825 - mock_rp.start_token_id = 999
1826 - mock_rp.extract_reasoning_streaming.return_value = DeltaMessage(
1827 - reasoning="thinking..."
1828 - )
1829 - mock_rp.is_reasoning_end_streaming.return_value = False
1830 -
1831 - result = self._call(
1832 - parser,
1833 - request_obj,
1834 - reasoning_parser=mock_rp,
1835 - reasoning_ended=False,
1836 - prompt_is_reasoning_end=True,
1837 - current_token_ids=[999, 1, 2],
1838 - )
1839 -
1840 - mock_rp.extract_reasoning_streaming.assert_called_once()
1841 - assert result == MistralStreamingResult(
1842 - delta_message=DeltaMessage(reasoning="thinking..."),
1843 - reasoning_ended=False,
1844 - tools_called=False,
1845 - current_text="hello",
1846 - current_token_ids=[999, 1, 2],
1847 - )
1848 -
1849 - def test_non_pre_v15_prompt_reasoning_end(
1850 - self, parser: MistralToolParser, request_obj: ChatCompletionRequest
1851 - ) -> None:
1852 - mock_tokenizer = MagicMock(spec=MistralTokenizer)
1853 - mock_tokenizer.version = 15
1854 - parser.model_tokenizer = mock_tokenizer
1855 -
1856 - mock_rp = MagicMock(spec=MistralReasoningParser)
1857 - mock_rp.start_token_id = 999
1858 -
1859 - content_delta = DeltaMessage(content="after reasoning")
1860 - with patch.object(
1861 - parser, "extract_tool_calls_streaming", return_value=content_delta
1862 - ):
1863 - result = self._call(
1864 - parser,
1865 - request_obj,
1866 - reasoning_parser=mock_rp,
1867 - reasoning_ended=False,
1868 - prompt_is_reasoning_end=True,
1869 - current_token_ids=[999, 1, 2],
1870 - output_token_ids=[10, 20, 30],
1871 - )
1872 -
1873 - mock_rp.extract_reasoning_streaming.assert_not_called()
1874 - assert result == MistralStreamingResult(
1875 - delta_message=content_delta,
1876 - reasoning_ended=True,
1877 - tools_called=False,
1878 - current_text="hello",
1879 - current_token_ids=[10, 20, 30],
1880 - )
1881 -
1882 - def test_reasoning_end_transition_with_content(
1883 - self, parser: MistralToolParser, request_obj: ChatCompletionRequest
1884 - ) -> None:
1885 - """When reasoning ends and the delta has content, that content is
1886 - cleared from delta_message and used as current_text for tool parsing."""
1887 - mock_rp = MagicMock()
1888 - mock_rp.start_token_id = 999
1889 - mock_rp.extract_reasoning_streaming.return_value = DeltaMessage(
1890 - reasoning="think", content="leftover"
1891 - )
1892 - mock_rp.is_reasoning_end_streaming.return_value = True
1893 - mock_rp.extract_content_ids.return_value = [50, 51]
1894 -
1895 - content_delta = DeltaMessage(content="leftover")
1896 - with patch.object(
1897 - parser, "extract_tool_calls_streaming", return_value=content_delta
1898 - ) as mock_extract:
1899 - result = self._call(
1900 - parser,
1901 - request_obj,
1902 - reasoning_parser=mock_rp,
1903 - reasoning_ended=False,
1904 - current_token_ids=[999, 1, 2],
1905 - output_token_ids=[10, 20, 30],
1906 - )
1907 -
1908 - mock_rp.extract_content_ids.assert_called_once_with([10, 20, 30])
1909 - _, call_kwargs = mock_extract.call_args
1910 - assert call_kwargs["previous_text"] == ""
1911 - assert call_kwargs["previous_token_ids"] == []
1912 - assert call_kwargs["delta_text"] == "leftover"
1913 - assert call_kwargs["current_token_ids"] == [50, 51]
1914 -
1915 - assert result == MistralStreamingResult(
1916 - delta_message=content_delta,
1917 - reasoning_ended=True,
1918 - tools_called=False,
1919 - current_text="leftover",
1920 - current_token_ids=[50, 51],
1921 - )
1922 -
1923 - def test_reasoning_end_transition_without_content(
1924 - self, parser: MistralToolParser, request_obj: ChatCompletionRequest
1925 - ) -> None:
1926 - """When reasoning ends but the delta has no content, current_text
1927 - is set to empty string."""
1928 - mock_rp = MagicMock()
1929 - mock_rp.start_token_id = 999
1930 - mock_rp.extract_reasoning_streaming.return_value = DeltaMessage(
1931 - reasoning="think"
1932 - )
1933 - mock_rp.is_reasoning_end_streaming.return_value = True
1934 - mock_rp.extract_content_ids.return_value = [50, 51]
1935 -
1936 - empty_delta = DeltaMessage(content="")
1937 - with patch.object(
1938 - parser, "extract_tool_calls_streaming", return_value=empty_delta
1939 - ) as mock_extract:
1940 - result = self._call(
1941 - parser,
1942 - request_obj,
1943 - reasoning_parser=mock_rp,
1944 - reasoning_ended=False,
1945 - current_token_ids=[999, 1, 2],
1946 - output_token_ids=[10, 20, 30],
1947 - )
1948 -
1949 - _, call_kwargs = mock_extract.call_args
1950 - assert call_kwargs["delta_text"] == ""
1951 - assert call_kwargs["current_token_ids"] == [50, 51]
1952 -
1953 - assert result == MistralStreamingResult(
1954 - delta_message=empty_delta,
1955 - reasoning_ended=True,
1956 - tools_called=False,
1957 - current_text="",
1958 - current_token_ids=[50, 51],
1959 - )
vllm/entrypoints/openai/chat_completion/serving.py
+35 -160
@@ -72,7 +72,7 @@ from vllm.renderers import ChatParams
72 from vllm.sampling_params import BeamSearchParams, SamplingParams
73 from vllm.tokenizers import TokenizerLike
74 from vllm.utils.collection_utils import as_list
75 -from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
75 +from vllm.utils.mistral import is_mistral_tool_parser
76
77 if TYPE_CHECKING:
78 from vllm.entrypoints.serve.render.serving import OpenAIServingRender
@@ -425,8 +425,6 @@ class OpenAIServingChat(OpenAIServing):
425 harmony_tools_streamed = [False] * num_choices
426 tools_streamed = [False] * num_choices
427
428 - is_mistral_grammar_path = request._grammar_from_tool_parser
429 -
428 if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
429 tool_choice_function_name = request.tool_choice.function.name
430 else:
@@ -450,15 +448,12 @@ class OpenAIServingChat(OpenAIServing):
448 # Only one of these will be used, thus previous_texts and
449 # all_previous_token_ids will not be used twice in the same iteration.
450 if (
453 - is_mistral_grammar_path
454 - or tool_choice_auto
451 + tool_choice_auto
452 or tool_choice_function_name
453 or request.tool_choice == "required"
454 or reasoning_parser
455 ):
456 all_previous_token_ids = [[] for _ in range(num_choices)]
460 - reasoning_end_arr = [False] * num_choices
461 - prompt_is_reasoning_end_arr: list[bool | None] = [None] * num_choices
457 else:
458 all_previous_token_ids = None
459
@@ -592,18 +587,6 @@ class OpenAIServingChat(OpenAIServing):
587 for output in res.outputs:
588 i = output.index
589 parser = parsers[i]
595 - tool_parser = parser.tool_parser if parser is not None else None
596 -
597 - if (
598 - reasoning_parser
599 - and res.prompt_token_ids
600 - and prompt_is_reasoning_end_arr[i] is None
601 - ):
602 - # only check once per choice, because prompt_token_ids
603 - # are the same for all deltas in that choice
604 - prompt_is_reasoning_end_arr[i] = (
605 - reasoning_parser.is_reasoning_end(res.prompt_token_ids)
606 - )
590 if finish_reason_sent[i]:
591 continue
592
@@ -658,8 +641,7 @@ class OpenAIServingChat(OpenAIServing):
641
642 # just update previous_texts and previous_token_ids
643 if (
661 - is_mistral_grammar_path
662 - or tool_choice_auto
644 + tool_choice_auto
645 or tool_choice_function_name
646 or request.tool_choice == "required"
647 or reasoning_parser
@@ -687,35 +669,6 @@ class OpenAIServingChat(OpenAIServing):
669 )
670 )
671 harmony_tools_streamed[i] |= tools_streamed_flag
690 - # Mistral grammar path: combined reasoning + tool streaming
691 - elif is_mistral_grammar_path:
692 - from vllm.tool_parsers.mistral_tool_parser import (
693 - MistralToolParser,
694 - )
695 -
696 - assert tool_parser is not None
697 - assert isinstance(tool_parser, MistralToolParser)
698 - assert reasoning_end_arr is not None
699 - output_token_ids = as_list(output.token_ids)
700 - result = tool_parser.extract_maybe_reasoning_and_tool_streaming(
701 - reasoning_parser=reasoning_parser,
702 - previous_text=previous_text,
703 - current_text=current_text,
704 - delta_text=delta_text,
705 - previous_token_ids=previous_token_ids,
706 - current_token_ids=current_token_ids,
707 - output_token_ids=output_token_ids,
708 - reasoning_ended=reasoning_end_arr[i],
709 - prompt_is_reasoning_end=(prompt_is_reasoning_end_arr[i]),
710 - request=request,
711 - )
712 - delta_message = result.delta_message
713 - reasoning_end_arr[i] = result.reasoning_ended
714 - current_text = result.current_text
715 - current_token_ids = result.current_token_ids
716 - if result.tools_called:
717 - tools_streamed[i] = True
718 -
672 elif parser is not None:
673 delta_message = parser.parse_delta(
674 delta_text=delta_text,
@@ -732,8 +685,7 @@ class OpenAIServingChat(OpenAIServing):
685
686 # update the previous values for the next iteration
687 if (
735 - is_mistral_grammar_path
736 - or tool_choice_auto
688 + tool_choice_auto
689 or tool_choice_function_name
690 or request.tool_choice == "required"
691 or reasoning_parser
@@ -1067,32 +1019,8 @@ class OpenAIServingChat(OpenAIServing):
1019 tool_calls = []
1020
1021 auto_tools_called = False
1070 - if is_mistral_tokenizer(tokenizer):
1071 - from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
1022
1073 - tool_call_class: type[ToolCall] = MistralToolCall
1074 - else:
1075 - tool_call_class = ToolCall
1076 -
1077 - use_mistral_tool_parser = request._grammar_from_tool_parser
1078 - if use_mistral_tool_parser:
1079 - from vllm.tool_parsers.mistral_tool_parser import MistralToolParser
1080 -
1081 - tool_call_items = MistralToolParser.build_non_streaming_tool_calls(
1082 - tool_calls
1083 - )
1084 - if tool_call_items:
1085 - auto_tools_called = (
1086 - request.tool_choice is None or request.tool_choice == "auto"
1087 - )
1088 - message = ChatMessage(
1089 - role=role,
1090 - reasoning=reasoning,
1091 - content=content,
1092 - tool_calls=tool_call_items,
1093 - )
1094 -
1095 - elif (not self.enable_auto_tools or not self.tool_parser) and (
1023 + if (not self.enable_auto_tools or not self.tool_parser) and (
1024 not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
1025 and request.tool_choice != "required"
1026 ):
@@ -1102,70 +1030,42 @@ class OpenAIServingChat(OpenAIServing):
1030 request.tool_choice
1031 and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
1032 ):
1105 - tool_call_class_items = []
1033 + tool_call_items = []
1034 tool_calls = tool_calls or []
1107 - for idx, tc in enumerate(tool_calls):
1108 - # Use native ID if available (e.g., Kimi K2),
1109 - # otherwise generate ID with correct id_type
1110 - if tc.id:
1111 - tool_call_class_items.append(
1112 - tool_call_class(id=tc.id, function=tc)
1035 + for tc in tool_calls:
1036 + if not tc.id:
1037 + tc.id = make_tool_call_id(
1038 + id_type=self.tool_call_id_type,
1039 + func_name=tc.name,
1040 + idx=history_tool_call_cnt,
1041 )
1114 - else:
1115 - # Generate ID using the correct format (kimi_k2 or random),
1116 - # but leave it to the class if it's Mistral to preserve
1117 - # 9-char IDs
1118 - if is_mistral_tokenizer(tokenizer):
1119 - tool_call_class_items.append(tool_call_class(function=tc))
1120 - else:
1121 - generated_id = make_tool_call_id(
1122 - id_type=self.tool_call_id_type,
1123 - func_name=tc.name,
1124 - idx=history_tool_call_cnt,
1125 - )
1126 - tool_call_class_items.append(
1127 - tool_call_class(id=generated_id, function=tc)
1128 - )
1042 + tool_call_items.append(ToolCall(id=tc.id, function=tc))
1043 history_tool_call_cnt += 1
1044 message = ChatMessage(
1045 role=role,
1046 reasoning=reasoning,
1133 - content="",
1134 - tool_calls=tool_call_class_items,
1047 + content=content or "",
1048 + tool_calls=tool_call_items,
1049 )
1050
1051 elif request.tool_choice and request.tool_choice == "required":
1138 - tool_call_class_items = []
1052 + tool_call_items = []
1053 tool_calls = tool_calls or []
1140 - for idx, tool_call in enumerate(tool_calls):
1141 - # Use native ID if available,
1142 - # otherwise generate ID with correct id_type
1143 - if tool_call.id:
1144 - tool_call_class_items.append(
1145 - tool_call_class(id=tool_call.id, function=tool_call)
1054 + for tool_call in tool_calls:
1055 + if not tool_call.id:
1056 + tool_call.id = make_tool_call_id(
1057 + id_type=self.tool_call_id_type,
1058 + func_name=tool_call.name,
1059 + idx=history_tool_call_cnt,
1060 )
1147 - else:
1148 - # Generate ID using the correct format (kimi_k2 or random),
1149 - # but leave it to the class if it's Mistral to preserve
1150 - # 9-char IDs
1151 - if is_mistral_tokenizer(tokenizer):
1152 - tool_call_class_items.append(
1153 - tool_call_class(function=tool_call)
1154 - )
1155 - else:
1156 - generated_id = make_tool_call_id(
1157 - id_type=self.tool_call_id_type,
1158 - func_name=tool_call.name,
1159 - idx=history_tool_call_cnt,
1160 - )
1161 - tool_call_class_items.append(
1162 - tool_call_class(id=generated_id, function=tool_call)
1163 - )
1061 + tool_call_items.append(
1062 + ToolCall(id=tool_call.id, function=tool_call)
1063 + )
1064 history_tool_call_cnt += 1
1065 message = ChatMessage(
1066 role=role,
1167 - content="",
1168 - tool_calls=tool_call_class_items,
1067 + content=content or "",
1068 + tool_calls=tool_call_items,
1069 reasoning=reasoning,
1070 )
1071
@@ -1181,34 +1081,17 @@ class OpenAIServingChat(OpenAIServing):
1081 and self.enable_auto_tools
1082 and self.tool_parser
1083 ):
1184 - # In the OpenAI API the finish_reason is "tools_called"
1185 - # if the tool choice is auto and the model produced a tool
1186 - # call. The same is not true for named function calls
1084 auto_tools_called = tool_calls is not None and len(tool_calls) > 0
1085 if tool_calls:
1086 tool_call_items = []
1190 - for idx, tc in enumerate(tool_calls):
1191 - # Use native ID if available (e.g., Kimi K2),
1192 - # otherwise generate ID with correct id_type
1193 - if tc.id:
1194 - tool_call_items.append(
1195 - tool_call_class(id=tc.id, function=tc)
1087 + for tc in tool_calls:
1088 + if not tc.id:
1089 + tc.id = make_tool_call_id(
1090 + id_type=self.tool_call_id_type,
1091 + func_name=tc.name,
1092 + idx=history_tool_call_cnt,
1093 )
1197 - else:
1198 - # Generate ID using the correct format (kimi_k2 or random),
1199 - # but leave it to the class if it's Mistral to preserve
1200 - # 9-char IDs
1201 - if is_mistral_tokenizer(tokenizer):
1202 - tool_call_items.append(tool_call_class(function=tc))
1203 - else:
1204 - generated_id = make_tool_call_id(
1205 - id_type=self.tool_call_id_type,
1206 - func_name=tc.name,
1207 - idx=history_tool_call_cnt,
1208 - )
1209 - tool_call_items.append(
1210 - tool_call_class(id=generated_id, function=tc)
1211 - )
1094 + tool_call_items.append(ToolCall(id=tc.id, function=tc))
1095 history_tool_call_cnt += 1
1096 message = ChatMessage(
1097 role=role,
@@ -1218,18 +1101,10 @@ class OpenAIServingChat(OpenAIServing):
1101 )
1102
1103 else:
1221 - # FOR NOW make it a chat message; we will have to detect
1222 - # the type to make it later.
1223 - ret_content = content
1224 -
1225 - # try to use content return from tool parser first,
1226 - # tool parser may do some modify for the content.
1227 - if content and len(content) > 0:
1228 - ret_content = content
1104 message = ChatMessage(
1105 role=role,
1106 reasoning=reasoning,
1232 - content=ret_content,
1107 + content=content,
1108 )
1109
1110 # undetermined case that is still important to handle
vllm/parser/abstract_parser.py
+3 -20
@@ -43,7 +43,6 @@ from vllm.tool_parsers.streaming import (
43 extract_required_tool_call_streaming,
44 )
45 from vllm.utils import random_uuid
46 -from vllm.utils.mistral import is_mistral_tool_parser
46
47 logger = init_logger(__name__)
48
@@ -546,14 +545,6 @@ class DelegatingParser(Parser):
545 if tool_parser is None:
546 return [], content
547
549 - # When the Mistral grammar factory injected structured outputs,
550 - # let the parser handle the output.
551 - use_mistral_tool_parser = (
552 - is_mistral_tool_parser(type(tool_parser))
553 - and isinstance(request, ChatCompletionRequest)
554 - and request._grammar_from_tool_parser
555 - )
556 -
548 supports_required_and_named = tool_parser.supports_required_and_named
549 is_named_tool_choice = request.tool_choice and isinstance(
550 request.tool_choice,
@@ -570,11 +561,7 @@ class DelegatingParser(Parser):
561 )
562
563 tool_calls = list[FunctionCall]()
573 - if (
574 - is_named_tool_choice
575 - and supports_required_and_named
576 - and not use_mistral_tool_parser
577 - ):
564 + if is_named_tool_choice and supports_required_and_named:
565 if content is None:
566 return [], None
567 tool_calls.append(
@@ -584,11 +571,7 @@ class DelegatingParser(Parser):
571 )
572 )
573 content = None
587 - elif (
588 - is_required_tool_choice
589 - and supports_required_and_named
590 - and not use_mistral_tool_parser
591 - ):
574 + elif is_required_tool_choice and supports_required_and_named:
575 # "required" with standard JSON-based parsing
576 parsed_calls = []
577 with contextlib.suppress(ValidationError):
@@ -604,7 +587,7 @@ class DelegatingParser(Parser):
587 )
588 )
589 content = None
607 - elif is_auto_tool_choice or use_mistral_tool_parser:
590 + elif is_auto_tool_choice:
591 # Automatic Tool Call Parsing (also used as fallback for
592 # required/named when supports_required_and_named=False)
593 tool_call_info = tool_parser.extract_tool_calls(
vllm/parser/mistral.py new
+77
@@ -0,0 +1,77 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from __future__ import annotations
5 +
6 +from typing import TYPE_CHECKING
7 +
8 +from vllm.entrypoints.openai.engine.protocol import DeltaMessage, FunctionCall
9 +from vllm.parser.abstract_parser import DelegatingParser
10 +
11 +if TYPE_CHECKING:
12 + from vllm.entrypoints.openai.chat_completion.protocol import (
13 + ChatCompletionRequest,
14 + )
15 + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
16 +
17 +
18 +class MistralParser(DelegatingParser):
19 + def __init__(self, tokenizer, tools=None, *args, **kwargs):
20 + super().__init__(tokenizer, tools, *args, **kwargs)
21 + from vllm.tool_parsers.mistral_tool_parser import MistralToolParser
22 +
23 + if not isinstance(self._tool_parser, MistralToolParser):
24 + raise ValueError(
25 + "MistralParser requires --tool-call-parser mistral, "
26 + f"got {self._tool_parser.__class__.__name__}."
27 + )
28 +
29 + def _maybe_force_auto_tool_parsing(
30 + self, request: ChatCompletionRequest | ResponsesRequest
31 + ) -> None:
32 + # When the Mistral grammar factory injected structured outputs,
33 + # the model emits v11+ format ([TOOL_CALLS]name{args}) that the
34 + # named/required parsers can't handle. Disable them so all
35 + # tool_choice modes fall back to auto tool parsing via
36 + # extract_tool_calls.
37 + if getattr(request, "_grammar_from_tool_parser", False):
38 + assert self._tool_parser is not None
39 + self._tool_parser.supports_required_and_named = False
40 +
41 + def parse(
42 + self,
43 + model_output: str,
44 + request: ChatCompletionRequest | ResponsesRequest,
45 + enable_auto_tools: bool = False,
46 + ) -> tuple[str | None, str | None, list[FunctionCall] | None]:
47 + self._maybe_force_auto_tool_parsing(request)
48 + reasoning, content, tool_calls = super().parse(
49 + model_output, request, enable_auto_tools
50 + )
51 + if tool_calls:
52 + from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
53 +
54 + # Named/required tool_choice builds FunctionCalls without
55 + # ID, backfill with Mistral-format IDs.
56 + for tc in tool_calls:
57 + if not tc.id:
58 + tc.id = MistralToolCall.generate_random_id()
59 + return reasoning, content, tool_calls
60 +
61 + def parse_delta(
62 + self,
63 + delta_text: str,
64 + delta_token_ids: list[int],
65 + request: ChatCompletionRequest | ResponsesRequest,
66 + prompt_token_ids: list[int] | None = None,
67 + *,
68 + finished: bool,
69 + ) -> DeltaMessage | None:
70 + self._maybe_force_auto_tool_parsing(request)
71 + return super().parse_delta(
72 + delta_text,
73 + delta_token_ids,
74 + request,
75 + prompt_token_ids,
76 + finished=finished,
77 + )
vllm/parser/parser_manager.py
+9
@@ -106,6 +106,15 @@ class ParserManager:
106 if reasoning_parser_cls is None and tool_parser_cls is None:
107 return None
108
109 + from vllm.utils.mistral import is_mistral_tool_parser
110 +
111 + if is_mistral_tool_parser(tool_parser_cls):
112 + from vllm.parser.mistral import MistralParser
113 +
114 + MistralParser.reasoning_parser_cls = reasoning_parser_cls
115 + MistralParser.tool_parser_cls = tool_parser_cls
116 + return MistralParser
117 +
118 from vllm.parser.abstract_parser import DelegatingParser
119
120 r_cls = reasoning_parser_cls
vllm/reasoning/mistral_reasoning_parser.py
+10 -1
@@ -1,7 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -from collections.abc import Sequence
4 +from collections.abc import Iterable, Sequence
5 from functools import cached_property
6 from typing import TYPE_CHECKING
7
@@ -76,6 +76,15 @@ class MistralReasoningParser(BaseThinkingReasoningParser):
76 has_eot_token = True
77 return False
78
79 + def is_reasoning_end_streaming(
80 + self, input_ids: Sequence[int], delta_ids: Iterable[int]
81 + ) -> bool:
82 + if self.end_token_id in delta_ids:
83 + return True
84 + # Grammar's think? is optional — if [THINK] was never generated,
85 + # reasoning was skipped entirely.
86 + return self.start_token_id not in input_ids
87 +
88 def extract_content_ids(self, input_ids: list[int]) -> list[int]:
89 """
90 Extract the content
vllm/tool_parsers/mistral_tool_parser.py
+2 -163
@@ -5,11 +5,10 @@ from __future__ import annotations
5
6 import json
7 from collections.abc import Sequence
8 -from dataclasses import dataclass
8 from enum import Enum, auto
9 from random import choices
10 from string import ascii_letters, digits
12 -from typing import TYPE_CHECKING, Any
11 +from typing import Any
12
13 import ijson
14 import regex as re
@@ -40,19 +39,14 @@ from vllm.entrypoints.openai.engine.protocol import (
39 )
40 from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
41 from vllm.logger import init_logger
43 -from vllm.reasoning.mistral_reasoning_parser import MistralReasoningParser
42 from vllm.sampling_params import StructuredOutputsParams
43 from vllm.tokenizers import TokenizerLike
46 -from vllm.tokenizers.mistral import MistralTokenizer
44 from vllm.tool_parsers.abstract_tool_parser import (
45 Tool,
46 ToolParser,
47 )
48 from vllm.utils.mistral import is_mistral_tokenizer
49
53 -if TYPE_CHECKING:
54 - from vllm.reasoning import ReasoningParser
55 -
50 logger = init_logger(__name__)
51
52 ALPHANUMERIC = ascii_letters + digits
@@ -99,19 +93,6 @@ def _is_pre_v11_tokeniser(model_tokenizer: TokenizerLike) -> bool:
93 return "[ARGS]" not in vocab
94
95
102 -@dataclass
103 -class MistralStreamingResult:
104 - r"""Encapsulates the mutable state returned from
105 - `MistralToolParser.extract_maybe_reasoning_and_tool_streaming`.
106 - """
107 -
108 - delta_message: DeltaMessage | None
109 - reasoning_ended: bool
110 - tools_called: bool
111 - current_text: str
112 - current_token_ids: list[int]
113 -
114 -
96 class MistralToolParser(ToolParser):
97 r"""Tool call parser for Mistral models, intended for use with either:
98
@@ -281,148 +262,6 @@ class MistralToolParser(ToolParser):
262 request._grammar_from_tool_parser = True
263 return request
264
284 - def extract_maybe_reasoning_and_tool_streaming(
285 - self,
286 - *,
287 - reasoning_parser: ReasoningParser | None,
288 - previous_text: str,
289 - current_text: str,
290 - delta_text: str,
291 - previous_token_ids: list[int],
292 - current_token_ids: list[int],
293 - output_token_ids: Sequence[int],
294 - reasoning_ended: bool,
295 - prompt_is_reasoning_end: bool | None,
296 - request: ChatCompletionRequest,
297 - ) -> MistralStreamingResult:
298 - r"""Streaming extraction with reasoning followed by tool-call parsing.
299 -
300 - This method encapsulates the combined reasoning extraction and
301 - tool-call streaming logic so that the serving layer only needs a
302 - thin routing branch.
303 -
304 - The flow is:
305 -
306 - 1. If a *reasoning_parser* is present and reasoning has **not** ended,
307 - extract reasoning tokens. Pre-v15 models may have pre-filled
308 - `[THINK]...[/THINK]` in system prompts, so we skip the
309 - prompt-level reasoning-end check for those.
310 - 2. Once reasoning ends (or if there is no reasoning parser), delegate
311 - to `extract_tool_calls_streaming` and track whether tools were
312 - called.
313 -
314 - Args:
315 - reasoning_parser: Optional reasoning parser instance.
316 - previous_text: Accumulated text from prior chunks.
317 - current_text: Full accumulated text including current chunk.
318 - delta_text: New text in this chunk.
319 - previous_token_ids: Token ids from prior chunks.
320 - current_token_ids: Full token ids including current chunk.
321 - output_token_ids: Raw output token ids from the engine.
322 - reasoning_ended: Whether reasoning has already ended.
323 - prompt_is_reasoning_end: Whether the prompt itself ends reasoning.
324 - request: The originating chat completion request.
325 - """
326 - delta_message: DeltaMessage | None = None
327 - tools_called = False
328 - reasoning_ended_at_entry = reasoning_ended
329 -
330 - # For MistralReasoningParser, only enter the reasoning block when
331 - # the model has actually emitted a [THINK] token. Other reasoning
332 - # parsers always expect thinking to be present.
333 - expect_thinking = (
334 - not isinstance(reasoning_parser, MistralReasoningParser)
335 - or reasoning_parser.start_token_id in current_token_ids
336 - )
337 - if reasoning_parser is not None and not reasoning_ended and expect_thinking:
338 - # Pre-v15 models may have pre-filled [THINK]...[/THINK] in
339 - # system prompts, so skip the prompt-level reasoning-end
340 - # check and wait for the output's own end-of-think.
341 - is_pre_v15 = (
342 - isinstance(self.model_tokenizer, MistralTokenizer)
343 - and self.model_tokenizer.version < 15
344 - )
345 -
346 - if not is_pre_v15 and prompt_is_reasoning_end:
347 - reasoning_ended = True
348 - current_token_ids = list(output_token_ids)
349 - else:
350 - delta_message = reasoning_parser.extract_reasoning_streaming(
351 - previous_text,
352 - current_text,
353 - delta_text,
354 - previous_token_ids,
355 - current_token_ids,
356 - output_token_ids,
357 - )
358 - if reasoning_parser.is_reasoning_end_streaming(
359 - current_token_ids, output_token_ids
360 - ):
361 - reasoning_ended = True
362 - current_token_ids = reasoning_parser.extract_content_ids(
363 - list(output_token_ids)
364 - )
365 - if delta_message and delta_message.content:
366 - current_text = delta_message.content
367 - delta_message.content = None
368 - else:
369 - current_text = ""
370 -
371 - if not reasoning_ended:
372 - return MistralStreamingResult(
373 - delta_message=delta_message,
374 - reasoning_ended=False,
375 - tools_called=False,
376 - current_text=current_text,
377 - current_token_ids=current_token_ids,
378 - )
379 -
380 - delta_token_ids = list(output_token_ids)
381 -
382 - # On the iteration where reasoning just ended, reset the text/token
383 - # state so the tool parser sees a clean history instead of the
384 - # accumulated reasoning text.
385 - if not reasoning_ended_at_entry and reasoning_ended:
386 - previous_text = ""
387 - previous_token_ids = []
388 - delta_text = current_text
389 - delta_token_ids = current_token_ids
390 -
391 - delta_message = self.extract_tool_calls_streaming(
392 - previous_text=previous_text,
393 - current_text=current_text,
394 - delta_text=delta_text,
395 - previous_token_ids=previous_token_ids,
396 - current_token_ids=current_token_ids,
397 - delta_token_ids=delta_token_ids,
398 - request=request,
399 - )
400 - if delta_message and delta_message.tool_calls:
401 - tools_called = True
402 -
403 - return MistralStreamingResult(
404 - delta_message=delta_message,
405 - reasoning_ended=reasoning_ended,
406 - tools_called=tools_called,
407 - current_text=current_text,
408 - current_token_ids=current_token_ids,
409 - )
410 -
411 - @staticmethod
412 - def build_non_streaming_tool_calls(
413 - tool_calls: list[FunctionCall] | None,
414 - ) -> list[ToolCall]:
415 - r"""Build `MistralToolCall` items for non-streaming responses."""
416 - if not tool_calls:
417 - return []
418 -
419 - return [
420 - MistralToolCall(id=tc.id, function=tc)
421 - if tc.id
422 - else MistralToolCall(function=tc)
423 - for tc in tool_calls
424 - ]
425 -
265 def extract_tool_calls(
266 self,
267 model_output: str,
@@ -536,7 +375,7 @@ class MistralToolParser(ToolParser):
375 return ExtractedToolCallInformation(
376 tools_called=True,
377 tool_calls=mistral_tool_calls,
539 - content=content if len(content) > 0 else None,
378 + content=content if content.strip() else None,
379 )
380
381 def extract_tool_calls_streaming(